Merge #211

211: Prebuilt Godot artifacts r=Bromeon a=Bromeon

Closes #12 
Closes #107 

From now on, gdext by default fetches pre-generated versions of these files, published in the [`godot4-prebuilt`](https://github.com/godot-rust/godot4-prebuilt) repo:
* `extension_api.json` (from Godot binary)
* `gdextension_interface.h` (from Godot binary)
* `gdextension_interface.rs` (through bindgen -- currently supports 3 platforms)

This has several benefits:
1. Significantly fewer dependencies, as bindgen is no longer needed, and thus smaller compile times.
2. Most CI jobs no longer need the Godot binary (clippy, test), speeding up CI _in addition_ to (1).
3. It's possible to change the Godot API behind gdext without manually rebuilding the artifacts.
4. Easy comparison between the Godot APIs of different released versions.


### Using a custom Godot binary

It is still possible to generate those files locally like before, through the use of the `custom-godot` feature on the `godot` crate.

This is necessary for any platform/configuration different from the 3 main supported ones (because bindgen generates different Rust bindings), as well as any in-development or modified Godot versions.


### Changing the Godot release

By default, the latest Godot release is used as input to gdext. Switching between different Godot versions is easily possible, although a bit cumbersome.

If you want to use an older version `4.0`, add this to your **workspace** (not sub-crate) `Cargo.toml`:
```toml
# We need to trick Cargo into seeing a different URL; rust-lang/cargo#5478
[patch."https://github.com/godot-rust/godot4-prebuilt"]
godot4-prebuilt = { git = "https://github.com//godot-rust/godot4-prebuilt", branch = "4.0"}
```
We're looking into ways to simplify this. In a crates.io release (#2), we would need to rethink this anyway, mapping Godot versions to Rust release versions (which is not trivial).

Co-authored-by: Jan Haller <[email protected]>

Author: bors[bot]

.github/composite/godot-itest/action.yml

@@ -19,6 +19,11 @@ inputs:
     default: ''
     description: "Command-line arguments passed to Godot"
 
+  godot-check-header:
+    required: false
+    default: 'false'
+    description: "Should the job check against latest gdextension_interface.h, and warn on difference"
+
   rust-toolchain:
     required: false
     default: 'stable'
@@ -60,42 +65,45 @@ runs:
         echo "GODOT_BUILT_FROM=_Built from [\`$godotVer\`](https://github.com/godotengine/godot/commit/$gitSha)._" >> $GITHUB_ENV
       shell: bash
 
-    # Note: if this fails, run `git diff -R > tweaks.patch` after updating the file manually
-    - name: "Copy and compare GDExtension header"
-      if: inputs.artifact-name == 'godot-linux'
-      run: |
-        mkdir -p godot-codegen/input
-        cp $RUNNER_DIR/godot_bin/gdextension_interface.h godot-codegen/input/gdextension_interface.h
-        git apply godot-codegen/input/tweak.patch -v
-        git diff --exit-code --quiet || {
-            echo "OUTCOME=header-diff" >> $GITHUB_ENV
-            echo "gdextension_interface.h is not up-to-date; abort."
-            echo ""
-            
-            echo "### :x: Outdated GDExtension API header" >> $GITHUB_STEP_SUMMARY
-            echo "gdextension_interface.h contains the following differences:" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`diff" >> $GITHUB_STEP_SUMMARY
-            git diff >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "After manually updating file, run:   \`git diff -R > tweak.patch\`." >> $GITHUB_STEP_SUMMARY
-
-        	exit 1
-        }
-      shell: bash
-
     - name: "Install Rust"
       uses: ./.github/composite/rust
       with:
         rust: ${{ inputs.rust-toolchain }}
         with-llvm: ${{ inputs.with-llvm }}
 
-    - name: "Build godot-rust"
+    - name: "Build gdext (itest)"
       run: |
         cargo build -p itest ${{ inputs.rust-extra-args }}
       shell: bash
       env:
         RUSTFLAGS: ${{ inputs.rust-env-rustflags }}
 
+    # Note: no longer fails, as we expect header to be forward-compatible; instead issues a warning
+    - name: "Copy and compare GDExtension header"
+      if: inputs.godot-check-header == 'true'
+      run: |
+        mv godot-ffi/src/gen/gdextension_interface.h godot-ffi/src/gen/gdextension_interface_prebuilt.h
+        mv $RUNNER_DIR/godot_bin/gdextension_interface.h godot-ffi/src/gen/gdextension_interface.h
+        git apply godot-bindings/res/tweak.patch
+        cd godot-ffi/src/gen
+        git diff --no-index --exit-code --quiet gdextension_interface_prebuilt.h gdextension_interface.h || {
+            echo "OUTCOME=header-diff" >> $GITHUB_ENV
+            echo "::warning::gdextension_interface.h is not up-to-date."
+            echo ""
+        
+            echo "### :warning: Outdated GDExtension API header" >> $GITHUB_STEP_SUMMARY
+            echo "gdextension_interface.h contains the following differences:" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`diff" >> $GITHUB_STEP_SUMMARY
+            git diff --no-index gdextension_interface_prebuilt.h gdextension_interface.h >> $GITHUB_STEP_SUMMARY || true
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "After manually updating file, run:   \`git diff -R > tweak.patch\`." >> $GITHUB_STEP_SUMMARY
+
+            # Undo modifications
+            mv gdextension_interface_prebuilt.h gdextension_interface.h
+        	#exit 1
+        }
+      shell: bash
+
     - name: "Run Godot integration tests"
       # Aborts immediately if Godot outputs certain keywords (would otherwise stall until CI runner times out).
       # Explanation:
@@ -120,19 +128,20 @@ runs:
       run: |
         if grep -q "ObjectDB instances leaked at exit" "${{ runner.temp }}/log.txt"; then
           echo "OUTCOME=godot-leak" >> $GITHUB_ENV
-          exit 2
+          exit 3
         fi
       shell: bash
 
     - name: "Conclusion"
       if: always()
       run: |
-        echo "Evaluate conclusion ($OUTCOME)"
+        echo "Evaluate conclusion: $OUTCOME"
         
         case $OUTCOME in
           "success")
-            echo "### :heavy_check_mark: Godot integration tests passed" > $GITHUB_STEP_SUMMARY
-            echo "$GODOT_BUILT_FROM" >> $GITHUB_STEP_SUMMARY
+             # Do not output success for now, to keep summary focused on warnings/errors
+             #echo "### :heavy_check_mark: Godot integration tests passed" > $GITHUB_STEP_SUMMARY
+             #echo "$GODOT_BUILT_FROM" >> $GITHUB_STEP_SUMMARY
             ;;
         
           "godot-runtime")

.github/composite/llvm/action.yml

@@ -24,7 +24,7 @@ runs:
     - name: "Cache LLVM and clang"
       id: cache-llvm
       # Note: conditionals not yet supported; see https://github.com/actions/runner/issues/834
-      # if: ${{ inputs.llvm == 'true' }}
+      # if: inputs.llvm == 'true'
       uses: actions/cache@v3
       with:
 #        path: |
@@ -34,7 +34,7 @@ runs:
         key: llvm-10.0
 
     - uses: KyleMayes/install-llvm-action@v1
-      # if: ${{ inputs.llvm == 'true' }}
+      # if: inputs.llvm == 'true'
       with:
         version: "10.0"
         directory: ${{ env.LLVM_INSTALL_DIR }}

.github/composite/rust/action.yml

@@ -61,8 +61,8 @@ runs:
         cache-on-failure: true
 
     - name: "Install LLVM"
+      if: inputs.with-llvm == 'true'
       uses: ./.github/composite/llvm
-      if: ${{ inputs.with-llvm == 'true' }}
 
     - name: "Set environment variables used by toolchain"
       run: |

.github/workflows/full-ci.yml

@@ -42,49 +42,28 @@ jobs:
       - name: "Install Rust"
         uses: ./.github/composite/rust
         with:
-          rust: stable
           components: rustfmt
 
       - name: "Check rustfmt"
         run: cargo fmt --all -- --check
 
 
   clippy:
-    name: clippy (${{ matrix.name }})
     runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        include: 
-          - name: linux
-            rust-toolchain: stable
-            godot-binary: godot.linuxbsd.editor.dev.x86_64
-          
-          - name: linux-double
-            rust-toolchain: stable
-            godot-binary: godot.linuxbsd.editor.dev.double.x86_64
-            rust-extra-args: --features double-precision
     steps:
       - uses: actions/checkout@v3
 
       - name: "Install Rust"
         uses: ./.github/composite/rust
-
-      # TODO get rid of Godot binary, once the JSON is either versioned or fetched from somewhere
-      # Replaces also backspaces on Windows, since they cause problems in Bash
-      - name: "Install Godot"
-        uses: ./.github/composite/godot-install
         with:
-          artifact-name: godot-${{ matrix.name }}
-          godot-binary: ${{ matrix.godot-binary }}
+          components: clippy
 
       - name: "Check clippy"
         run: |
           cargo clippy --all-targets $GDEXT_FEATURES ${{ matrix.rust-extra-args }} -- \
           -D clippy::suspicious -D clippy::style -D clippy::complexity -D clippy::perf \
           -D clippy::dbg_macro -D clippy::todo -D clippy::unimplemented -D warnings 
 
-
   unit-test:
     name: unit-test (${{ matrix.name }}${{ matrix.rust-special }})
     runs-on: ${{ matrix.os }}
@@ -98,28 +77,19 @@ jobs:
         include:
           - name: macos
             os: macos-11
-            rust-toolchain: stable
-            godot-binary: godot.macos.editor.dev.x86_64
-            with-llvm: true
 
           - name: windows
             os: windows-latest
-            rust-toolchain: stable-x86_64-pc-windows-msvc
-            godot-binary: godot.windows.editor.dev.x86_64.exe
 
           # Don't use latest Ubuntu (22.04) as it breaks lots of ecosystem compatibility.
           # If ever moving to ubuntu-latest, need to manually install libtinfo5 for LLVM.
           - name: linux
             os: ubuntu-20.04
-            rust-toolchain: stable
-            godot-binary: godot.linuxbsd.editor.dev.x86_64
 
           - name: linux
             os: ubuntu-20.04
-            rust-toolchain: stable
             rust-special: -minimal-deps
-            godot-binary: godot.linuxbsd.editor.dev.x86_64
-          
+
     steps:
       - uses: actions/checkout@v3
 
@@ -128,26 +98,17 @@ jobs:
         with:
           rust: stable
           cache-key: ${{ matrix.rust-special }} # '-minimal-deps' or empty/not defined
-          with-llvm: ${{ matrix.with-llvm }}
 
       - name: "Install Rust nightly (minimal deps)"
+        if: matrix.rust-special == '-minimal-deps'
         uses: ./.github/composite/rust
         with:
           rust: nightly
           cache-key: minimal-deps-nightly
-        if: ${{ matrix.rust-special == '-minimal-deps' }}
 
       - name: "Install minimal dependency versions from Cargo"
+        if: matrix.rust-special == '-minimal-deps'
         run: cargo +nightly update -Z minimal-versions
-        if: ${{ matrix.rust-special == '-minimal-deps' }}
-
-      # TODO get rid of Godot binary, once the JSON is either versioned or fetched from somewhere
-      # Replaces also backspaces on Windows, since they cause problems in Bash
-      - name: "Install Godot"
-        uses: ./.github/composite/godot-install
-        with:
-          artifact-name: godot-${{ matrix.name }}
-          godot-binary: ${{ matrix.godot-binary }}
 
       - name: "Compile tests"
         run: cargo test $GDEXT_FEATURES --no-run ${{ matrix.rust-extra-args }}
@@ -170,41 +131,52 @@ jobs:
         include:
           - name: macos
             os: macos-12
-            rust-toolchain: stable
             godot-binary: godot.macos.editor.dev.x86_64
-            with-llvm: true
-          
+
           - name: macos-double
             os: macos-12
-            rust-toolchain: stable
             godot-binary: godot.macos.editor.dev.double.x86_64
             rust-extra-args: --features double-precision
+
+          - name: macos-nightly
+            os: macos-12
+            artifact-name: macos
+            godot-binary: godot.macos.editor.dev.x86_64
+            rust-extra-args: --features godot/custom-godot
             with-llvm: true
 
           - name: windows
             os: windows-latest
-            rust-toolchain: stable-x86_64-pc-windows-msvc
             godot-binary: godot.windows.editor.dev.x86_64.exe
 
           - name: windows-double
             os: windows-latest
-            rust-toolchain: stable-x86_64-pc-windows-msvc
             godot-binary: godot.windows.editor.dev.double.x86_64.exe
             rust-extra-args: --features double-precision
 
+          - name: windows-nightly
+            os: windows-latest
+            artifact-name: windows
+            godot-binary: godot.windows.editor.dev.x86_64.exe
+            rust-extra-args: --features godot/custom-godot
+
           # Don't use latest Ubuntu (22.04) as it breaks lots of ecosystem compatibility.
           # If ever moving to ubuntu-latest, need to manually install libtinfo5 for LLVM.
           - name: linux
             os: ubuntu-20.04
-            rust-toolchain: stable
             godot-binary: godot.linuxbsd.editor.dev.x86_64
 
           - name: linux-double
             os: ubuntu-20.04
-            rust-toolchain: stable
             godot-binary: godot.linuxbsd.editor.dev.double.x86_64
             rust-extra-args: --features double-precision
 
+          - name: linux-nightly
+            os: ubuntu-20.04
+            artifact-name: linux
+            godot-binary: godot.linuxbsd.editor.dev.x86_64
+            rust-extra-args: --features godot/custom-godot
+
           # Special Godot binaries compiled with AddressSanitizer/LeakSanitizer to detect UB/leaks.
           # Additionally, the Godot source is patched to make dlclose() a no-op, as unloading dynamic libraries loses stacktrace and
           # cause false positives like println!. See https://github.com/google/sanitizers/issues/89.
@@ -231,13 +203,14 @@ jobs:
       - name: "Run Godot integration test"
         uses: ./.github/composite/godot-itest
         with:
-          artifact-name: godot-${{ matrix.name }}
+          artifact-name: godot-${{ matrix.artifact-name || matrix.name }}
           godot-binary: ${{ matrix.godot-binary }}
           godot-args: ${{ matrix.godot-args }}
           rust-extra-args: ${{ matrix.rust-extra-args }}
-          rust-toolchain: ${{ matrix.rust-toolchain }}
+          rust-toolchain: ${{ matrix.rust-toolchain || 'stable' }}
           rust-env-rustflags: ${{ matrix.rust-env-rustflags }}
           with-llvm: ${{ matrix.with-llvm }}
+          godot-check-header: ${{ matrix.name == 'linux' }}
 
 
   license-guard:

.github/workflows/minimal-ci.yml

@@ -44,40 +44,21 @@ jobs:
 
 
   clippy:
-    name: clippy (${{ matrix.name }})
     runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        include: 
-          - name: linux
-            rust-toolchain: stable
-            godot-binary: godot.linuxbsd.editor.dev.x86_64
-          
-          - name: linux-double
-            rust-toolchain: stable
-            godot-binary: godot.linuxbsd.editor.dev.double.x86_64
-            rust-extra-args: --features double-precision
     steps:
       - uses: actions/checkout@v3
 
       - name: "Install Rust"
         uses: ./.github/composite/rust
-
-      # TODO get rid of Godot binary, once the JSON is either versioned or fetched from somewhere
-      # Replaces also backspaces on Windows, since they cause problems in Bash
-      - name: "Install Godot"
-        uses: ./.github/composite/godot-install
         with:
-          artifact-name: godot-${{ matrix.name }}
-          godot-binary: ${{ matrix.godot-binary }}
+          components: clippy
 
       - name: "Check clippy"
         run: |
           cargo clippy --all-targets $GDEXT_FEATURES ${{ matrix.rust-extra-args }} -- \
           -D clippy::suspicious -D clippy::style -D clippy::complexity -D clippy::perf \
           -D clippy::dbg_macro -D clippy::todo -D clippy::unimplemented -D warnings
-
+          
 
   unit-test:
     name: unit-test
@@ -88,12 +69,6 @@ jobs:
       - name: "Install Rust"
         uses: ./.github/composite/rust
 
-      - name: "Install Godot"
-        uses: ./.github/composite/godot-install
-        with:
-          artifact-name: godot-linux
-          godot-binary: godot.linuxbsd.editor.dev.x86_64
-
       - name: "Compile tests"
         run: cargo test $GDEXT_FEATURES --no-run
 
@@ -113,7 +88,6 @@ jobs:
         with:
           artifact-name: godot-linux
           godot-binary: godot.linuxbsd.editor.dev.x86_64
-          #godot_ver: ${{ matrix.godot }}
 
 
   license-guard: