Allow users to generate API from user-provided JSON and headers

- Implement codegen from user-provided json (`godot_json.rs`, similar to `godot_exe.rs`)
- Add `linux-custom-api-json` job to pipeline
- Use the latest GDExtension headers by default, but allow users to specify their own in case if they are using their own, modified version of the engine.
- Allow to use `double-precision` feature with an `api-custom-json`.

Author: Yarwin

.github/composite/godot-itest/action.yml

@@ -7,6 +7,11 @@ name: godot
 description: "Run Godot integration tests"
 
 inputs:
+  godot-indirect-json:
+    required: false
+    default: ''
+    description: "If set to true, compiles godot-rust with the `api-custom-json` feature against the generated JSON file"
+
   artifact-name:
     required: true
     description: "Name of the compiled Godot artifact to download"
@@ -125,9 +130,18 @@ runs:
 
       shell: bash
 
+    # else if
+    - name: "Dump extension api"
+      if: inputs.godot-indirect-json == 'true' && inputs.godot-prebuilt-patch == ''
+      run: |
+        $GODOT4_BIN --headless --dump-extension-api
+        mv extension_api.json $RUNNER_DIR/godot_bin/extension_api.json
+        echo "GODOT4_GDEXTENSION_JSON=$RUNNER_DIR/godot_bin/extension_api.json" >> $GITHUB_ENV
+      shell: bash
+
     # else
     - name: "No patch selected"
-      if: inputs.godot-prebuilt-patch == ''
+      if: inputs.godot-prebuilt-patch == '' && inputs.godot-indirect-json != 'true'
       run: |
           echo "No patch selected; use default godot4-prebuilt version."
       shell: bash

.github/workflows/full-ci.yml

@@ -333,6 +333,15 @@ jobs:
             # Important to keep both experimental-threads and codegen-full. Some itests (native_st_audio) require both.
             rust-extra-args: --features itest/experimental-threads,itest/codegen-full-experimental,godot/api-custom,godot/serde,itest/register-docs
 
+          # Compiles godot-rust with `api-custom-json` feature against the JSON file generated via `--dump-extension-api`.
+          # Uses latest 4.x headers, while `extension_api.json` comes from the latest Godot binary.
+          - name: linux-custom-api-json
+            os: ubuntu-22.04
+            artifact-name: linux-nightly
+            godot-binary: godot.linuxbsd.editor.dev.x86_64
+            rust-extra-args: --features godot/api-custom-json
+            godot-indirect-json: true
+
           - name: linux-release
             os: ubuntu-22.04
             artifact-name: linux-release-nightly
@@ -406,7 +415,8 @@ jobs:
           rust-cache-key: ${{ matrix.rust-cache-key }}
           with-llvm: ${{ contains(matrix.name, 'macos') && contains(matrix.rust-extra-args, 'api-custom') }}
           godot-check-header: ${{ matrix.godot-check-header }}
-          
+          godot-indirect-json: ${{ matrix.godot-indirect-json }}
+
       - name: "Build and test hot-reload"
         if: ${{ matrix.with-hot-reload }}
         working-directory: itest/hot-reload/godot

.github/workflows/minimal-ci.yml

@@ -161,6 +161,13 @@ jobs:
             rust-extra-args: --features itest/codegen-full
             with-hot-reload: true
 
+          - name: linux-custom-api-json
+            os: ubuntu-22.04
+            artifact-name: linux-nightly
+            godot-binary: godot.linuxbsd.editor.dev.x86_64
+            rust-extra-args: --features godot/api-custom-json
+            godot-indirect-json: true
+
           - name: linux-features-experimental
             os: ubuntu-22.04
             artifact-name: linux-nightly
@@ -221,6 +228,7 @@ jobs:
           rust-target: ${{ matrix.rust-target }}
           with-llvm: ${{ contains(matrix.name, 'macos') && contains(matrix.rust-extra-args, 'api-custom') }}
           godot-check-header: ${{ matrix.godot-check-header }}
+          godot-indirect-json: ${{ matrix.godot-indirect-json }}
 
       - name: "Build and test hot-reload"
         if: ${{ matrix.with-hot-reload }}

godot-bindings/Cargo.toml

@@ -34,6 +34,7 @@ api-4-4 = []
 default = []
 
 api-custom = ["dep:bindgen", "dep:regex", "dep:which"]
+api-custom-json = ["dep:nanoserde", "dep:bindgen", "dep:regex", "dep:which"]
 api-custom-extheader = []
 
 [dependencies]
@@ -42,6 +43,8 @@ gdextension-api = { workspace = true }
 bindgen = { workspace = true, optional = true }
 regex = { workspace = true, optional = true }
 which = { workspace = true, optional = true }
+# required by `api-custom-json` to parse the extension api header (to get the Godot version).
+nanoserde = { workspace = true, optional = true }
 
 [dev-dependencies]
 # For tests, we need regex unconditionally.

godot-bindings/build.rs

@@ -13,6 +13,7 @@
 fn main() {    
     let mut count = 0;
     if cfg!(feature = "api-custom") { count += 1; }
+    if cfg!(feature = "api-custom-json") { count += 1; }
 
     // [version-sync] [[
     //  [line] \tif cfg!(feature = "api-$kebabVersion") { count += 1; }

godot-bindings/src/godot_exe.rs

@@ -7,12 +7,11 @@
 
 //! Commands related to Godot executable
 
-use crate::godot_version::parse_godot_version;
-use crate::header_gen::generate_rust_binding;
+use crate::godot_version::{parse_godot_version, validate_godot_version};
+use crate::header_gen::{generate_rust_binding, patch_c_header};
 use crate::watch::StopWatch;
 use crate::GodotVersion;
 
-use regex::Regex;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Output};
@@ -55,8 +54,7 @@ pub fn write_gdextension_headers(
     is_h_provided: bool,
     watch: &mut StopWatch,
 ) {
-    // None=(unknown, no engine), Some=(version of Godot). Later verified by header itself.
-    // Even though we don't support 4.0 anymore, we still detect it, for better error messages.
+    // Use Godot binary to dump GDExtension headers if they weren't provided by the user.
     if !is_h_provided {
         // No external C header file: Godot binary is present, we use it to dump C header
         let godot_bin = locate_godot_binary();
@@ -78,7 +76,7 @@ pub fn write_gdextension_headers(
     // Listening to changes on files that are generated by this build step cause an infinite loop with cargo watch of
     // build -> detect change -> rebuild -> detect change -> ...
     // rerun_on_changed(inout_h_path);
-    patch_c_header(inout_h_path);
+    patch_c_header(inout_h_path, inout_h_path);
     watch.record("patch_header_h");
 
     generate_rust_binding(inout_h_path, out_rs_path);
@@ -111,28 +109,9 @@ pub(crate) fn read_godot_version(godot_bin: &Path) -> GodotVersion {
     let output = execute(cmd, "read Godot version");
     let stdout = std::str::from_utf8(&output.stdout).expect("convert Godot version to UTF-8");
 
-    let version = match parse_godot_version(stdout) {
-        Ok(parsed) => {
-            assert_eq!(
-                parsed.major,
-                4,
-                "Only Godot versions with major version 4 are supported; found version {}.",
-                stdout.trim()
-            );
-
-            assert!(
-                parsed.minor > 0,
-                "Godot 4.0 is no longer supported by godot-rust; found version {}.",
-                stdout.trim()
-            );
-
-            parsed
-        }
-        Err(e) => {
-            // Don't treat this as fatal error
-            panic!("failed to parse Godot version '{stdout}': {e}")
-        }
-    };
+    let version = parse_godot_version(stdout)
+        .unwrap_or_else(|err| panic!("failed to parse Godot version '{stdout}': {err}"));
+    validate_godot_version(&version, stdout);
 
     // `--dump-extension-api`, `--dump-gdextension-interface` etc. are only available in Debug builds (editor, debug export template).
     // If we try to run them in release builds, Godot tries to run, causing a popup alert with an unhelpful message:
@@ -191,53 +170,6 @@ fn dump_header_file(godot_bin: &Path, out_file: &Path) {
     println!("Generated {}/gdextension_interface.h.", cwd.display());
 }
 
-fn patch_c_header(inout_h_path: &Path) {
-    // The C header path *must* be passed in by the invoking crate, as the path cannot be relative to this crate.
-    // Otherwise, it can be something like `/home/runner/.cargo/git/checkouts/gdext-76630c89719e160c/efd3b94/godot-bindings`.
-
-    println!("Patch C header '{}'...", inout_h_path.display());
-
-    let mut c = fs::read_to_string(inout_h_path)
-        .unwrap_or_else(|_| panic!("failed to read C header file {}", inout_h_path.display()));
-
-    // Detect whether header is legacy (4.0) format. This should generally already be checked outside.
-    assert!(
-        c.contains("GDExtensionInterfaceGetProcAddress"),
-        "C header file '{}' seems to be GDExtension version 4.0, which is no longer support by godot-rust.",
-        inout_h_path.display()
-    );
-
-    // Patch for variant converters and type constructors.
-    c = c.replace(
-        "typedef void (*GDExtensionVariantFromTypeConstructorFunc)(GDExtensionVariantPtr, GDExtensionTypePtr);",
-        "typedef void (*GDExtensionVariantFromTypeConstructorFunc)(GDExtensionUninitializedVariantPtr, GDExtensionTypePtr);"
-    )
-    .replace(
-        "typedef void (*GDExtensionTypeFromVariantConstructorFunc)(GDExtensionTypePtr, GDExtensionVariantPtr);",
-        "typedef void (*GDExtensionTypeFromVariantConstructorFunc)(GDExtensionUninitializedTypePtr, GDExtensionVariantPtr);"
-    )
-    .replace(
-        "typedef void (*GDExtensionPtrConstructor)(GDExtensionTypePtr p_base, const GDExtensionConstTypePtr *p_args);",
-        "typedef void (*GDExtensionPtrConstructor)(GDExtensionUninitializedTypePtr p_base, const GDExtensionConstTypePtr *p_args);"
-    );
-
-    // Use single regex with independent "const"/"Const", as there are definitions like this:
-    // typedef const void *GDExtensionMethodBindPtr;
-    let c = Regex::new(r"typedef (const )?void \*GDExtension(Const)?([a-zA-Z0-9]+?)Ptr;") //
-        .expect("regex for mut typedef")
-        .replace_all(&c, "typedef ${1}struct __Gdext$3 *GDExtension${2}${3}Ptr;");
-
-    // println!("Patched contents:\n\n{}\n\n", c.as_ref());
-
-    // Write the modified contents back to the file
-    fs::write(inout_h_path, c.as_ref()).unwrap_or_else(|_| {
-        panic!(
-            "failed to write patched C header file {}",
-            inout_h_path.display()
-        )
-    });
-}
-
 pub(crate) fn locate_godot_binary() -> PathBuf {
     if let Ok(string) = std::env::var("GODOT4_BIN") {
         println!("Found GODOT4_BIN with path to executable: '{string}'");

godot-bindings/src/godot_json.rs

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) godot-rust; Bromeon and contributors.
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+//! Commands related to parsing user-provided JSON and extension headers.
+
+// At first re-using mapping from godot-codegen json.rs might seem more desirable but there are few issues to consider:
+// * Overall JSON file structure might change slightly from version to version, while header should stay consistent (otherwise it defeats the purpose of having any header at all).
+// Having two parsers – minimal one inherent to api-custom-json feature and codegen one – makes handling all the edge cases easier.
+// * `godot-codegen` depends on `godot-bindings` thus simple re-using types from former in side the latter is not possible (cyclic dependency).
+// Moving said types to `godot-bindings` would increase the cognitive overhead (since domain mapping is responsibility of `godot-codegen`, while godot-bindings is responsible for providing required resources & emitting the version).
+// In the future we might experiment with splitting said types into separate crates.
+
+use crate::depend_on_custom_json::header_gen::{generate_rust_binding, patch_c_header};
+use crate::godot_version::validate_godot_version;
+use crate::{GodotVersion, StopWatch};
+use nanoserde::DeJson;
+use std::fs;
+use std::path::Path;
+
+// GDExtension headers are backward compatible (new incremental changes in general are exposed as additions to the existing API) while godot-rust simply ignores extra declarations in header file.
+// Therefore, latest headers should work fine for all the past and future Godot versions – as long as the engine remains unchanged.
+// [version-sync] [[
+//  [include] current.minor
+//  [line] use gdextension_api::version_$snakeVersion::load_gdextension_header_h as load_latest_gdextension_headers;
+use gdextension_api::version_4_4::load_gdextension_header_h as load_latest_gdextension_headers;
+// ]]
+
+/// A minimal version of deserialized JsonExtensionApi that includes only the header.
+#[derive(DeJson)]
+struct JsonExtensionApi {
+    pub header: JsonHeader,
+}
+
+/// Deserialized "header" key in given `extension_api.json`.
+#[derive(DeJson)]
+struct JsonHeader {
+    pub version_major: u8,
+    pub version_minor: u8,
+    pub version_patch: u8,
+    pub version_status: String,
+    pub version_build: String,
+    pub version_full_name: String,
+}
+
+impl JsonHeader {
+    fn into_godot_version(self) -> GodotVersion {
+        GodotVersion {
+            full_string: self.version_full_name,
+            major: self.version_major,
+            minor: self.version_minor,
+            patch: self.version_patch,
+            status: self.version_status,
+            custom_rev: Some(self.version_build),
+        }
+    }
+}
+
+pub fn load_custom_gdextension_json() -> String {
+    let path = std::env::var("GODOT4_GDEXTENSION_JSON").expect(
+        "godot-rust with `api-custom-json` feature requires GODOT4_GDEXTENSION_JSON \
+        environment variable (with the path to the said json).",
+    );
+    let json_path = Path::new(&path);
+
+    fs::read_to_string(json_path).unwrap_or_else(|_| {
+        panic!(
+            "failed to open file with custom GDExtension JSON {}.",
+            json_path.display()
+        )
+    })
+}
+
+pub(crate) fn read_godot_version() -> GodotVersion {
+    let extension_api: JsonExtensionApi = DeJson::deserialize_json(&load_custom_gdextension_json())
+        .expect("failed to deserialize JSON");
+    let version = extension_api.header.into_godot_version();
+
+    validate_godot_version(&version, &version.full_string);
+
+    version
+}
+
+pub(crate) fn write_gdextension_headers(
+    out_h_path: &Path,
+    out_rs_path: &Path,
+    watch: &mut StopWatch,
+) {
+    // Allow to use custom gdextension headers in unlikely case if one uses heavily modified version of the engine.
+    if let Ok(path) = std::env::var("GODOT4_GDEXTENSION_HEADERS") {
+        let in_h_path = Path::new(&path);
+        patch_c_header(in_h_path, out_h_path);
+        watch.record("patch_header_h");
+    } else {
+        let h_contents = load_latest_gdextension_headers();
+        fs::write(out_h_path, h_contents.as_ref())
+            .unwrap_or_else(|e| panic!("failed to write gdextension_interface.h: {e}"));
+        watch.record("write_header_h");
+    }
+
+    generate_rust_binding(out_h_path, out_rs_path);
+    watch.record("generate_header_rs");
+}

godot-bindings/src/godot_version.rs

@@ -5,7 +5,9 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-//#![allow(unused_variables, dead_code)]
+// This file is explicitly included in unit tests
+// while all the functions included are used only with `custom-api` and `custom-api-json` features.
+#![allow(unused_variables, dead_code)]
 
 use crate::GodotVersion;
 use regex::{Captures, Regex};
@@ -52,6 +54,21 @@ pub fn parse_godot_version(version_str: &str) -> Result<GodotVersion, Box<dyn Er
     })
 }
 
+pub(crate) fn validate_godot_version(godot_version: &GodotVersion, version_str: &str) {
+    assert_eq!(
+        godot_version.major,
+        4,
+        "Only Godot versions with major version 4 are supported; found version {}.",
+        version_str.trim()
+    );
+
+    assert!(
+        godot_version.minor > 0,
+        "Godot 4.0 is no longer supported by godot-rust; found version {}.",
+        version_str.trim()
+    );
+}
+
 /// Extracts and parses a named capture group from a regex match.
 fn cap<T: FromStr>(caps: &Captures, key: &str) -> Result<Option<T>, Box<dyn Error>> {
     caps.name(key)