Initial WASM/Cross build factory code (#322)

* Initial WASM/Cross build code
* Changes to use HOSTRUNNER instead of RUNSHARED
* Add 3.11 branches (#323)
* Add tier labels corresponding to PEP 11 (#326)
* Add environment variable to set settings path (#328)
* Split WASM build factories
* use config.guess directly
* Fix pkg-config cross build, add suffix to create unique workers

Co-authored-by: Brett Cannon <[email protected]>
Co-authored-by: Christian Heimes <[email protected]>
Co-authored-by: Ethan Smith <[email protected]>
Co-authored-by: Pablo Galindo Salgado <[email protected]>
Co-authored-by: Zachary Ware <[email protected]>

Author: 5 people

master/custom/builders.py

@@ -36,6 +36,9 @@
     MacOSArmWithBrewBuild,
     WindowsARM64Build,
     WindowsARM64ReleaseBuild,
+    Wasm32EmscriptenNodeBuild,
+    Wasm32EmscriptenBrowserBuild,
+    Wasm32WASIBuild,
 )
 
 STABLE = "stable"
@@ -49,7 +52,7 @@
 def get_builders(settings):
     # Override with a default simple worker if we are using local workers
     if settings.use_local_worker:
-        return [("Test Builder", "local-worker", UnixBuild, STABLE)]
+        return [("Test Builder", "local-worker", UnixBuild, STABLE, NO_TIER)]
 
     return [
         # -- Stable builders --
@@ -207,6 +210,11 @@ def get_builders(settings):
         ("ARM64 Windows Non-Debug", "linaro-win-arm64", WindowsARM64ReleaseBuild, STABLE, NO_TIER),
         ("ARM64 Windows Azure", "linaro2-win-arm64", WindowsARM64Build, UNSTABLE, NO_TIER),
         ("ARM64 Windows Non-Debug Azure", "linaro2-win-arm64", WindowsARM64ReleaseBuild, UNSTABLE, NO_TIER),
+
+        # WebAssembly
+        ("wasm32-emscripten node (threaded)", "bcannon-wasm", Wasm32EmscriptenNodeBuild, UNSTABLE, NO_TIER),
+        ("wasm32-emscripten browser (dynamic linking)", "bcannon-wasm", Wasm32EmscriptenBrowserBuild, UNSTABLE, NO_TIER),
+        ("wasm32-wasi", "bcannon-wasm", Wasm32WASIBuild, UNSTABLE, NO_TIER),
     ]
 
 

master/custom/factories.py

@@ -1,6 +1,13 @@
 import os.path
 from buildbot.process import factory
-from buildbot.steps.shell import Configure, Compile, ShellCommand
+from buildbot.steps.shell import (
+    Configure,
+    Compile,
+    ShellCommand,
+    SetPropertyFromCommand,
+)
+
+from buildbot.plugins import util
 
 from .steps import (
     Test,
@@ -577,3 +584,253 @@ class WindowsARM64ReleaseBuild(WindowsARM64Build):
     testFlags = WindowsARM64Build.testFlags + ["+d"]
     # keep default cleanFlags, both configurations get cleaned
     factory_tags = ["win-arm64", "nondebug"]
+
+##############################################################################
+##############################  WASM BUILDS  #################################
+##############################################################################
+
+
+class UnixCrossBuild(UnixBuild):
+    configureFlags = [
+        "--with-pydebug",
+        "--with-build-python=../build/python"
+    ]
+    extra_configure_flags = []
+    host_configure_cmd = ["../../configure"]
+    host = None
+    host_make_cmd = ["make"]
+    can_execute_python = True
+    
+    def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
+        assert self.host is not None, "Must set self.host on cross builds"
+
+        out_of_tree_dir = "build_oot"
+        oot_dir_path = os.path.join("build", out_of_tree_dir)
+        oot_build_path = os.path.join(oot_dir_path, "build")
+        oot_host_path = os.path.join(oot_dir_path, "host")
+
+        self.addStep(
+            SetPropertyFromCommand(
+                name="Gather build triple from worker",
+                description="Get the build triple config.guess",
+                command="./config.guess",
+                property="build_triple",
+                warnOnFailure=True,
+            )
+        )
+
+        # Create out of tree directory for "build", the platform we are
+        # currently running on
+        self.addStep(
+            ShellCommand(
+                name="mkdir build out-of-tree directory",
+                description="Create build out-of-tree directory",
+                command=["mkdir", "-p", oot_build_path],
+                warnOnFailure=True,
+            )
+        )
+        # Create directory for "host", the platform we want to compile *for*
+        self.addStep(
+            ShellCommand(
+                name="mkdir host out-of-tree directory",
+                description="Create host out-of-tree directory",
+                command=["mkdir", "-p", oot_host_path],
+                warnOnFailure=True,
+            )
+        )
+
+        # First, we build the "build" Python, which we need to cross compile
+        # the "host" Python
+        self.addStep(
+            Configure(
+                name="Configure build Python",
+                command=["../../configure"],
+                workdir=oot_build_path
+            )
+        )
+        if parallel:
+            compile = ["make", parallel]
+        else:
+            compile = ["make"]
+        
+        self.addStep(
+            Compile(
+                name="Compile build Python",
+                command=compile,
+                workdir=oot_build_path
+            )
+        )
+
+        # Now that we have a "build" architecture Python, we can use that
+        # to build a "host" (also known as the target we are cross compiling)
+        # to 
+        configure_cmd = self.host_configure_cmd + ["--prefix", "$(PWD)/target/host"]
+        configure_cmd += self.configureFlags + self.extra_configure_flags
+        configure_cmd += [util.Interpolate("--build=%(prop:build_triple)s")]
+        configure_cmd += [f"--host={self.host}"]
+        self.addStep(
+            Configure(
+                name="Configure host Python",
+                command=configure_cmd,
+                env=self.compile_environ,
+                workdir=oot_host_path
+            )
+        )
+
+        testopts = self.testFlags
+        if "-R" not in self.testFlags:
+            testopts += " --junit-xml test-results.xml"
+        if parallel:
+            testopts = testopts + " " + parallel
+        if "-j" not in testopts:
+            testopts = "-j2 " + testopts
+
+        # Timeout for the buildworker process
+        self.test_timeout = self.test_timeout or TEST_TIMEOUT
+        # Timeout for faulthandler
+        faulthandler_timeout = self.test_timeout - 5 * 60
+
+        test = [
+            "make",
+            "buildbottest",
+            "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
+            "TESTPYTHONOPTS=" + self.interpreterFlags,
+            "TESTTIMEOUT=" + str(faulthandler_timeout),
+        ]
+
+        if parallel:
+            compile = self.host_make_cmd + [parallel, self.makeTarget]
+        else:
+            compile = self.host_make_cmd + [self.makeTarget]
+        self.addStep(
+            Compile(
+                name="Compile host Python",
+                command=compile,
+                env=self.compile_environ,
+                workdir=oot_host_path,
+            )
+        )
+        if self.can_execute_python:
+            self.addStep(
+                ShellCommand(
+                    name="pythoninfo",
+                    description="pythoninfo",
+                    command=["make", "pythoninfo"],
+                    warnOnFailure=True,
+                    env=self.test_environ,
+                    workdir=oot_host_path,
+                )
+            )
+            self.addStep(
+                Test(
+                    command=test,
+                    timeout=self.test_timeout,
+                    usePTY=test_with_PTY,
+                    env=self.test_environ,
+                    workdir=oot_host_path,
+                )
+            )
+            if branch not in ("3",) and "-R" not in self.testFlags:
+                filename = os.path.join(oot_host_path, "test-results.xml")
+                self.addStep(UploadTestResults(branch, filename=filename))
+        self.addStep(
+            Clean(
+                name="Clean build Python",
+                workdir=oot_build_path,
+            )
+        )
+        self.addStep(
+            Clean(
+                name="Clean host Python",
+                workdir=oot_host_path,
+            )
+        )
+
+
+class Wasm32EmscriptenBuild(UnixCrossBuild):
+    """wasm32-emscripten builder
+
+    * Emscripten SDK >= 3.1.12 must be installed
+    * ccache must be installed
+    * Emscripten PATHs must be pre-pended to PATH
+    * ``which node`` must be equal $EMSDK_NODE
+    """
+    factory_tags = ["wasm", "emscripten"]
+    compile_environ = {
+        "CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-emscripten",
+        "EM_COMPILER_WRAPPER": "ccache",
+    }
+
+    host = "wasm32-unknown-emscripten"
+    host_configure_cmd = ["emconfigure", "../../configure"]
+    host_make_cmd = ["emmake", "make"]
+
+
+class Wasm32EmscriptenNodeBuild(Wasm32EmscriptenBuild):
+    buildersuffix = ".emscripten-node"
+    extra_configure_flags = [
+        "--with-emscripten-target=node",
+        "--disable-wasm-dynamic-linking",
+        "--enable-wasm-pthreads",
+    ]
+
+
+class Wasm32EmscriptenBrowserBuild(Wasm32EmscriptenBuild):
+    buildersuffix = ".emscripten-browser"
+    extra_configure_flags = [
+        "--with-emscripten-target=browser",
+        "--enable-wasm-dynamic-linking",
+        "--disable-wasm-pthreads",
+    ]
+    # browser builds do not accept argv from CLI
+    can_execute_python = False
+
+
+class Wasm32WASIBuild(UnixCrossBuild):
+    """wasm32-wasi builder
+
+    * WASI SDK >= 16 must be installed to default path /opt/wasi-sdk
+    * WASIX must be installed to /opt/wasix
+    * ccache must be installed
+    * wasmtime must be installed and on PATH
+    """
+    buildersuffix = ".wasi"
+    factory_tags = ["wasm", "wasi"]
+    extra_configure_flags = [
+        # debug builds exhaust the limited call stack on WASI
+        "--without-pydebug",
+        # ipv6 is not supported on WASI
+        "--disable-ipv6",
+    ]
+    wasi_sdk = "/opt/wasi-sdk"
+    wasi_sysroot = f"{wasi_sdk}/share/wasi-sysroot"
+    wasix = "/opt/wasix"
+    compile_environ = {
+        "CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-wasi",
+        # use Clang from WASI-SDK
+        "CC": f"ccache {wasi_sdk}/bin/clang",
+        "LDSHARED": f"{wasi_sdk}/bin/wasm-ld",
+        "AR": f"{wasi_sdk}/bin/llvm-ar",
+        # use WASIX library with POSIX stubs
+        "CFLAGS": f"-isystem {wasix}/include",
+        "LDFLAGS": f"-L{wasix}/lib -lwasix",
+        # WASI-SDK does not have a 'wasm32-unknown-wasi-pkg-config' script
+        # force pkg-config into cross-compiling mode
+        "PKG_CONFIG_PATH": "",
+        "PKG_CONFIG_SYSROOT_DIR": wasi_sysroot,
+        "PKG_CONFIG_LIBDIR": f"{wasi_sysroot}/lib/pkgconfig:{wasi_sysroot}/share/pkgconfig",
+    }
+    host = "wasm32-unknown-wasi"
+
+    def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
+        self.addStep(
+            ShellCommand(
+                name="Touch srcdir Modules/Setup.local",
+                description="Hack to work around wasmtime mapdir issue",
+                command=["touch", "Modules/Setup.local"],
+                haltOnFailure=True,
+            )
+        )
+        super().setup(
+            parallel, branch, test_with_PTY=test_with_PTY, **kwargs
+        )

master/custom/workers.py

@@ -289,5 +289,10 @@ def get_workers(settings):
             tags=['windows', 'arm64'],
             parallel_tests=2,
         ),
-
+        cpw(
+            name="bcannon-wasm",
+            tags=['wasm', 'emscripten', 'wasi'],
+            branches=['3.11', '3.x'],
+            parallel_tests=4,
+        ),
     ]

master/master.cfg

@@ -236,6 +236,14 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
             branch=branchname,
             **extra_factory_args.get(worker, {}),
         )
+        tags = [branchname, stability,] + getattr(f, "tags", [])
+        if tier:
+            tags.append(tier)
+
+        # Only 3.11+ for WebAssembly builds
+        if "wasm" in tags and branchname in {"3.8", "3.9", "3.10"}:
+            continue
+
         if name in DAILYBUILDERS:
             dailybuildernames.append(buildername)
         else: