Use a dynamic environment for PR jobs

Author: Kobzol

.github/workflows/ci.yml

@@ -45,21 +45,16 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          SECRETS: "${{ toJSON(secrets) }}"
+          GITHUB_CTX: "${{ toJSON(github) }}"
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   pr:
-    name: "PR - ${{ matrix.name }}"
+    name: "${{ matrix.name }}"
     needs:
       - calculate_matrix
-    env:
-      PR_CI_JOB: 1
-      CI_JOB_NAME: "${{ matrix.name }}"
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
-      CACHE_DOMAIN: ci-caches.rust-lang.org
+    env: "${{ matrix.env }}"
     if: "github.event_name == 'pull_request'"
     continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
     strategy:

library/backtrace

@@ -11,6 +11,7 @@
 """
 
 import json
+import os
 from pathlib import Path
 
 import yaml
@@ -19,7 +20,24 @@
 
 
 if __name__ == "__main__":
+    github_ctx = json.loads(os.environ["GITHUB_CTX"])
+    secrets = json.loads(os.environ["SECRETS"])
+
+    print(github_ctx)
+
     with open(JOBS_YAML_PATH) as f:
         jobs = yaml.safe_load(f)
-    job_output = jobs["pr"]
-    print(f"jobs={json.dumps(job_output)}")
+    selected_jobs = jobs["pr"]
+
+    for job in selected_jobs:
+        name = job.get("name")
+        if name is None:
+            raise Exception(f"Job {job} has no `name` attribute")
+        env = job.get("env", {})
+        env["CI_JOB_NAME"] = name
+        # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+        env["HEAD_SHA"] = ""  # ${{ github.event.pull_request.head.sha || github.sha }}
+        env["DOCKER_TOKEN"] = secrets["github_token"]
+        job["env"] = env
+
+    print(f"jobs={json.dumps(selected_jobs)}")

library/stdarch

@@ -351,15 +351,16 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          SECRETS: ${{ toJSON(secrets) }}
+          GITHUB_CTX: ${{ toJSON(github) }}
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   pr:
     <<: *base-ci-job
-    name: PR - ${{ matrix.name }}
+    name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
-    env:
-      <<: [*shared-ci-variables, *public-variables]
-      PR_CI_JOB: 1
+    env: ${{ matrix.env }}
     if: github.event_name == 'pull_request'
     continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
     strategy:

src/ci/github-actions/calculate-job-matrix.py

@@ -2,7 +2,32 @@
 # dynamically in CI from ci.yml.
 # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
 # modify this file.
-shared_defs:
+variables:
+  - &shared-ci-variables
+    CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+
+  - &public-variables
+    SCCACHE_BUCKET: rust-lang-ci-sccache2
+    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+    CACHE_DOMAIN: ci-caches.rust-lang.org
+
+  - &prod-variables
+    SCCACHE_BUCKET: rust-lang-ci-sccache2
+    DEPLOY_BUCKET: rust-lang-ci2
+    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
+    AWS_REGION: us-west-1
+    CACHE_DOMAIN: ci-caches.rust-lang.org
+
+runners:
   - &base-job
     env: { }
 
@@ -37,14 +62,26 @@ shared_defs:
   - &job-aarch64-linux
     os: [ self-hosted, ARM64, linux ]
 
+envs:
+  - &pr-env
+    <<: [ *shared-ci-variables, *public-variables ]
+    PR_CI_JOB: 1
+
 pr:
-  - name: mingw-check
+  - name: PR - mingw-check
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: mingw-check-tidy
+  - name: PR - mingw-check-tidy
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: x86_64-gnu-llvm-17
+  - name: PR - x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
+      <<: *pr-env
     <<: *job-linux-16c
-  - name: x86_64-gnu-tools
+  - name: PR - x86_64-gnu-tools
+    env:
+      <<: *pr-env
     <<: *job-linux-16c