| 1 | # Releases the self-hosted runner, g1t-runner (crates/runner): builds it for |
| 2 | # Linux, macOS and Windows on x64 and arm64, pushes its container image, |
| 3 | # builds the images `g1t-runner sandbox` runs on people's own computers, |
| 4 | # signs the release (binaries and sandbox images in one manifest), and |
| 5 | # publishes it to g1t.sh/downloads/runner/ (the g1t-downloads R2 bucket). |
| 6 | # Runners already out there update themselves to it. |
| 7 | # |
| 8 | # A release is a tag `runner-v<version>`, where the version is the one in |
| 9 | # crates/runner/Cargo.toml; or run it by hand. scripts/runner-release.mjs |
| 10 | # and scripts/sandbox-image.mjs do the work, and the first's header says how |
| 11 | # to make the release key the first time; CONTRIBUTING.md ("Deploying") |
| 12 | # says how a release is made. |
| 13 | # |
| 14 | # The image jobs run on g1t's own machines (g1t-4core), whose jobs have a |
| 15 | # Docker Engine of their own (crates/runner/src/docker/engine.rs). arm64 |
| 16 | # images need QEMU's emulators registered with the kernel; each image job |
| 17 | # tries that, and when the machine does not allow it, builds amd64 alone |
| 18 | # and says so in a warning rather than failing the release. |
| 19 | name: Runner release |
| 20 | |
| 21 | on: |
| 22 | push: |
| 23 | tags: ["runner-v*"] |
| 24 | workflow_dispatch: |
| 25 | |
| 26 | permissions: |
| 27 | contents: read |
| 28 | |
| 29 | concurrency: |
| 30 | group: runner-release |
| 31 | cancel-in-progress: false |
| 32 | |
| 33 | env: |
| 34 | CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }} |
| 35 | WRANGLER_SEND_METRICS: "false" |
| 36 | |
| 37 | jobs: |
| 38 | binaries: |
| 39 | # One job per target, side by side: each cross-compile takes about six |
| 40 | # minutes, and one after another they were most of a release's wait. |
| 41 | name: Build ${{ matrix.target }} |
| 42 | runs-on: ubuntu-latest |
| 43 | timeout-minutes: 60 |
| 44 | strategy: |
| 45 | fail-fast: true |
| 46 | matrix: |
| 47 | target: [linux-x64, linux-arm64, windows-x64] |
| 48 | env: |
| 49 | G1T_RUNNER_RELEASE_KEY: ${{ vars.RUNNER_RELEASE_PUBLIC_KEY }} |
| 50 | steps: |
| 51 | - uses: actions/checkout@v5 |
| 52 | - name: The tag names this version |
| 53 | if: startsWith(github.ref, 'refs/tags/runner-v') |
| 54 | run: | |
| 55 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' crates/runner/Cargo.toml | head -1)" |
| 56 | [ "runner-v$version" = "${GITHUB_REF_NAME}" ] || { echo "::error::The tag is ${GITHUB_REF_NAME}, but crates/runner is $version"; exit 1; } |
| 57 | # zig and cargo-zigbuild, kept on main by the Caches workflow |
| 58 | # (caches.yml): installing them took four minutes of every release. |
| 59 | # Change the key there and here to take newer versions. |
| 60 | - name: Cache zig and cargo-zigbuild |
| 61 | id: zig |
| 62 | uses: actions/cache/restore@v4 |
| 63 | with: |
| 64 | path: | |
| 65 | ~/.local/lib/python3*/site-packages/ziglang |
| 66 | ~/.local/bin |
| 67 | ~/.cargo/bin/cargo-zigbuild |
| 68 | key: zig-0.13.0-zigbuild-${{ runner.os }}-1 |
| 69 | - name: Install zig and cargo-zigbuild |
| 70 | if: steps.zig.outputs.cache-hit != 'true' |
| 71 | run: | |
| 72 | # The runner's Debian marks its Python as externally managed (PEP 668); zig goes under ~/.local anyway. |
| 73 | pip install --user --break-system-packages ziglang==0.13.0 |
| 74 | cargo install --locked cargo-zigbuild |
| 75 | - name: zig on the PATH |
| 76 | run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" |
| 77 | # rust: the toolchain; cargo: Cargo.lock's crates.io and git packages, |
| 78 | # not the workspace's own versions (scripts/deploy/ci-cache.mjs). |
| 79 | - name: Cache keys |
| 80 | id: keys |
| 81 | run: node scripts/deploy/ci-cache.mjs keys |
| 82 | # Every crate the lockfile names, as CI's Rust job saves them on main. |
| 83 | - name: Cache crates |
| 84 | uses: actions/cache/restore@v4 |
| 85 | with: |
| 86 | path: | |
| 87 | ~/.cargo/registry/index |
| 88 | ~/.cargo/registry/cache |
| 89 | key: cargo-crates-${{ runner.os }}-${{ steps.keys.outputs.cargo }} |
| 90 | restore-keys: cargo-crates-${{ runner.os }}- |
| 91 | # Each target's build. A tag's run restores only from its own tag and |
| 92 | # from main, never from the last release's tag, so the Caches workflow |
| 93 | # (caches.yml) builds and saves this on main whenever the key is new; |
| 94 | # a release finds it whole, compiles the workspace's crates that |
| 95 | # changed, and saves nothing. |
| 96 | - name: Cache the runner's build |
| 97 | uses: actions/cache/restore@v4 |
| 98 | with: |
| 99 | path: | |
| 100 | target |
| 101 | !target/runner-release |
| 102 | !target/**/incremental |
| 103 | key: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} |
| 104 | restore-keys: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}- |
| 105 | # Crates whose inputs did not change come back from the Actions cache |
| 106 | # instead of being compiled (scripts/sccache.sh); without it, the job |
| 107 | # builds as before. |
| 108 | - name: sccache |
| 109 | run: bash scripts/sccache.sh install |
| 110 | - name: Build Linux and Windows |
| 111 | # The macOS targets need Apple's SDK to link (the Security and |
| 112 | # CoreFoundation frameworks), which zig does not carry, so they are |
| 113 | # left out until this builds in the rust-cross/cargo-zigbuild image |
| 114 | # as the CLI's release does (scripts/cli-release.mjs). A Mac runner |
| 115 | # keeps the version it has, since the manifest names no file for it. |
| 116 | run: node scripts/runner-release.mjs build --targets ${{ matrix.target }} |
| 117 | - name: sccache's hits |
| 118 | if: always() |
| 119 | run: bash scripts/sccache.sh stats |
| 120 | - uses: actions/upload-artifact@v4 |
| 121 | with: |
| 122 | name: binaries-${{ matrix.target }} |
| 123 | path: target/runner-release/ |
| 124 | overwrite: true |
| 125 | |
| 126 | image: |
| 127 | name: Container image |
| 128 | needs: binaries |
| 129 | runs-on: g1t-4core |
| 130 | # Production's secrets, without a deployment: a release is not a deploy of |
| 131 | # g1t.sh, and would otherwise show on its card as one in progress. |
| 132 | environment: |
| 133 | name: production |
| 134 | deployment: false |
| 135 | timeout-minutes: 45 |
| 136 | # Its token pushes the image to g1t's registry. |
| 137 | permissions: |
| 138 | contents: read |
| 139 | packages: write |
| 140 | outputs: |
| 141 | # The pushed image by digest, which the sandbox's manifest names. |
| 142 | runner_image: ${{ steps.push.outputs.runner_image }} |
| 143 | steps: |
| 144 | - uses: actions/checkout@v5 |
| 145 | - uses: actions/download-artifact@v4 |
| 146 | with: |
| 147 | pattern: binaries-* |
| 148 | merge-multiple: true |
| 149 | path: release |
| 150 | - name: QEMU for arm64, if this machine allows it |
| 151 | id: qemu |
| 152 | run: | |
| 153 | if docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 \ |
| 154 | && [ "$(docker run --rm --platform linux/arm64 debian:bookworm-slim uname -m 2>/dev/null)" = aarch64 ]; then |
| 155 | echo "arches=amd64 arm64" >> "$GITHUB_OUTPUT" |
| 156 | else |
| 157 | echo "::warning::This machine cannot run arm64 programs (QEMU's emulators could not be registered), so the runner's image is built for amd64 alone this time." |
| 158 | echo "arches=amd64" >> "$GITHUB_OUTPUT" |
| 159 | fi |
| 160 | - name: Build and push |
| 161 | id: push |
| 162 | # To g1t's own registry, where flagon-io keeps it public. |
| 163 | env: |
| 164 | IMAGE: g1t.sh/flagon-io/g1t-runner |
| 165 | ARCHES: ${{ steps.qemu.outputs.arches }} |
| 166 | run: | |
| 167 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' crates/runner/Cargo.toml | head -1)" |
| 168 | echo "${{ secrets.G1T_TOKEN }}" | docker login g1t.sh -u g1t --password-stdin |
| 169 | tags=() |
| 170 | for arch in $ARCHES; do |
| 171 | mkdir -p "context-$arch" |
| 172 | cp deploy/runner/Dockerfile "context-$arch/" |
| 173 | name="g1t-runner-linux-$([ "$arch" = amd64 ] && echo x64 || echo arm64)" |
| 174 | cp "release/$version/$name" "context-$arch/g1t-runner" |
| 175 | docker buildx build --platform "linux/$arch" -t "$IMAGE:$version-$arch" --load "context-$arch" |
| 176 | docker push "$IMAGE:$version-$arch" |
| 177 | tags+=("$IMAGE:$version-$arch") |
| 178 | done |
| 179 | docker buildx imagetools create -t "$IMAGE:$version" -t "$IMAGE:latest" "${tags[@]}" |
| 180 | digest="$(docker buildx imagetools inspect "$IMAGE:$version" --format '{{json .Manifest}}' | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.parse(s).digest))')" |
| 181 | echo "runner_image=$IMAGE@$digest" >> "$GITHUB_OUTPUT" |
| 182 | |
| 183 | sandbox: |
| 184 | name: Sandbox images |
| 185 | needs: [binaries, image] |
| 186 | runs-on: g1t-4core |
| 187 | timeout-minutes: 60 |
| 188 | steps: |
| 189 | - uses: actions/checkout@v5 |
| 190 | - uses: actions/download-artifact@v4 |
| 191 | with: |
| 192 | pattern: binaries-* |
| 193 | merge-multiple: true |
| 194 | path: target/runner-release |
| 195 | - name: QEMU for arm64, if this machine allows it |
| 196 | id: qemu |
| 197 | run: | |
| 198 | if docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 \ |
| 199 | && [ "$(docker run --rm --platform linux/arm64 debian:bookworm-slim uname -m 2>/dev/null)" = aarch64 ]; then |
| 200 | echo "arches=x64 arm64" >> "$GITHUB_OUTPUT" |
| 201 | else |
| 202 | echo "::warning::This machine cannot run arm64 programs, so the sandbox's images are built for x64 alone this time: an arm64 Mac or Linux machine keeps the sandbox image it has." |
| 203 | echo "arches=x64" >> "$GITHUB_OUTPUT" |
| 204 | fi |
| 205 | - name: Build the WSL root filesystem and the VM's disk |
| 206 | env: |
| 207 | ARCHES: ${{ steps.qemu.outputs.arches }} |
| 208 | run: | |
| 209 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' crates/runner/Cargo.toml | head -1)" |
| 210 | dir="target/runner-release/$version/" |
| 211 | for arch in $ARCHES; do |
| 212 | node scripts/sandbox-image.mjs build --arch "$arch" --runner "${dir}g1t-runner-linux-$arch" --out "$dir" |
| 213 | done |
| 214 | - name: The manifest's sandbox section |
| 215 | env: |
| 216 | RUNNER_SANDBOX_IMAGE: ${{ needs.image.outputs.runner_image }} |
| 217 | run: | |
| 218 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' crates/runner/Cargo.toml | head -1)" |
| 219 | node scripts/sandbox-image.mjs section --dir "target/runner-release/$version/" |
| 220 | - uses: actions/upload-artifact@v4 |
| 221 | with: |
| 222 | name: sandbox |
| 223 | path: | |
| 224 | target/runner-release/*/g1t-sandbox-* |
| 225 | target/runner-release/*/sandbox.json |
| 226 | overwrite: true |
| 227 | |
| 228 | publish: |
| 229 | name: Sign and publish |
| 230 | needs: [binaries, image, sandbox] |
| 231 | # The binaries go out even when an image job failed: the manifest then |
| 232 | # names no sandbox images, and computers keep the sandbox they have. |
| 233 | if: ${{ always() && needs.binaries.result == 'success' }} |
| 234 | runs-on: ubuntu-latest |
| 235 | environment: |
| 236 | name: production |
| 237 | deployment: false |
| 238 | timeout-minutes: 30 |
| 239 | steps: |
| 240 | - uses: actions/checkout@v5 |
| 241 | - uses: actions/download-artifact@v4 |
| 242 | with: |
| 243 | pattern: binaries-* |
| 244 | merge-multiple: true |
| 245 | path: target/runner-release |
| 246 | - uses: actions/download-artifact@v4 |
| 247 | if: ${{ needs.sandbox.result == 'success' }} |
| 248 | with: |
| 249 | name: sandbox |
| 250 | path: target/runner-release |
| 251 | - name: Sign |
| 252 | env: |
| 253 | RUNNER_RELEASE_KEY: ${{ secrets.RUNNER_RELEASE_KEY }} |
| 254 | G1T_RUNNER_RELEASE_KEY: ${{ vars.RUNNER_RELEASE_PUBLIC_KEY }} |
| 255 | run: | |
| 256 | node scripts/runner-release.mjs manifest |
| 257 | node scripts/runner-release.mjs sign |
| 258 | node scripts/runner-release.mjs verify |
| 259 | # npm's downloads, kept between runs: an install reads them first |
| 260 | # and asks the registry only for what changed, so a dropped |
| 261 | # connection to it rarely stops the job. |
| 262 | - name: Cache npm's downloads |
| 263 | uses: actions/cache/restore@v4 |
| 264 | with: |
| 265 | path: ~/.npm |
| 266 | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} |
| 267 | restore-keys: npm-full-${{ runner.os }}- |
| 268 | - name: Install Wrangler |
| 269 | run: node scripts/deploy/npm-ci.mjs --prefer-offline --workspaces=false --no-audit --no-fund |
| 270 | - name: Publish to g1t.sh/downloads/runner |
| 271 | env: |
| 272 | CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} |
| 273 | run: node scripts/runner-release.mjs publish |