Skip to content

Commit

Rust builds go through sccache, cached in the repository's Actions cache: deploy's Rust and image jobs and CI's Rust job

A checkout gives every source a new mtime, so Cargo compiled every workspace crate again however much of target/ was restored. Now each rustc call that makes a library is looked up by its inputs in the repository's Actions cache with sccache's GitHub Actions backend, and an unchanged crate comes back from it. scripts/sccache.sh install downloads sccache 0.18.0 (musl), checks its sha256, starts the server (which reads and writes its check entry), and only then sets RUSTC_WRAPPER, SCCACHE_GHA_ENABLED, SCCACHE_IDLE_TIMEOUT=0, SCCACHE_IGNORE_SERVER_IO_ERROR=1 and CARGO_INCREMENTAL=0 for the steps after it; on any failure the job warns and builds as before. It runs before deploy.mjs install, so worker-build's install and its wasm32 builds go through it; wasm-bindgen and wasm-opt are untouched. scripts/sccache.sh stats ends each Rust job (if: always()) and puts the hits and misses in the run's summary. CI also runs on pushes to main, its Rust job only, so main's cache (the one pull requests restore from) is kept current; TypeScript and Build skip on push. Measured on the development machine (sccache's local disk cache, -j 4, every workspace source touched as a checkout does): repos and events for wasm32 release, 44.3 s and 44.6 s before, 24.2 s with the cache filled; CI's cargo test --workspace --no-run, 70.2 s and 74.6 s before, 40.6 s. What remains is what rustc links: each Worker's cdylib, binaries, test harnesses. Restoring mtimes from git was left out: a restored target/ can come from another commit, and a stale crate could deploy. crates/actions/tests/repository_workflows.rs checks the steps' order and conditions, CI's push trigger, and the pinned checksum.

syntaqxcommitted Parentbca39e9Browse files
5 files+269−160/5 viewed
+17−0
77 # typescript type checks and tests of the apps and TS services, the
88 # deploy and ops scripts, and the deploy manifest
99 # build the site, sudo and the docs build as they deploy
10+#
11+# On a push to main only `rust` runs, to keep main's caches current: a pull
12+# request's run restores from main's cache, never from another pull
13+# request's, so without it each pull request would start from nothing.
1014 name: CI
1115
1216 on:
1317 pull_request:
1418 branches: [main]
19+ push:
20+ branches: [main]
1521 workflow_dispatch:
1622
1723 # Its token only reads: it checks the code out and nothing more.
4753 !target/debug/incremental
4854 key: cargo-test-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'services/runner/base.json') }}
4955 restore-keys: cargo-test-${{ runner.os }}-
56+ # The workspace's library crates, which Cargo compiles again on every
57+ # checkout, come back from the repository's Actions cache when their
58+ # inputs did not change (scripts/sccache.sh). Test harnesses and
59+ # Workers' own crates are linked, and still compiled.
60+ - name: sccache
61+ run: bash scripts/sccache.sh install
5062 - name: Tests
5163 run: cargo test --workspace --locked --quiet
64+ - name: sccache's hits and misses
65+ if: ${{ always() }}
66+ run: bash scripts/sccache.sh stats
5267
5368 typescript:
5469 name: TypeScript
70+ if: ${{ github.event_name != 'push' }}
5571 runs-on: ubuntu-latest
5672 timeout-minutes: 30
5773 steps:
7187
7288 build:
7389 name: Build
90+ if: ${{ github.event_name != 'push' }}
7491 runs-on: ubuntu-latest
7592 timeout-minutes: 30
7693 steps:
+22−5
180180 key: cargo-crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
181181 restore-keys: cargo-crates-${{ runner.os }}-
182182 # The compiled dependencies of this job's units, for wasm32 and the
183− # build scripts and proc macros they run. The workspace's own crates
184− # are compiled again whatever is cached (a checkout's sources are
185− # newer), so an entry is saved only when the dependencies change: a
186− # new Cargo.lock, or a new base image (base.json names its Rust).
187− # Otherwise the nearest earlier entry, of any group, is a start.
183+ # build scripts and proc macros they run. Cargo calls rustc again for
184+ # the workspace's own crates whatever is cached (a checkout's sources
185+ # are newer); sccache, below, answers those calls. So an entry is
186+ # saved only when the dependencies change: a new Cargo.lock, or a new
187+ # base image (base.json names its Rust). Otherwise the nearest earlier
188+ # entry, of any group, is a start.
188189 - name: Cache the Cargo target
189190 if: ${{ matrix.rust }}
190191 uses: actions/cache@v4
215216 !target/**/incremental
216217 key: runner-musl-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'services/runner/base.json') }}
217218 restore-keys: runner-musl-${{ runner.os }}-
219+ # Every rustc call that makes a library (the workspace's crates,
220+ # which Cargo compiles again on every checkout, and any dependency
221+ # not restored above) is looked up by its inputs in the repository's
222+ # Actions cache: unchanged crates come back from it instead of being
223+ # compiled. A Worker's own crate (a cdylib) and the runner's binary
224+ # are still compiled. worker-build runs Cargo, so its wasm32 builds
225+ # go through it too; wasm-bindgen and wasm-opt are not rustc.
226+ # Pinned by version and sha256 in scripts/sccache.sh; without the
227+ # cache, the job builds as before.
228+ - name: sccache
229+ if: ${{ matrix.rust || matrix.image }}
230+ run: bash scripts/sccache.sh install
218231 - name: Install
219232 run: node scripts/deploy.mjs install --only "${{ matrix.units }}"
220233 - name: Deploy ${{ matrix.units }}
224237 # are not drafted as incidents. Optional: without it, nothing is sent.
225238 STATUS_DEPLOY_TOKEN: ${{ secrets.STATUS_DEPLOY_TOKEN }}
226239 run: node scripts/deploy.mjs deploy --only "${{ matrix.units }}" --force --no-migrations --concurrency 2
240+ # Hits and misses, on the log and in the run's summary.
241+ - name: sccache's hits and misses
242+ if: ${{ always() && (matrix.rust || matrix.image) }}
243+ run: bash scripts/sccache.sh stats
227244
228245 edge:
229246 name: edge (${{ matrix.group }})
+60−0
236236 assert!(source.contains("target/x86_64-unknown-linux-musl/release"));
237237 }
238238
239+/// Whether a step runs, for a job going `status`, with `matrix`.
240+fn step_runs(step: &workflow::Step, matrix: Value, status: Status) -> bool {
241+ let mut contexts = Map::new();
242+ contexts.insert("matrix".into(), matrix);
243+ let scope = Scope { contexts: &contexts, status, hash_files: None };
244+ expr::condition(step.condition.as_deref().unwrap_or_default(), &scope).unwrap()
245+}
246+
247+#[test]
248+fn rust_builds_go_through_sccache_before_cargo_and_report_after() {
249+ let step = |job: &workflow::Job, run: &str| -> (usize, workflow::Step) {
250+ let at = job.steps.iter().position(|s| s.run.as_deref() == Some(run)).unwrap_or_else(|| panic!("{}: no step runs {run}", job.id));
251+ (at, job.steps[at].clone())
252+ };
253+ let deploy = read("deploy.yml");
254+ for stage in ["core", "edge", "front"] {
255+ let job = deploy.jobs.iter().find(|j| j.id == stage).unwrap();
256+ let (install_at, install) = step(job, "bash scripts/sccache.sh install");
257+ let (stats_at, stats) = step(job, "bash scripts/sccache.sh stats");
258+ // Before anything runs Cargo (worker-build's install, the build),
259+ // and the statistics last.
260+ let install_step = job.steps.iter().position(|s| s.name.as_deref() == Some("Install")).unwrap();
261+ assert!(install_at < install_step, "{stage}");
262+ assert_eq!(stats_at, job.steps.len() - 1, "{stage}");
263+ for (rust, image, uses) in [(true, false, true), (false, true, true), (false, false, false)] {
264+ let matrix = json!({ "group": "g", "units": "u", "rust": rust, "image": image });
265+ assert_eq!(step_runs(&install, matrix.clone(), Status::Success), uses, "{stage} rust={rust} image={image}");
266+ assert_eq!(step_runs(&stats, matrix.clone(), Status::Success), uses, "{stage}");
267+ // A failed build still says what was cached.
268+ assert_eq!(step_runs(&stats, matrix, Status::Failure), uses, "{stage}");
269+ }
270+ }
271+ let ci = read("ci.yml");
272+ let rust = ci.jobs.iter().find(|j| j.id == "rust").unwrap();
273+ let (install_at, _) = step(rust, "bash scripts/sccache.sh install");
274+ let (tests_at, _) = step(rust, "cargo test --workspace --locked --quiet");
275+ let (stats_at, stats) = step(rust, "bash scripts/sccache.sh stats");
276+ assert!(install_at < tests_at && tests_at < stats_at);
277+ assert!(step_runs(&stats, json!({}), Status::Failure));
278+ // main's runs keep the caches pull requests restore from: only Rust.
279+ assert!(ci.trigger("push").unwrap().branches.allows("main"));
280+ assert!(ci.trigger("pull_request").is_some());
281+ for (id, on_push) in [("rust", true), ("typescript", false), ("build", false)] {
282+ let job = ci.jobs.iter().find(|j| j.id == id).unwrap();
283+ for (event, expected) in [("push", on_push), ("pull_request", true)] {
284+ let mut contexts = Map::new();
285+ contexts.insert("github".into(), json!({ "event_name": event, "ref": "refs/heads/main" }));
286+ let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
287+ assert_eq!(expr::condition(job.condition.as_deref().unwrap_or_default(), &scope).unwrap(), expected, "{id} on {event}");
288+ }
289+ }
290+ // The download is pinned by version and checksum.
291+ let script = std::fs::read_to_string(workflows_dir().join("../../scripts/sccache.sh")).unwrap();
292+ assert!(script.contains("VERSION=0.18.0"));
293+ assert!(script.lines().any(|line| line.strip_prefix("SHA256=").is_some_and(|sum| sum.len() == 64)));
294+ assert!(script.contains("sha256sum -c"));
295+ assert!(script.contains("RUSTC_WRAPPER="));
296+ assert!(script.contains("GITHUB_STEP_SUMMARY"));
297+}
298+
239299 #[test]
240300 fn the_runner_base_rebuilds_weekly_on_a_machine_with_docker() {
241301 let base = read("runner-base.yml");
+61−11
438438 | The runner binary, cold (builder container) | | 99 s |
439439 | A Rust CI job's build (events, search, repos; 4 vCPUs) | | 51 s cold, 13 s with the Cargo target restored (107 MB zstd entry) |
440440
441+- **sccache** (`scripts/sccache.sh`, pinned by version and sha256): a
442+ checkout gives every source a new mtime, so Cargo calls rustc again for
443+ every workspace crate a unit uses, however much of `target/` was
444+ restored. With `RUSTC_WRAPPER=sccache` and its GitHub Actions backend,
445+ each call that makes a library is looked up by its inputs in the
446+ repository's Actions cache (the same cache as `actions/cache`, scoped to
447+ `main`), and an unchanged crate comes back from it. Measured on the
448+ development machine on 2026-10-08, `repos` and `events` for wasm32,
449+ release, `-j 4`, the dependencies already built, every workspace source
450+ touched as a checkout does (sccache's local disk cache; the Actions
451+ cache adds a download per hit):
452+
453+ | | Time |
454+ | --- | --- |
455+ | Without sccache (before) | 44.3 s, 44.6 s |
456+ | sccache, empty cache (its first run, filling it) | 48.8 s |
457+ | sccache, filled (6 hits: `contracts`, `kit`, `rules`, `scan`, `secrets`, `blobstore`) | 24.2 s |
458+
459+ CI's build the same way (`cargo test --workspace --no-run`, debug,
460+ `-j 4`, `CARGO_INCREMENTAL=0`): 70.2 s and 74.6 s without sccache, 66.5 s
461+ filling it, 40.6 s filled (7 library hits; 23 test harnesses and
462+ cdylibs compiled).
463+
464+ What is left is each Worker's own crate: a `cdylib` is linked, and
465+ sccache does not cache what rustc links (binaries, cdylibs, proc
466+ macros, build scripts, test harnesses). That compile is real work
467+ anyway: a unit deploys because it, or a crate it uses, changed. The
468+ same holds for CI's `cargo test`: the workspace's libraries come back
469+ from the cache, the test harnesses are compiled. Every Rust job of
470+ Deploy and CI's Rust job end with sccache's hits and misses in the
471+ run's summary. If the cache cannot be reached, or the download fails
472+ its checksum, the job warns and builds as before. On g1t, sccache
473+ speaks the toolkit cache's older protocol (`GITHUB_SERVER_URL` is not
474+ github.com): a lookup, a reservation, one `PATCH` of the whole entry
475+ (`Content-Range: bytes 0-N/*`) and a commit per miss; a lookup and a
476+ blob `GET` per hit. Its storage check saves `sccache/.sccache_check`
477+ once, and takes the `409` on later runs as "already there".
478+- **Not restoring mtimes:** setting each file's mtime from git (so Cargo
479+ would trust the restored `target/`) was considered and left out. A
480+ restored `target/` may come from another commit (the cache's
481+ `restore-keys` take the nearest earlier entry, of any group), and a
482+ file changed by an older commit than that build would look unchanged:
483+ a stale crate deployed. sccache looks at what is compiled, not when.
441484 - **Only what changed** is the largest saving: a change to one service
442− deploys one service.
485+ deploys one service, and a change only to a crate's tests deploys
486+ nothing.
443487
444488 ## The workflow
445489
468512 be rebuilt alone (`image: true` in the matrix, also on `g1t-4core`, where
469513 it builds and pushes the image with the job's own Docker Engine). `fail-fast: false`, so one failed job does not cut
470514 another off mid-upload; the next stage then does not start.
471−- **Tests:** there is no CI workflow on g1t yet; `main` is kept passing by
472− the merge queue's checks. `check` runs the deploy tool's own tests. When a
473− CI workflow is added, make `migrate` and the stages wait for it (`workflow_run`, or a job in
474− this file).
515+- **Tests:** `.g1t/workflows/ci.yml` tests every pull request into `main`
516+ (and, on pushes to `main`, runs only its Rust job, to keep the caches
517+ pull requests restore from current). `check` runs the deploy tool's own
518+ tests. Deploy does not wait for CI.
475519 - **Machines:** Rust jobs and the runner's image run on `g1t-4core` (4 vCPUs,
476520 12 GiB, 20 GB), the others on the standard machine
477521 (`runs-on: ${{ (matrix.rust || matrix.image) && 'g1t-4core' || 'ubuntu-latest' }}`).
482526 downloaded tools, `~/.cargo/registry/cache`, and the Cargo target's
483527 release dependencies (`target/release` and
484528 `target/wasm32-unknown-unknown/release`, without `incremental` or
485− `.wasm`), keyed by the build group, `Cargo.lock` and `base.json`. The
486− workspace's own crates are compiled again on every run (a checkout's
487− sources are newer than any cache); the crates.io dependencies are not.
488− npm's cache is not kept: every job runs `npm ci` of only what its units
489− need (`deploy.mjs install`: Wrangler alone for Rust jobs).
529+ `.wasm`), keyed by the build group, `Cargo.lock` and `base.json`. Cargo
530+ calls rustc again for the workspace's own crates on every run (a
531+ checkout's sources are newer than any cache); **sccache** answers those
532+ calls from the repository's Actions cache when a crate's inputs did not
533+ change (see [Build speed](#build-speed)). npm's cache is not kept: every
534+ job runs `npm ci` of only what its units need (`deploy.mjs install`:
535+ Wrangler alone for Rust jobs).
490536 - **Conditions:** each stage runs with `!failure() && !cancelled()`, which
491537 on g1t (as on GitHub) is true when no job before it failed, however far
492538 back: a `migrate` job skipped for having nothing to apply does not stop
516562 miss. The image job adds `x86_64-unknown-linux-musl` (about 30 MB from
517563 `static.rust-lang.org`) and keeps its Cargo target in the cache. worker-build
518564 fetches wasm-bindgen and wasm-opt from GitHub releases and esbuild from
519−npm. All of those hosts are on the list every workflow job may reach.
565+npm. Each Rust job downloads sccache (about 10 MB) from its GitHub release
566+too (`scripts/sccache.sh`, which checks its sha256); it is not in the base
567+image, and putting it there means a pinned download in the base's
568+Dockerfile and a base rebuild (`build-base`). All of those hosts are on
569+the list every workflow job may reach.
520570
521571 ### Network
522572
+109−0
1+#!/usr/bin/env bash
2+# sccache for a workflow's Rust builds. Every rustc call that makes a
3+# library is looked up by what it compiles (sources, flags, toolchain,
4+# dependencies) in the repository's Actions cache, the one `actions/cache`
5+# uses, through sccache's GitHub Actions backend. A checkout gives every
6+# source file a new mtime, so Cargo calls rustc again for the workspace's
7+# own crates however much of `target/` was restored; with sccache, the
8+# crates whose inputs did not change come back from the cache instead of
9+# being compiled. Binaries, cdylibs (a Worker's own crate), proc macros and
10+# test harnesses are linked, and sccache compiles those as before.
11+#
12+# bash scripts/sccache.sh install # a step before the first cargo
13+# bash scripts/sccache.sh stats # the last step, with if: always()
14+#
15+# install downloads the pinned release and checks its sha256, starts the
16+# server (which reads and writes a check entry in the cache), and only then
17+# sets RUSTC_WRAPPER for the steps after it. If anything there fails, the
18+# job builds as it did without sccache, and the step says why.
19+# stats prints the server's hits and misses, and adds them to the job's
20+# summary.
21+#
22+# See docs/DEPLOYING.md ("Build speed") and the Actions guide's
23+# "Caching Rust builds".
24+set -euo pipefail
25+
26+VERSION=0.18.0
27+TARGET=x86_64-unknown-linux-musl
28+# sccache-v0.18.0-x86_64-unknown-linux-musl.tar.gz, from the release's
29+# .sha256 file.
30+SHA256=45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89
31+
32+BIN="$HOME/.cargo/bin"
33+SCCACHE="$BIN/sccache"
34+
35+# What sccache and Cargo read, here and in the steps after this one.
36+export SCCACHE_GHA_ENABLED=true
37+# One server for the whole job: its statistics are the job's.
38+export SCCACHE_IDLE_TIMEOUT=0
39+# If the server goes away mid-build, rustc runs without it.
40+export SCCACHE_IGNORE_SERVER_IO_ERROR=1
41+# sccache cannot cache incremental builds; a fresh checkout gains nothing
42+# from them anyway.
43+export CARGO_INCREMENTAL=0
44+
45+warn() { echo "::warning title=sccache::$1, so this job builds without it"; }
46+
47+fetch() {
48+ local name="sccache-v${VERSION}-${TARGET}"
49+ local dir="${RUNNER_TEMP:-/tmp}/sccache-${VERSION}"
50+ if [ -x "$SCCACHE" ] && [ "$("$SCCACHE" --version 2>/dev/null | awk '{print $2}')" = "$VERSION" ]; then
51+ return 0
52+ fi
53+ mkdir -p "$dir" "$BIN"
54+ curl -fsSL --retry 3 -o "$dir/$name.tar.gz" "https://github.com/mozilla/sccache/releases/download/v${VERSION}/${name}.tar.gz" || return 1
55+ echo "${SHA256} $dir/$name.tar.gz" | sha256sum -c --quiet - || return 1
56+ tar -xzf "$dir/$name.tar.gz" -C "$dir" || return 1
57+ install -m 0755 "$dir/$name/sccache" "$SCCACHE"
58+}
59+
60+install_sccache() {
61+ if [ -z "${ACTIONS_RUNTIME_TOKEN:-}" ] || [ -z "${ACTIONS_CACHE_URL:-}${ACTIONS_RESULTS_URL:-}" ]; then
62+ warn "The job has no Actions cache to use (ACTIONS_RUNTIME_TOKEN, ACTIONS_CACHE_URL)"
63+ return 0
64+ fi
65+ if ! fetch; then
66+ warn "sccache ${VERSION} could not be downloaded or did not match its checksum"
67+ return 0
68+ fi
69+ if ! "$SCCACHE" --start-server; then
70+ warn "sccache's server did not start (it could not read the cache)"
71+ return 0
72+ fi
73+ {
74+ echo "RUSTC_WRAPPER=$SCCACHE"
75+ echo "SCCACHE_GHA_ENABLED=$SCCACHE_GHA_ENABLED"
76+ echo "SCCACHE_IDLE_TIMEOUT=$SCCACHE_IDLE_TIMEOUT"
77+ echo "SCCACHE_IGNORE_SERVER_IO_ERROR=$SCCACHE_IGNORE_SERVER_IO_ERROR"
78+ echo "CARGO_INCREMENTAL=$CARGO_INCREMENTAL"
79+ } >> "$GITHUB_ENV"
80+ echo "sccache ${VERSION}: rustc goes through it from here on, cached in this repository's Actions cache."
81+}
82+
83+stats() {
84+ if [ -z "${RUSTC_WRAPPER:-}" ] || [ ! -x "$SCCACHE" ]; then
85+ echo "sccache was not used in this job."
86+ return 0
87+ fi
88+ local out
89+ out="$("$SCCACHE" --show-stats 2>&1)" || true
90+ echo "$out"
91+ if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
92+ {
93+ echo "### sccache"
94+ echo
95+ echo '```'
96+ echo "$out"
97+ echo '```'
98+ } >> "$GITHUB_STEP_SUMMARY"
99+ fi
100+}
101+
102+case "${1:-}" in
103+ install) install_sccache ;;
104+ stats) stats ;;
105+ *)
106+ echo "usage: bash scripts/sccache.sh install|stats" >&2
107+ exit 2
108+ ;;
109+esac