Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge branch 'worktree-agent-a63425cb2d4935c5f' | 1 | // The keys of the Rust caches in g1t's own workflows, and the pruning of a |
| 2 | // Cargo target before it is saved. CONTRIBUTING.md ("How g1t's own CI | |
| 3 | // caches") has the whole scheme. | |
| 4 | // | |
| 5 | // node scripts/deploy/ci-cache.mjs keys | |
| 6 | // node scripts/deploy/ci-cache.mjs prune <profile dir>... -- <cargo args> | |
| 7 | // | |
| 8 | // keys writes to the step's outputs (GITHUB_OUTPUT): | |
| 9 | // | |
| 10 | // rust rustc's version, such as 1.99.0. Everything Cargo compiles | |
| 11 | // depends on it, so a new toolchain starts a new line of entries | |
| 12 | // rather than restoring the old one's, whose files it would | |
| 13 | // only carry along unused. | |
| 14 | // cargo a hash of the packages Cargo.lock takes from crates.io and git, | |
| 15 | // leaving out the workspace's own: a release bumps a crate's | |
| 16 | // version, which changes Cargo.lock and nothing that is cached. | |
| 17 | // Keyed by the whole file, 25 of the 31 changes to Cargo.lock in | |
| 18 | // the week before this were version bumps, and each saved every | |
| 19 | // Rust cache again (the test build's 0.7 to 1.7 GB each time). | |
| 20 | // desktop the same for apps/desktop/src-tauri/Cargo.lock. | |
| 21 | // | |
| 22 | // prune runs Cargo again with --message-format=json once the build is done | |
| 23 | // (everything is fresh, so it only lists what it built) and deletes, from | |
| 24 | // each profile dir's deps/, build/ and .fingerprint/, the files and folders | |
| 25 | // of units this build did not use. A target restored from an older entry | |
| 26 | // (a restore key) keeps the old versions of whatever changed beside the new | |
| 27 | // ones, and saving it carried them on forever: CI's test build grew from | |
| 28 | // 671 MB to 1,687 MB in a day, two thirds of the way to the 2 GB an entry | |
| 29 | // may hold, and its restore from 15 s to a minute. Anything unexpected | |
| 30 | // (Cargo fails, lists nothing) prunes nothing. | |
| 31 | ||
| 32 | import { spawnSync } from "node:child_process"; | |
| 33 | import { createHash } from "node:crypto"; | |
| 34 | import { appendFileSync, existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; | |
| 35 | import { basename, join } from "node:path"; | |
| 36 | import { fileURLToPath } from "node:url"; | |
| 37 | ||
| 38 | const ROOT = fileURLToPath(new URL("../..", import.meta.url)); | |
| 39 | ||
| 40 | /** The `[[package]]` blocks of a Cargo.lock that come from a registry or git: not the workspace's own. */ | |
| 41 | export function externalPackages(lock) { | |
| 42 | return lock | |
| 43 | .split(/^\[\[package\]\]\s*$/m) | |
| 44 | .slice(1) | |
| 45 | .map((block) => block.trim()) | |
| 46 | .filter((block) => /^source = /m.test(block)) | |
| 47 | .sort(); | |
| 48 | } | |
| 49 | ||
| 50 | /** 16 hex characters of the SHA-256 of a Cargo.lock's external packages. */ | |
| 51 | export function depsHash(lock) { | |
| 52 | return createHash("sha256").update(externalPackages(lock).join("\n\n")).digest("hex").slice(0, 16); | |
| 53 | } | |
| 54 | ||
| 55 | /** `1.99.0` from `rustc 1.99.0 (90b35a623 2026-09-30)`; null if it is not that. */ | |
| 56 | export function rustVersion(text) { | |
| 57 | return text.match(/^rustc (\S+)/)?.[1] ?? null; | |
| 58 | } | |
| 59 | ||
| 60 | function keys() { | |
| 61 | const rustc = spawnSync("rustc", ["-V"], { encoding: "utf8" }); | |
| 62 | const rust = rustVersion(rustc.stdout ?? ""); | |
| 63 | if (!rust) throw new Error(`rustc -V said ${JSON.stringify(rustc.stdout ?? rustc.error?.message)}`); | |
| 64 | const outputs = { rust }; | |
| 65 | for (const [name, path] of [ | |
| 66 | ["cargo", "Cargo.lock"], | |
| 67 | ["desktop", "apps/desktop/src-tauri/Cargo.lock"], | |
| 68 | ]) { | |
| 69 | const file = join(ROOT, path); | |
| 70 | if (existsSync(file)) outputs[name] = depsHash(readFileSync(file, "utf8")); | |
| 71 | } | |
| 72 | const lines = Object.entries(outputs).map(([k, v]) => `${k}=${v}`); | |
| 73 | console.log(lines.join("\n")); | |
| 74 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); | |
| 75 | } | |
| 76 | ||
| 77 | /** The 16-hex-character hash Cargo puts on a unit's files and folders (`libserde-0123456789abcdef.rlib`), or null. */ | |
| 78 | export function unitHash(name) { | |
| 79 | return name.match(/-([0-9a-f]{16})(?:\.[^/]*)?$/)?.[1] ?? null; | |
| 80 | } | |
| 81 | ||
| 82 | /** The hashes of every unit in Cargo's JSON messages: its files, and a build script's out dir. */ | |
| 83 | export function usedHashes(messages) { | |
| 84 | const used = new Set(); | |
| 85 | for (const line of messages.split("\n")) { | |
| 86 | if (!line.startsWith("{")) continue; | |
| 87 | let m; | |
| 88 | try { | |
| 89 | m = JSON.parse(line); | |
| 90 | } catch { | |
| 91 | continue; | |
| 92 | } | |
| 93 | const paths = [...(m.filenames ?? []), ...(m.executable ? [m.executable] : [])]; | |
| 94 | if (m.reason === "build-script-executed" && m.out_dir) paths.push(m.out_dir.replace(/[\\/]out$/, "")); | |
| 95 | for (const path of paths) { | |
| 96 | // A unit's own folder (build/foo-<hash>/build-script-build) or file (deps/libfoo-<hash>.rlib). | |
| 97 | for (const part of path.split(/[\\/]/).slice(-2)) { | |
| 98 | const hash = unitHash(part); | |
| 99 | if (hash) used.add(hash); | |
| 100 | } | |
| 101 | } | |
| 102 | } | |
| 103 | return used; | |
| 104 | } | |
| 105 | ||
| 106 | /** What to delete from a profile dir: deps/ files and build/, .fingerprint/ folders of units not in `used`. */ | |
| 107 | export function stale(dir, used) { | |
| 108 | const out = []; | |
| 109 | for (const sub of ["deps", "build", ".fingerprint"]) { | |
| 110 | const at = join(dir, sub); | |
| 111 | if (!existsSync(at)) continue; | |
| 112 | for (const name of readdirSync(at)) { | |
| 113 | const hash = unitHash(name); | |
| 114 | if (hash && !used.has(hash)) out.push(join(at, name)); | |
| 115 | } | |
| 116 | } | |
| 117 | return out; | |
| 118 | } | |
| 119 | ||
| 120 | function bytes(path) { | |
| 121 | const s = statSync(path, { throwIfNoEntry: false }); | |
| 122 | if (!s) return 0; | |
| 123 | if (!s.isDirectory()) return s.size; | |
| 124 | return readdirSync(path).reduce((sum, name) => sum + bytes(join(path, name)), 0); | |
| 125 | } | |
| 126 | ||
| 127 | function prune(args) { | |
| 128 | const split = args.indexOf("--"); | |
| 129 | const dirs = split < 0 ? [] : args.slice(0, split); | |
| 130 | const cargo = split < 0 ? [] : args.slice(split + 1); | |
| 131 | if (!dirs.length || !cargo.length) throw new Error("usage: ci-cache.mjs prune <profile dir>... -- <cargo args>"); | |
| 132 | const listed = spawnSync("cargo", [...cargo, "--message-format=json"], { | |
| 133 | cwd: ROOT, | |
| 134 | encoding: "utf8", | |
| 135 | maxBuffer: 1 << 30, | |
| 136 | stdio: ["ignore", "pipe", "inherit"], | |
| 137 | }); | |
| 138 | const used = listed.status === 0 ? usedHashes(listed.stdout) : new Set(); | |
| 139 | if (!used.size) { | |
| 140 | console.log(`::warning title=Cargo cache::cargo ${cargo.join(" ")} did not list what it built, so nothing is pruned`); | |
| 141 | return; | |
| 142 | } | |
| 143 | for (const dir of dirs) { | |
| 144 | const before = bytes(join(ROOT, dir)); | |
| 145 | const gone = stale(join(ROOT, dir), used); | |
| 146 | for (const path of gone) rmSync(path, { recursive: true, force: true }); | |
| 147 | const after = bytes(join(ROOT, dir)); | |
| 148 | const mb = (n) => `${Math.round(n / 1_048_576)} MB`; | |
| 149 | console.log(`${dir}: ${gone.length} files and folders of units this build did not use, ${mb(before - after)}; ${mb(after)} left`); | |
| 150 | } | |
| 151 | } | |
| 152 | ||
| 153 | if (process.argv[1] && basename(process.argv[1]) === "ci-cache.mjs") { | |
| 154 | const [command, ...rest] = process.argv.slice(2); | |
| 155 | if (command === "keys") keys(); | |
| 156 | else if (command === "prune") prune(rest); | |
| 157 | else { | |
| 158 | console.error("usage: node scripts/deploy/ci-cache.mjs keys | prune <profile dir>... -- <cargo args>"); | |
| 159 | process.exit(2); | |
| 160 | } | |
| 161 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.