| 1 | // The keys of the Rust caches in g1t's own workflows, and the pruning of a |
| 2 | // Cargo target before it is saved. CONTRIBUTING.md ("How g1t's own CI |
| 3 | // caches") has the whole scheme. |
| 4 | // |
| 5 | // node scripts/deploy/ci-cache.mjs keys |
| 6 | // node scripts/deploy/ci-cache.mjs prune <profile dir>... -- <cargo args> |
| 7 | // |
| 8 | // keys writes to the step's outputs (GITHUB_OUTPUT): |
| 9 | // |
| 10 | // rust rustc's version, such as 1.99.0. Everything Cargo compiles |
| 11 | // depends on it, so a new toolchain starts a new line of entries |
| 12 | // rather than restoring the old one's, whose files it would |
| 13 | // only carry along unused. |
| 14 | // cargo a hash of the packages Cargo.lock takes from crates.io and git, |
| 15 | // leaving out the workspace's own: a release bumps a crate's |
| 16 | // version, which changes Cargo.lock and nothing that is cached. |
| 17 | // Keyed by the whole file, 25 of the 31 changes to Cargo.lock in |
| 18 | // the week before this were version bumps, and each saved every |
| 19 | // Rust cache again (the test build's 0.7 to 1.7 GB each time). |
| 20 | // desktop the same for apps/desktop/src-tauri/Cargo.lock. |
| 21 | // zigbuild the first 12 hex characters of the cargo-zigbuild image's |
| 22 | // digest (scripts/cli-release.mjs): the CLI's release builds in |
| 23 | // it, with its own Rust, so its cache is keyed by the image |
| 24 | // rather than by this machine's rustc. |
| 25 | // |
| 26 | // prune runs Cargo again with --message-format=json once the build is done |
| 27 | // (everything is fresh, so it only lists what it built) and deletes, from |
| 28 | // each profile dir's deps/, build/ and .fingerprint/, the files and folders |
| 29 | // of units this build did not use. A target restored from an older entry |
| 30 | // (a restore key) keeps the old versions of whatever changed beside the new |
| 31 | // ones, and saving it carried them on forever: CI's test build grew from |
| 32 | // 671 MB to 1,687 MB in a day, two thirds of the way to the 2 GB an entry |
| 33 | // may hold, and its restore from 15 s to a minute. Anything unexpected |
| 34 | // (Cargo fails, lists nothing) prunes nothing. |
| 35 | |
| 36 | import { spawnSync } from "node:child_process"; |
| 37 | import { createHash } from "node:crypto"; |
| 38 | import { appendFileSync, existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; |
| 39 | import { basename, join } from "node:path"; |
| 40 | import { fileURLToPath } from "node:url"; |
| 41 | |
| 42 | import { BUILDER } from "../cli-release.mjs"; |
| 43 | |
| 44 | const ROOT = fileURLToPath(new URL("../..", import.meta.url)); |
| 45 | |
| 46 | /** The `[[package]]` blocks of a Cargo.lock that come from a registry or git: not the workspace's own. */ |
| 47 | export function externalPackages(lock) { |
| 48 | return lock |
| 49 | .split(/^\[\[package\]\]\s*$/m) |
| 50 | .slice(1) |
| 51 | .map((block) => block.trim()) |
| 52 | .filter((block) => /^source = /m.test(block)) |
| 53 | .sort(); |
| 54 | } |
| 55 | |
| 56 | /** 16 hex characters of the SHA-256 of a Cargo.lock's external packages. */ |
| 57 | export function depsHash(lock) { |
| 58 | return createHash("sha256").update(externalPackages(lock).join("\n\n")).digest("hex").slice(0, 16); |
| 59 | } |
| 60 | |
| 61 | /** 12 hex characters of an image reference's digest (`name@sha256:…`); null for a tag, which moves. */ |
| 62 | export function imageDigest(reference) { |
| 63 | return reference.match(/@sha256:([0-9a-f]{64})$/)?.[1].slice(0, 12) ?? null; |
| 64 | } |
| 65 | |
| 66 | /** `1.99.0` from `rustc 1.99.0 (90b35a623 2026-09-30)`; null if it is not that. */ |
| 67 | export function rustVersion(text) { |
| 68 | return text.match(/^rustc (\S+)/)?.[1] ?? null; |
| 69 | } |
| 70 | |
| 71 | function keys() { |
| 72 | const rustc = spawnSync("rustc", ["-V"], { encoding: "utf8" }); |
| 73 | const rust = rustVersion(rustc.stdout ?? ""); |
| 74 | if (!rust) throw new Error(`rustc -V said ${JSON.stringify(rustc.stdout ?? rustc.error?.message)}`); |
| 75 | const zigbuild = imageDigest(BUILDER); |
| 76 | if (!zigbuild) throw new Error(`scripts/cli-release.mjs builds in ${BUILDER}, which is not pinned by digest`); |
| 77 | const outputs = { rust, zigbuild }; |
| 78 | for (const [name, path] of [ |
| 79 | ["cargo", "Cargo.lock"], |
| 80 | ["desktop", "apps/desktop/src-tauri/Cargo.lock"], |
| 81 | ]) { |
| 82 | const file = join(ROOT, path); |
| 83 | if (existsSync(file)) outputs[name] = depsHash(readFileSync(file, "utf8")); |
| 84 | } |
| 85 | const lines = Object.entries(outputs).map(([k, v]) => `${k}=${v}`); |
| 86 | console.log(lines.join("\n")); |
| 87 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); |
| 88 | } |
| 89 | |
| 90 | /** The 16-hex-character hash Cargo puts on a unit's files and folders (`libserde-0123456789abcdef.rlib`), or null. */ |
| 91 | export function unitHash(name) { |
| 92 | return name.match(/-([0-9a-f]{16})(?:\.[^/]*)?$/)?.[1] ?? null; |
| 93 | } |
| 94 | |
| 95 | /** The hashes of every unit in Cargo's JSON messages: its files, and a build script's out dir. */ |
| 96 | export function usedHashes(messages) { |
| 97 | const used = new Set(); |
| 98 | for (const line of messages.split("\n")) { |
| 99 | if (!line.startsWith("{")) continue; |
| 100 | let m; |
| 101 | try { |
| 102 | m = JSON.parse(line); |
| 103 | } catch { |
| 104 | continue; |
| 105 | } |
| 106 | const paths = [...(m.filenames ?? []), ...(m.executable ? [m.executable] : [])]; |
| 107 | if (m.reason === "build-script-executed" && m.out_dir) paths.push(m.out_dir.replace(/[\\/]out$/, "")); |
| 108 | for (const path of paths) { |
| 109 | // A unit's own folder (build/foo-<hash>/build-script-build) or file (deps/libfoo-<hash>.rlib). |
| 110 | for (const part of path.split(/[\\/]/).slice(-2)) { |
| 111 | const hash = unitHash(part); |
| 112 | if (hash) used.add(hash); |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | return used; |
| 117 | } |
| 118 | |
| 119 | /** What to delete from a profile dir: deps/ files and build/, .fingerprint/ folders of units not in `used`. */ |
| 120 | export function stale(dir, used) { |
| 121 | const out = []; |
| 122 | for (const sub of ["deps", "build", ".fingerprint"]) { |
| 123 | const at = join(dir, sub); |
| 124 | if (!existsSync(at)) continue; |
| 125 | for (const name of readdirSync(at)) { |
| 126 | const hash = unitHash(name); |
| 127 | if (hash && !used.has(hash)) out.push(join(at, name)); |
| 128 | } |
| 129 | } |
| 130 | return out; |
| 131 | } |
| 132 | |
| 133 | function bytes(path) { |
| 134 | const s = statSync(path, { throwIfNoEntry: false }); |
| 135 | if (!s) return 0; |
| 136 | if (!s.isDirectory()) return s.size; |
| 137 | return readdirSync(path).reduce((sum, name) => sum + bytes(join(path, name)), 0); |
| 138 | } |
| 139 | |
| 140 | function prune(args) { |
| 141 | const split = args.indexOf("--"); |
| 142 | const dirs = split < 0 ? [] : args.slice(0, split); |
| 143 | const cargo = split < 0 ? [] : args.slice(split + 1); |
| 144 | if (!dirs.length || !cargo.length) throw new Error("usage: ci-cache.mjs prune <profile dir>... -- <cargo args>"); |
| 145 | const listed = spawnSync("cargo", [...cargo, "--message-format=json"], { |
| 146 | cwd: ROOT, |
| 147 | encoding: "utf8", |
| 148 | maxBuffer: 1 << 30, |
| 149 | stdio: ["ignore", "pipe", "inherit"], |
| 150 | }); |
| 151 | const used = listed.status === 0 ? usedHashes(listed.stdout) : new Set(); |
| 152 | if (!used.size) { |
| 153 | console.log(`::warning title=Cargo cache::cargo ${cargo.join(" ")} did not list what it built, so nothing is pruned`); |
| 154 | return; |
| 155 | } |
| 156 | for (const dir of dirs) { |
| 157 | const before = bytes(join(ROOT, dir)); |
| 158 | const gone = stale(join(ROOT, dir), used); |
| 159 | for (const path of gone) rmSync(path, { recursive: true, force: true }); |
| 160 | const after = bytes(join(ROOT, dir)); |
| 161 | const mb = (n) => `${Math.round(n / 1_048_576)} MB`; |
| 162 | console.log(`${dir}: ${gone.length} files and folders of units this build did not use, ${mb(before - after)}; ${mb(after)} left`); |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | if (process.argv[1] && basename(process.argv[1]) === "ci-cache.mjs") { |
| 167 | const [command, ...rest] = process.argv.slice(2); |
| 168 | if (command === "keys") keys(); |
| 169 | else if (command === "prune") prune(rest); |
| 170 | else { |
| 171 | console.error("usage: node scripts/deploy/ci-cache.mjs keys | prune <profile dir>... -- <cargo args>"); |
| 172 | process.exit(2); |
| 173 | } |
| 174 | } |