| 1 | // Which of a CI Rust part's crates a run tests. On a pull request, the |
| 2 | // workspace crates whose files it changes and every workspace crate that |
| 3 | // depends on them (from `cargo metadata`, through dependencies of every |
| 4 | // kind); anywhere else, every one. Most pull requests touch a few crates, |
| 5 | // and a part compiled and tested all of its own whatever changed. |
| 6 | // |
| 7 | // node scripts/deploy/ci-affected.mjs -p <crate>... |
| 8 | // node scripts/deploy/ci-affected.mjs --job desktop|build |
| 9 | // |
| 10 | // With --job it says whether a pull request needs one of CI's other jobs |
| 11 | // (JOBS), as the output `run` (true or false), on the same terms: not a |
| 12 | // pull request, or anything failing, runs it. |
| 13 | // |
| 14 | // It takes the part's `-p` arguments and writes to the step's outputs |
| 15 | // (GITHUB_OUTPUT): |
| 16 | // |
| 17 | // crates those of them to test, in the same order: all, some or none |
| 18 | // whole true when that is all of them |
| 19 | // |
| 20 | // What a pull request changes is its test merge commit against its first |
| 21 | // parent, the base branch's tip it was merged into (services/repos |
| 22 | // test_merge.rs), so the checkout fetches two commits deep. When the commit |
| 23 | // is not a merge (g1t could not make one and the run builds the head), or |
| 24 | // anything here fails, every crate is tested. |
| 25 | // |
| 26 | // A file outside every crate's folder tests the crates that read it |
| 27 | // (READS), or every crate when the build itself reads it (EVERYTHING), and |
| 28 | // nothing otherwise: the site, the TypeScript services and the docs are |
| 29 | // not the Rust build's. ci-affected.test.mjs checks that every file a |
| 30 | // crate's source names outside its own folder is in one or the other. |
| 31 | |
| 32 | import { spawnSync } from "node:child_process"; |
| 33 | import { appendFileSync } from "node:fs"; |
| 34 | import { basename, dirname, relative, sep } from "node:path"; |
| 35 | import { fileURLToPath } from "node:url"; |
| 36 | |
| 37 | const ROOT = fileURLToPath(new URL("../..", import.meta.url)); |
| 38 | |
| 39 | /** |
| 40 | * Files any change to which tests every crate: the workspace's manifest |
| 41 | * and lockfile, the toolchain and Cargo's settings, this workflow, and the |
| 42 | * scripts its Rust job runs. A path ending in `/` is a folder. |
| 43 | */ |
| 44 | export const EVERYTHING = [ |
| 45 | "Cargo.toml", |
| 46 | "Cargo.lock", |
| 47 | "rust-toolchain", |
| 48 | "rust-toolchain.toml", |
| 49 | ".cargo/", |
| 50 | ".g1t/workflows/ci.yml", |
| 51 | "scripts/sccache.sh", |
| 52 | "scripts/deploy/ci-affected.mjs", |
| 53 | "scripts/deploy/ci-cache.mjs", |
| 54 | // ci-cache.mjs keys reads the CLI builder's digest from it. |
| 55 | "scripts/cli-release.mjs", |
| 56 | ]; |
| 57 | |
| 58 | /** |
| 59 | * Files outside a crate's folder (or in another's) that a crate's code or |
| 60 | * tests read, and the crates that read them. A path ending in `/` is a |
| 61 | * folder. |
| 62 | */ |
| 63 | export const READS = [ |
| 64 | // repository_workflows.rs reads g1t's workflows, and the managed deploy |
| 65 | // workflow and g1t's own actions. |
| 66 | [".g1t/workflows/", ["g1t-actions"]], |
| 67 | [".github/workflows/", ["g1t-actions"]], |
| 68 | ["workflows/", ["g1t-actions"]], |
| 69 | ["actions/", ["g1t-actions"]], |
| 70 | // The Rust contracts are checked against their TypeScript mirror. |
| 71 | ["packages/contracts/src/", ["g1t-contracts"]], |
| 72 | // The CLI checks the commands its docs and the plugins name, and the |
| 73 | // API's routes against the OpenAPI document; the agent sessions hand out |
| 74 | // the plugin's skills. |
| 75 | ["apps/docs/src/content/docs/", ["g1t"]], |
| 76 | ["apps/docs/src/data/openapi.json", ["g1t", "g1t-api"]], |
| 77 | ["plugins/", ["g1t", "g1t-integrations"]], |
| 78 | ["README.md", ["g1t"]], |
| 79 | // The sandbox's image, built into the runner. |
| 80 | ["deploy/sandbox/", ["g1t-runner"]], |
| 81 | // The merge queue's tests check that the runner's sweep builds a queue |
| 82 | // waiting on a refusal again, every five minutes. |
| 83 | ["services/runner/src/index.ts", ["g1t-work"]], |
| 84 | ["services/runner/wrangler.jsonc", ["g1t-work"]], |
| 85 | // Code scanning's tests read the scanner's fixtures. |
| 86 | ["crates/scan/fixtures/", ["g1t-security"]], |
| 87 | ]; |
| 88 | |
| 89 | /** |
| 90 | * The desktop job's files: the app, outside the workspace with a lockfile |
| 91 | * of its own and no path dependency on it; the release scripts whose tests |
| 92 | * the job runs, and the CLI's manifest cli-release.mjs reads its version |
| 93 | * from; what installs Tauri's packages; and the toolchain and this |
| 94 | * workflow's own files. |
| 95 | */ |
| 96 | export const DESKTOP = [ |
| 97 | "apps/desktop/", |
| 98 | "scripts/desktop-release.mjs", |
| 99 | "scripts/desktop-release.test.mjs", |
| 100 | "scripts/mobile-release.mjs", |
| 101 | "scripts/mobile-release.test.mjs", |
| 102 | "scripts/cli-release.mjs", |
| 103 | "scripts/cli-release.test.mjs", |
| 104 | "crates/g1t/Cargo.toml", |
| 105 | "scripts/apt-install.sh", |
| 106 | "rust-toolchain", |
| 107 | "rust-toolchain.toml", |
| 108 | ".cargo/", |
| 109 | ".g1t/workflows/ci.yml", |
| 110 | "scripts/deploy/ci-affected.mjs", |
| 111 | "scripts/deploy/ci-cache.mjs", |
| 112 | ]; |
| 113 | |
| 114 | /** |
| 115 | * CI's other jobs a pull request may pass without running, and whether a |
| 116 | * changed file needs each: `desktop` its own files (DESKTOP), `build` (the |
| 117 | * site, sudo and the docs) any file but Rust source, which none of them |
| 118 | * reads. TypeScript runs on every pull request: its tests and the |
| 119 | * conformance suite read Rust source and manifests throughout. |
| 120 | */ |
| 121 | export const JOBS = { |
| 122 | desktop: (file) => DESKTOP.some((entry) => under(file, entry)), |
| 123 | build: (file) => !file.endsWith(".rs"), |
| 124 | }; |
| 125 | |
| 126 | /** Whether a change to `files` needs `job` (a key of JOBS) to run. */ |
| 127 | export function needs(job, files) { |
| 128 | return files.some(JOBS[job]); |
| 129 | } |
| 130 | |
| 131 | /** Whether `path` is `entry`, or under it when `entry` is a folder. */ |
| 132 | function under(path, entry) { |
| 133 | return entry.endsWith("/") ? path.startsWith(entry) : path === entry; |
| 134 | } |
| 135 | |
| 136 | /** |
| 137 | * The workspace's crates from `cargo metadata --no-deps`: each one's name, |
| 138 | * its folder relative to the workspace root, and the workspace crates it |
| 139 | * depends on. |
| 140 | */ |
| 141 | export function workspace(metadata) { |
| 142 | const root = metadata.workspace_root; |
| 143 | const members = new Set(metadata.workspace_members); |
| 144 | const packages = metadata.packages.filter((p) => members.has(p.id)); |
| 145 | const names = new Set(packages.map((p) => p.name)); |
| 146 | return packages.map((p) => ({ |
| 147 | name: p.name, |
| 148 | dir: relative(root, dirname(p.manifest_path)).split(sep).join("/"), |
| 149 | deps: [...new Set(p.dependencies.filter((d) => d.path && names.has(d.name)).map((d) => d.name))], |
| 150 | })); |
| 151 | } |
| 152 | |
| 153 | /** |
| 154 | * The crates to test for a change to `files` (paths from the repository's |
| 155 | * root): `null` for every one, otherwise their names. A file belongs to the |
| 156 | * crate whose folder holds it most closely. |
| 157 | */ |
| 158 | export function affected(crates, files) { |
| 159 | const touched = new Set(); |
| 160 | for (const file of files) { |
| 161 | if (EVERYTHING.some((entry) => under(file, entry))) return null; |
| 162 | const owner = crates.filter((c) => file.startsWith(`${c.dir}/`)).sort((a, b) => b.dir.length - a.dir.length)[0]; |
| 163 | if (owner) touched.add(owner.name); |
| 164 | for (const [entry, readers] of READS) if (under(file, entry)) readers.forEach((name) => touched.add(name)); |
| 165 | } |
| 166 | const dependents = new Map(crates.map((c) => [c.name, []])); |
| 167 | for (const c of crates) for (const dep of c.deps) dependents.get(dep)?.push(c.name); |
| 168 | const out = new Set(); |
| 169 | const queue = [...touched].filter((name) => dependents.has(name)); |
| 170 | while (queue.length) { |
| 171 | const name = queue.pop(); |
| 172 | if (out.has(name)) continue; |
| 173 | out.add(name); |
| 174 | queue.push(...dependents.get(name)); |
| 175 | } |
| 176 | return [...out].sort(); |
| 177 | } |
| 178 | |
| 179 | /** A part's crates from its `-p` arguments; throws on anything else. */ |
| 180 | export function partCrates(args) { |
| 181 | const crates = []; |
| 182 | for (let i = 0; i < args.length; i += 2) { |
| 183 | if (args[i] !== "-p" || !args[i + 1]) throw new Error(`expected -p <crate>..., got ${args.join(" ")}`); |
| 184 | crates.push(args[i + 1]); |
| 185 | } |
| 186 | return crates; |
| 187 | } |
| 188 | |
| 189 | function git(...args) { |
| 190 | const out = spawnSync("git", args, { cwd: ROOT, encoding: "utf8", maxBuffer: 1 << 28 }); |
| 191 | if (out.status !== 0) throw new Error(`git ${args.join(" ")}: ${(out.stderr || out.error?.message || "").trim()}`); |
| 192 | return out.stdout; |
| 193 | } |
| 194 | |
| 195 | /** The files a pull request's test merge commit changes, or why they are not known. */ |
| 196 | function changedFiles() { |
| 197 | if (process.env.GITHUB_EVENT_NAME !== "pull_request") return { why: `a ${process.env.GITHUB_EVENT_NAME ?? "local"} run tests every crate` }; |
| 198 | const parents = git("rev-list", "--parents", "-n", "1", "HEAD").trim().split(" ").slice(1); |
| 199 | if (parents.length !== 2) return { why: "HEAD is not the pull request's test merge commit" }; |
| 200 | // Both sides of a rename: the crate it left changed too. |
| 201 | const files = git("diff", "--name-only", "--no-renames", "-z", parents[0], "HEAD").split("\0").filter(Boolean); |
| 202 | return { files }; |
| 203 | } |
| 204 | |
| 205 | /** `--job <name>`: writes `run`, whether this run needs the job. */ |
| 206 | function mainJob(job) { |
| 207 | if (!JOBS[job]) throw new Error(`no job ${job}: one of ${Object.keys(JOBS).join(", ")}`); |
| 208 | let run = true; |
| 209 | try { |
| 210 | const { files, why } = changedFiles(); |
| 211 | if (files) { |
| 212 | run = needs(job, files); |
| 213 | console.log(`${files.length} files changed; ${run ? `the ${job} job's among them` : `none of them the ${job} job's, so it passes without running`}`); |
| 214 | } else { |
| 215 | console.log(why.replace("tests every crate", `runs the ${job} job`)); |
| 216 | } |
| 217 | } catch (error) { |
| 218 | console.log(`::warning title=Affected files::${error.message.split("\n")[0]}; the ${job} job runs`); |
| 219 | run = true; |
| 220 | } |
| 221 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `run=${run}\n`); |
| 222 | } |
| 223 | |
| 224 | function main(args) { |
| 225 | if (args[0] === "--job") return mainJob(args[1]); |
| 226 | const part = partCrates(args); |
| 227 | let test = part; |
| 228 | try { |
| 229 | const { files, why } = changedFiles(); |
| 230 | if (files) { |
| 231 | const meta = spawnSync("cargo", ["metadata", "--no-deps", "--format-version", "1"], { cwd: ROOT, encoding: "utf8", maxBuffer: 1 << 28 }); |
| 232 | if (meta.status !== 0) throw new Error(`cargo metadata: ${(meta.stderr || meta.error?.message || "").trim()}`); |
| 233 | const names = affected(workspace(JSON.parse(meta.stdout)), files); |
| 234 | if (names) test = part.filter((name) => names.includes(name)); |
| 235 | console.log( |
| 236 | `${files.length} files changed; ${names ? `crates to test: ${names.join(", ") || "none"}` : "the build's own files among them, so every crate"}`, |
| 237 | ); |
| 238 | } else { |
| 239 | console.log(why); |
| 240 | } |
| 241 | } catch (error) { |
| 242 | console.log(`::warning title=Affected crates::${error.message.split("\n")[0]}; every crate is tested`); |
| 243 | test = part; |
| 244 | } |
| 245 | const outputs = { crates: test.map((name) => `-p ${name}`).join(" "), whole: String(test.length === part.length) }; |
| 246 | console.log(`This part tests ${test.length} of its ${part.length} crates${test.length ? `: ${test.join(", ")}` : ""}`); |
| 247 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `crates=${outputs.crates}\nwhole=${outputs.whole}\n`); |
| 248 | } |
| 249 | |
| 250 | if (process.argv[1] && basename(process.argv[1]) === "ci-affected.mjs") main(process.argv.slice(2)); |