Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| The runner base keeps its old comment, so the pushed base image still matches its folder | 1 | // The deploy manifest (deploy/stack.jsonc) and what it implies: each |
| 2 | // unit's Wrangler config, the shared crates and packages it is built from, | |
| 3 | // and which units a set of changed files touches. Pure apart from reading | |
| 4 | // files and `cargo metadata`, so the tests can drive it. | |
| 5 | ||
| 6 | import { execFileSync } from "node:child_process"; | |
| 7 | import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; | |
| 8 | import { dirname, join, relative } from "node:path"; | |
| 9 | import { fileURLToPath } from "node:url"; | |
| 10 | ||
| 11 | export const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../.."); | |
| 12 | export const STACK_FILE = "deploy/stack.jsonc"; | |
| 13 | export const KINDS = ["rust-worker", "ts-worker", "react-router", "astro"]; | |
| 14 | export const SELF_HOST = ["run", "off", "separate", "none"]; | |
| 15 | ||
| 16 | /** Strips comments and trailing commas from JSONC. Strings are respected. */ | |
| 17 | export function parseJsonc(text) { | |
| 18 | let result = ""; | |
| 19 | let inString = false; | |
| 20 | for (let i = 0; i < text.length; i++) { | |
| 21 | const char = text[i]; | |
| 22 | if (inString) { | |
| 23 | result += char; | |
| 24 | if (char === "\\") result += text[++i]; | |
| 25 | else if (char === '"') inString = false; | |
| 26 | } else if (char === '"') { | |
| 27 | inString = true; | |
| 28 | result += char; | |
| 29 | } else if (char === "/" && text[i + 1] === "/") { | |
| 30 | while (i < text.length && text[i] !== "\n") i++; | |
| 31 | result += "\n"; | |
| 32 | } else if (char === "/" && text[i + 1] === "*") { | |
| 33 | i = text.indexOf("*/", i + 2) + 1; | |
| 34 | } else { | |
| 35 | result += char; | |
| 36 | } | |
| 37 | } | |
| 38 | return JSON.parse(result.replace(/,(\s*[}\]])/g, "$1")); | |
| 39 | } | |
| 40 | ||
| 41 | const readJsonc = (path) => parseJsonc(readFileSync(path, "utf8")); | |
| 42 | const posix = (path) => path.replaceAll("\\", "/"); | |
| 43 | ||
| 44 | /** | |
| 45 | * The manifest, each unit with its Wrangler config beside it. | |
| 46 | * Units keep the manifest's order. | |
| 47 | */ | |
| 48 | export function loadStack(root = ROOT) { | |
| 49 | const raw = readJsonc(join(root, STACK_FILE)); | |
| 50 | const units = Object.entries(raw.units).map(([id, unit]) => { | |
| 51 | const configPath = join(root, unit.path, "wrangler.jsonc"); | |
| 52 | const config = existsSync(configPath) ? readJsonc(configPath) : null; | |
| 53 | return { | |
| 54 | id, | |
| 55 | secrets: [], | |
| 56 | setup: [], | |
| 57 | inputs: [], | |
| 58 | ...unit, | |
| 59 | config, | |
| 60 | bindsTo: (config?.services ?? []).map((binding) => binding.service), | |
| 61 | }; | |
| 62 | }); | |
| 63 | return { stages: raw.stages, resources: raw.resources ?? {}, units }; | |
| 64 | } | |
| 65 | ||
| 66 | /** The deployable stages (every stage but migrations), in order. */ | |
| 67 | export const codeStages = (stack) => stack.stages.filter((stage) => stage !== "migrations"); | |
| 68 | ||
| 69 | /** | |
| 70 | * Workspace crates: name -> { dir, deps: [names of workspace crates it | |
| 71 | * depends on as a normal or build dependency] }. From `cargo metadata | |
| 72 | * --no-deps`, which reads only the workspace's manifests. | |
| 73 | */ | |
| 74 | export function cargoWorkspace(root = ROOT, metadata = null) { | |
| 75 | const meta = | |
| 76 | metadata ?? | |
| 77 | JSON.parse( | |
| 78 | execFileSync("cargo", ["metadata", "--no-deps", "--format-version", "1", "--offline"], { | |
| 79 | cwd: root, | |
| 80 | encoding: "utf8", | |
| 81 | maxBuffer: 64 * 1024 * 1024, | |
| 82 | }), | |
| 83 | ); | |
| 84 | const crates = new Map(); | |
| 85 | for (const pkg of meta.packages) { | |
| 86 | crates.set(pkg.name, { | |
| 87 | dir: posix(relative(meta.workspace_root ?? root, dirname(pkg.manifest_path))), | |
| 88 | deps: [], | |
| 89 | raw: pkg, | |
| 90 | }); | |
| 91 | } | |
| 92 | for (const crate of crates.values()) { | |
| 93 | crate.deps = crate.raw.dependencies | |
| 94 | // Dev-dependencies are not in what deploys. | |
| 95 | .filter((dep) => dep.kind !== "dev" && dep.path) | |
| 96 | .map((dep) => dep.name) | |
| 97 | .filter((name) => crates.has(name)); | |
| 98 | delete crate.raw; | |
| 99 | } | |
| 100 | return crates; | |
| 101 | } | |
| 102 | ||
| 103 | /** | |
| 104 | * npm workspace packages: name -> { dir, deps: [workspace package names] }. | |
| 105 | * dependencies and devDependencies both count: a build reads either. | |
| 106 | */ | |
| 107 | export function npmWorkspace(root = ROOT) { | |
| 108 | const rootPkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); | |
| 109 | const dirs = []; | |
| 110 | for (const pattern of rootPkg.workspaces ?? []) { | |
| 111 | if (pattern.endsWith("/*")) { | |
| 112 | const parent = pattern.slice(0, -2); | |
| 113 | if (!existsSync(join(root, parent))) continue; | |
| 114 | for (const name of readdirSync(join(root, parent)).sort()) { | |
| 115 | if (existsSync(join(root, parent, name, "package.json"))) dirs.push(`${parent}/${name}`); | |
| 116 | } | |
| 117 | } else if (existsSync(join(root, pattern, "package.json"))) { | |
| 118 | dirs.push(pattern); | |
| 119 | } | |
| 120 | } | |
| 121 | const packages = new Map(); | |
| 122 | const declared = new Map(); | |
| 123 | for (const dir of dirs) { | |
| 124 | const pkg = JSON.parse(readFileSync(join(root, dir, "package.json"), "utf8")); | |
| 125 | packages.set(pkg.name, { dir, deps: [] }); | |
| 126 | declared.set(pkg.name, Object.keys({ ...pkg.dependencies, ...pkg.devDependencies })); | |
| 127 | } | |
| 128 | for (const [name, pkg] of packages) pkg.deps = declared.get(name).filter((dep) => packages.has(dep)); | |
| 129 | return packages; | |
| 130 | } | |
| 131 | ||
| 132 | /** Every name reachable from `start` through `graph` (start excluded). */ | |
| 133 | function closure(graph, start) { | |
| 134 | const seen = new Set(); | |
| 135 | const stack = [...(graph.get(start)?.deps ?? [])]; | |
| 136 | while (stack.length) { | |
| 137 | const name = stack.pop(); | |
| 138 | if (seen.has(name)) continue; | |
| 139 | seen.add(name); | |
| 140 | stack.push(...(graph.get(name)?.deps ?? [])); | |
| 141 | } | |
| 142 | return [...seen]; | |
| 143 | } | |
| 144 | ||
| 145 | /** Files at the root every unit of a kind is built with. */ | |
| 146 | export function globalInputs(kind) { | |
| 147 | const inputs = ["package.json"]; | |
| 148 | if (kind === "rust-worker") inputs.push("Cargo.toml", "Cargo.lock", "scripts/build-rust-worker.mjs"); | |
| 149 | // A Rust worker's JavaScript is a small shim worker-build bundles itself, | |
| 150 | // so the npm lockfile only changes what npm-built units ship. | |
| 151 | else inputs.push("package-lock.json", "tsconfig.base.json"); | |
| 152 | return inputs; | |
| 153 | } | |
| 154 | ||
| 155 | /** | |
| 156 | * Adds to each unit what it is built from: | |
| 157 | * crate / pkg: its own crate or package name, when it has one; | |
| 158 | * dependsOn: folders of the shared crates and packages it uses; | |
| 159 | * inputs: the manifest's own inputs plus the root files its kind uses; | |
| 160 | * image: for a Containers image, the folders and files it is built from | |
| 161 | * (`dirs`, `files`), and its base's folder (`baseDirs`). | |
| 162 | */ | |
| 163 | export function resolveStack(stack, { cargo, npm }) { | |
| 164 | const crateByDir = new Map([...cargo].map(([name, crate]) => [crate.dir, name])); | |
| 165 | const pkgByDir = new Map([...npm].map(([name, pkg]) => [pkg.dir, name])); | |
| 166 | for (const unit of stack.units) { | |
| 167 | const crate = crateByDir.get(unit.path) ?? null; | |
| 168 | const pkg = pkgByDir.get(unit.path) ?? null; | |
| 169 | const dirs = new Set(); | |
| 170 | if (crate) for (const name of closure(cargo, crate)) dirs.add(cargo.get(name).dir); | |
| 171 | if (pkg) for (const name of closure(npm, pkg)) dirs.add(npm.get(name).dir); | |
| 172 | dirs.delete(unit.path); | |
| 173 | unit.crate = crate; | |
| 174 | unit.pkg = pkg; | |
| 175 | unit.dependsOn = [...dirs].sort(); | |
| Merge remote-tracking branch 'origin/main' into workspace-chat | 176 | // The Rust crates it is built from, whose tests, benches and examples |
| 177 | // are not. | |
| 178 | unit.crateDirs = crate ? [unit.path, ...closure(cargo, crate).map((name) => cargo.get(name).dir)].sort() : []; | |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 179 | unit.inputs = [...new Set([...(unit.inputs ?? []), ...globalInputs(unit.kind)])].sort(); |
| 180 | if (unit.image) { | |
| 181 | const name = unit.image.crate; | |
| 182 | const crates = name && cargo.has(name) ? [name, ...closure(cargo, name)] : []; | |
| 183 | // The image is the base (recorded in its lock) plus the binary: a | |
| 184 | // change to the base's folder reaches the image only through a new | |
| 185 | // lock, which `build-base` writes. | |
| 186 | const lock = unit.image.base?.lock; | |
| 187 | unit.image = { | |
| 188 | ...unit.image, | |
| 189 | dirs: crates.map((c) => cargo.get(c).dir).sort(), | |
| 190 | files: [unit.image.dockerfile, ...(lock ? [lock] : []), "Cargo.toml", "Cargo.lock", "scripts/build-runner.mjs"], | |
| 191 | baseDirs: unit.image.base ? [unit.image.base.context] : [], | |
| 192 | }; | |
| 193 | for (const dir of unit.image.dirs) if (dir !== unit.path) unit.dependsOn.push(dir); | |
| 194 | unit.dependsOn = [...new Set(unit.dependsOn)].sort(); | |
| Merge remote-tracking branch 'origin/main' into workspace-chat | 195 | unit.crateDirs = [...new Set([...unit.crateDirs, ...unit.image.dirs])].sort(); |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 196 | unit.inputs = [...new Set([...unit.inputs, "Cargo.toml", "Cargo.lock", "scripts/build-runner.mjs"])].sort(); |
| 197 | } | |
| 198 | } | |
| 199 | return stack; | |
| 200 | } | |
| 201 | ||
| 202 | /** The manifest, resolved against this checkout. */ | |
| 203 | export function resolvedStack(root = ROOT) { | |
| 204 | return resolveStack(loadStack(root), { cargo: cargoWorkspace(root), npm: npmWorkspace(root) }); | |
| 205 | } | |
| 206 | ||
| 207 | const under = (file, dir) => file === dir || file.startsWith(`${dir}/`); | |
| 208 | ||
| Merge remote-tracking branch 'origin/main' into workspace-chat | 209 | /** A Rust crate's folders that its library and binaries are never built from. */ |
| 210 | export const CRATE_TEST_DIRS = ["tests", "benches", "examples"]; | |
| 211 | ||
| 212 | /** | |
| 213 | * Whether `file` is in the tests, benches or examples of one of | |
| 214 | * `crateDirs`: Cargo builds those only for `cargo test`, `cargo bench` and | |
| 215 | * `--example`, never into what deploys. | |
| 216 | */ | |
| 217 | export function inCrateTests(file, crateDirs = []) { | |
| 218 | return crateDirs.some((dir) => CRATE_TEST_DIRS.some((sub) => under(file, `${dir}/${sub}`))); | |
| 219 | } | |
| 220 | ||
| 221 | const dirOf = (path) => path.slice(0, Math.max(0, path.lastIndexOf("/"))); | |
| 222 | const baseOf = (path) => path.slice(path.lastIndexOf("/") + 1); | |
| 223 | ||
| 224 | /** | |
| 225 | * The `mod name;` declarations in Rust source `text` that `match` (by name, | |
| 226 | * or by a `#[path]`), each with whether it is under `#[cfg(test)]`. | |
| 227 | * Only outline modules (`mod x;`); the attributes are those directly above. | |
| 228 | */ | |
| 229 | function declarations(text, match) { | |
| 230 | const found = []; | |
| 231 | const pattern = /((?:#\[[^\]]*\]\s*)*)(?:pub(?:\([^)]*\))?\s+)?mod\s+(r#)?(\w+)\s*;/g; | |
| 232 | for (const [, attrs, , name] of text.matchAll(pattern)) { | |
| 233 | const path = /#\[\s*path\s*=\s*"([^"]+)"\s*\]/.exec(attrs)?.[1] ?? null; | |
| 234 | if (!match(name, path)) continue; | |
| 235 | found.push(/#\[\s*cfg\s*\(\s*test\s*\)\s*\]/.test(attrs)); | |
| 236 | } | |
| 237 | return found; | |
| 238 | } | |
| 239 | ||
| 240 | /** | |
| 241 | * Whether Rust source `file`, in the `src/` of one of `crateDirs`, is | |
| 242 | * compiled only for tests: every module declaration of it found is under | |
| 243 | * `#[cfg(test)]`, or is in a file that is itself only for tests. A file | |
| 244 | * nothing is found to declare (a new crate root, a deleted file, a module | |
| 245 | * declared some way this does not read) counts as built, so a change to it | |
| 246 | * deploys. | |
| 247 | * | |
| 248 | * read(path): a file's text at the commit being deployed, "" if absent | |
| 249 | * list(dir): the files directly in a folder at that commit | |
| 250 | */ | |
| 251 | export function testOnlySource(file, crateDirs, { read, list = () => [] }, depth = 0) { | |
| 252 | if (!file.endsWith(".rs") || depth > 16) return false; | |
| 253 | const crateDir = crateDirs.find((dir) => file.startsWith(`${dir}/src/`)); | |
| 254 | if (!crateDir) return false; | |
| 255 | const src = `${crateDir}/src`; | |
| 256 | const dir = dirOf(file); | |
| 257 | const base = baseOf(file).slice(0, -".rs".length); | |
| 258 | // Crate roots and binaries are not declared by anything. | |
| 259 | if (dir === src && ["lib", "main"].includes(base)) return false; | |
| 260 | if (dir === `${src}/bin` || (base === "main" && dirOf(dir) === `${src}/bin`)) return false; | |
| 261 | // Where `mod name;` for this file would be: `a/name.rs` and `a/name/mod.rs` | |
| 262 | // are declared by `a.rs` or `a/mod.rs` (by `lib.rs` or `main.rs` in src). | |
| 263 | const name = base === "mod" ? baseOf(dir) : base; | |
| 264 | const parentDir = base === "mod" ? dirOf(dir) : dir; | |
| 265 | const parents = | |
| 266 | parentDir === src ? [`${src}/lib.rs`, `${src}/main.rs`] : [`${parentDir}/mod.rs`, `${dirOf(parentDir)}/${baseOf(parentDir)}.rs`]; | |
| 267 | // One declaration that is built is enough to stop: most changed files | |
| 268 | // are ordinary modules, settled by reading their parent. | |
| 269 | let declared = false; | |
| 270 | const testOnly = (declarer, isTest) => { | |
| 271 | declared = true; | |
| 272 | return isTest || testOnlySource(declarer, crateDirs, { read, list }, depth + 1); | |
| 273 | }; | |
| 274 | for (const parent of parents) { | |
| 275 | for (const isTest of declarations(read(parent), (found, path) => found === name && path === null)) { | |
| 276 | if (!testOnly(parent, isTest)) return false; | |
| 277 | } | |
| 278 | } | |
| 279 | // `#[path = "x.rs"] mod y;` in a file beside it names it directly. | |
| 280 | const fileName = baseOf(file); | |
| 281 | for (const sibling of list(dir).filter((path) => path.endsWith(".rs") && path !== file)) { | |
| 282 | for (const isTest of declarations(read(sibling), (_, path) => path === fileName || path === `./${fileName}`)) { | |
| 283 | if (!testOnly(sibling, isTest)) return false; | |
| 284 | } | |
| 285 | } | |
| 286 | return declared; | |
| 287 | } | |
| 288 | ||
| The runner base keeps its old comment, so the pushed base image still matches its folder | 289 | /** |
| 290 | * Why a set of changed files (repository-relative, `/`-separated) touches | |
| 291 | * a unit: the first file that does, and through what. Null if none does. | |
| Merge remote-tracking branch 'origin/main' into workspace-chat | 292 | * Its crates' tests, benches and examples do not. |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 293 | */ |
| 294 | export function touches(unit, files) { | |
| Merge remote-tracking branch 'origin/main' into workspace-chat | 295 | files = files.filter((file) => !inCrateTests(file, unit.crateDirs)); |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 296 | for (const file of files) { |
| 297 | if (under(file, unit.path)) return { file, via: "its own folder" }; | |
| 298 | } | |
| 299 | for (const file of files) { | |
| 300 | const dir = unit.dependsOn.find((d) => under(file, d)); | |
| 301 | if (dir) return { file, via: dir }; | |
| 302 | if (unit.inputs.includes(file)) return { file, via: file }; | |
| 303 | } | |
| 304 | return null; | |
| 305 | } | |
| 306 | ||
| 307 | /** Whether changed files touch a unit's Containers image. */ | |
| 308 | export function touchesImage(unit, files) { | |
| 309 | if (!unit.image) return false; | |
| Merge remote-tracking branch 'origin/main' into workspace-chat | 310 | return files.some( |
| 311 | (file) => unit.image.files.includes(file) || (unit.image.dirs.some((dir) => under(file, dir)) && !inCrateTests(file, unit.image.dirs)), | |
| 312 | ); | |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 313 | } |
| 314 | ||
| 315 | /** Whether changed files touch the folder a unit's base image is built from. */ | |
| 316 | export function touchesBase(unit, files) { | |
| 317 | return Boolean(unit.image?.baseDirs?.length) && files.some((file) => unit.image.baseDirs.some((dir) => under(file, dir))); | |
| 318 | } | |
| 319 | ||
| 320 | /** Units named on the command line: short names, folders or Worker names. */ | |
| 321 | export function pick(stack, names) { | |
| 322 | const wanted = names.flatMap((name) => name.split(",")).map((name) => name.trim().replace(/\/$/, "")).filter(Boolean); | |
| 323 | const found = []; | |
| 324 | for (const name of wanted) { | |
| 325 | const unit = stack.units.find((u) => u.id === name || u.path === name || u.worker === name); | |
| 326 | if (!unit) throw new Error(`No unit called ${name}. Units: ${stack.units.map((u) => u.id).join(", ")}`); | |
| 327 | if (!found.includes(unit)) found.push(unit); | |
| 328 | } | |
| 329 | return found; | |
| 330 | } | |
| 331 | ||
| 332 | /** Units grouped by stage, in stage order, keeping the manifest's order inside each. */ | |
| 333 | export function byStage(stack, units) { | |
| 334 | return codeStages(stack) | |
| 335 | .map((stage) => ({ stage, units: units.filter((u) => u.stage === stage) })) | |
| 336 | .filter((group) => group.units.length); | |
| 337 | } | |
| 338 | ||
| The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook. | 339 | /** |
| 340 | * A stage's units in the order they can go out. Cloudflare refuses a | |
| 341 | * Worker bound to a Worker that does not exist, so a unit never deployed | |
| 342 | * before (`isNew`, a new or renamed Worker) goes out before the units of | |
| 343 | * its stage that bind to it. Units in one wave go in parallel. Without a | |
| 344 | * new unit, the stage is one wave. | |
| 345 | */ | |
| 346 | export function waves(units, isNew) { | |
| 347 | const out = []; | |
| 348 | let left = [...units]; | |
| 349 | while (left.length) { | |
| 350 | // Workers that do not exist yet and are still to deploy. | |
| 351 | const missing = new Set(left.filter(isNew).map((u) => u.worker)); | |
| 352 | const ready = left.filter((u) => !u.bindsTo.some((target) => target !== u.worker && missing.has(target))); | |
| 353 | // New units that bind to each other: no order helps, so all at once. | |
| 354 | const wave = ready.length ? ready : left; | |
| 355 | out.push(wave); | |
| 356 | left = left.filter((u) => !wave.includes(u)); | |
| 357 | } | |
| 358 | return out; | |
| 359 | } | |
| 360 | ||
| The runner base keeps its old comment, so the pushed base image still matches its folder | 361 | /** The most Rust workers one CI job builds; more are split across jobs. */ |
| 362 | export const RUST_PER_JOB = 4; | |
| 363 | ||
| 364 | /** | |
| 365 | * How a stage's units are split into CI jobs, so units that share a build | |
| 366 | * share a sandbox: Rust workers (one Cargo target, at most RUST_PER_JOB to | |
| 367 | * a job, each on a 4-vCPU machine), the TypeScript Workers (cheap), | |
| 368 | * each site that runs a framework build, and each unit whose Containers | |
| 369 | * image must be rebuilt (`images`: ids), which needs Docker. | |
| 370 | */ | |
| 371 | export function buildGroups(units, images = []) { | |
| 372 | const groups = new Map(); | |
| 373 | const add = (key, id) => { | |
| 374 | if (!groups.has(key)) groups.set(key, []); | |
| 375 | groups.get(key).push(id); | |
| 376 | }; | |
| 377 | const rust = units.filter((u) => u.kind === "rust-worker"); | |
| 378 | const shards = Math.ceil(rust.length / RUST_PER_JOB); | |
| 379 | rust.forEach((unit, i) => add(shards > 1 ? `rust-${(i % shards) + 1}` : "rust", unit.id)); | |
| 380 | for (const unit of units.filter((u) => u.kind !== "rust-worker")) { | |
| 381 | add(images.includes(unit.id) ? `${unit.id}-image` : unit.kind === "ts-worker" ? "ts" : unit.id, unit.id); | |
| 382 | } | |
| Merge branch 'main' into actions-toolkit-oidc-artifacts | 383 | // `image`: the job builds a Containers image (Docker, on a larger machine). |
| 384 | return [...groups].map(([group, ids]) => ({ group, units: ids.join(","), rust: group.startsWith("rust"), image: group.endsWith("-image") })); | |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 385 | } |
| 386 | ||
| 387 | /** | |
| 388 | * What is wrong with the manifest, as sentences. Empty when it is right. | |
| 389 | * `wranglerConfigs`: every wrangler.jsonc in the repository (relative paths). | |
| 390 | */ | |
| 391 | export function problems(stack, wranglerConfigs = findWranglerConfigs()) { | |
| 392 | const out = []; | |
| 393 | const stages = codeStages(stack); | |
| 394 | if (stack.stages[0] !== "migrations") out.push('The first stage is "migrations".'); | |
| 395 | const byWorker = new Map(); | |
| 396 | for (const unit of stack.units) { | |
| 397 | const where = `Unit ${unit.id}`; | |
| 398 | if (!KINDS.includes(unit.kind)) out.push(`${where}: kind is one of ${KINDS.join(", ")}.`); | |
| 399 | if (!stages.includes(unit.stage)) out.push(`${where}: stage is one of ${stages.join(", ")}.`); | |
| 400 | if (!SELF_HOST.includes(unit.self_host)) out.push(`${where}: self_host is one of ${SELF_HOST.join(", ")}.`); | |
| 401 | if (!unit.config) { | |
| 402 | out.push(`${where}: ${unit.path}/wrangler.jsonc does not exist.`); | |
| 403 | continue; | |
| 404 | } | |
| 405 | if (unit.config.name !== unit.worker) out.push(`${where}: worker is ${unit.config.name} in its wrangler.jsonc, not ${unit.worker}.`); | |
| 406 | byWorker.set(unit.worker, unit); | |
| 407 | const dbs = (unit.config.d1_databases ?? []).filter((db) => db.migrations_dir); | |
| 408 | const d1 = dbs[0] ? { database: dbs[0].database_name, migrations: dbs[0].migrations_dir } : null; | |
| 409 | if (dbs.length > 1) out.push(`${where}: more than one D1 database with migrations; the deploy tool applies one per unit.`); | |
| 410 | if (JSON.stringify(d1) !== JSON.stringify(unit.d1 ?? null)) { | |
| 411 | out.push(`${where}: d1 is ${JSON.stringify(d1)} in its wrangler.jsonc, not ${JSON.stringify(unit.d1 ?? null)}.`); | |
| 412 | } | |
| 413 | const building = unit.config.build?.command ?? ""; | |
| 414 | if (unit.kind === "rust-worker" && !building.includes("scripts/build-rust-worker.mjs")) { | |
| 415 | out.push(`${where}: a Rust worker builds with node ../../scripts/build-rust-worker.mjs, not "${building}".`); | |
| 416 | } | |
| 417 | if (unit.kind !== "rust-worker" && building) out.push(`${where}: only Rust workers have a build command; ${unit.kind} builds in the deploy tool.`); | |
| 418 | for (const id of (unit.config.kv_namespaces ?? []).map((kv) => kv.id)) { | |
| 419 | if (!stack.resources.kv?.[id]) out.push(`${where}: KV namespace ${id} has no name under resources.kv.`); | |
| 420 | } | |
| 421 | const hasImage = (unit.config.containers ?? []).some((c) => !String(c.image).includes("registry")); | |
| 422 | if (hasImage !== Boolean(unit.image)) out.push(`${where}: image is set exactly when its wrangler.jsonc builds a Containers image.`); | |
| 423 | } | |
| 424 | const listed = new Set(stack.units.map((u) => posix(join(u.path, "wrangler.jsonc")))); | |
| 425 | for (const config of wranglerConfigs) { | |
| 426 | if (!listed.has(config)) out.push(`${config} is not in ${STACK_FILE}: add its unit.`); | |
| 427 | } | |
| 428 | // Stage order follows bindings: a unit binds only to units that ship in | |
| 429 | // its stage or before it. | |
| 430 | for (const unit of stack.units) { | |
| 431 | for (const target of unit.bindsTo) { | |
| 432 | const other = byWorker.get(target); | |
| 433 | if (!other) { | |
| 434 | out.push(`Unit ${unit.id} binds to ${target}, which no unit deploys.`); | |
| 435 | } else if (stages.indexOf(other.stage) > stages.indexOf(unit.stage)) { | |
| 436 | out.push(`Unit ${unit.id} (${unit.stage}) binds to ${other.id} (${other.stage}), which ships after it.`); | |
| 437 | } | |
| 438 | } | |
| 439 | } | |
| 440 | return out; | |
| 441 | } | |
| 442 | ||
| 443 | const SKIP_DIRS = new Set(["node_modules", "target", ".git", "build", "dist", ".wrangler", ".generated", ".astro"]); | |
| 444 | ||
| 445 | /** Every wrangler.jsonc or wrangler.toml checked into the repository. */ | |
| 446 | export function findWranglerConfigs(root = ROOT) { | |
| 447 | const found = []; | |
| 448 | const walk = (dir) => { | |
| 449 | for (const name of readdirSync(join(root, dir))) { | |
| 450 | if (SKIP_DIRS.has(name) || name.startsWith(".")) continue; | |
| 451 | const path = dir ? `${dir}/${name}` : name; | |
| 452 | if (statSync(join(root, path)).isDirectory()) walk(path); | |
| 453 | else if (/^wrangler\.(jsonc?|toml)$/.test(name)) found.push(path); | |
| 454 | } | |
| 455 | }; | |
| 456 | walk(""); | |
| 457 | return found.sort(); | |
| 458 | } | |
| 459 | ||
| 460 | /** | |
| 461 | * Relative imports in a unit's non-test sources that leave its folder: | |
| 462 | * each { file, target }. The manifest must cover each one, through a | |
| 463 | * workspace dependency or `inputs`. | |
| 464 | */ | |
| 465 | export function outsideImports(root, unit) { | |
| 466 | const found = []; | |
| 467 | const pattern = /(?:from\s+|import\s*\(\s*|import\s+)["'](\.{1,2}\/[^"']+)["']/g; | |
| 468 | const walk = (dir) => { | |
| 469 | for (const name of readdirSync(join(root, dir))) { | |
| 470 | if (SKIP_DIRS.has(name) || name.startsWith(".")) continue; | |
| 471 | const path = `${dir}/${name}`; | |
| 472 | if (statSync(join(root, path)).isDirectory()) { | |
| 473 | walk(path); | |
| 474 | continue; | |
| 475 | } | |
| 476 | if (!/\.(m?[jt]sx?|astro)$/.test(name) || /\.test\.[jt]sx?$/.test(name) || name.endsWith(".d.ts")) continue; | |
| 477 | const text = readFileSync(join(root, path), "utf8"); | |
| 478 | for (const match of text.matchAll(pattern)) { | |
| 479 | const target = posix(join(dirname(path), match[1])); | |
| 480 | if (!under(target, unit.path)) found.push({ file: path, target }); | |
| 481 | } | |
| 482 | } | |
| 483 | }; | |
| 484 | walk(unit.path); | |
| 485 | return found; | |
| 486 | } | |
| 487 | ||
| 488 | /** | |
| 489 | * npm ci of only what the units need: Wrangler alone for Rust workers, | |
| 490 | * and each npm-built unit's workspace with the packages it uses. A CI job | |
| 491 | * that deploys three Rust workers does not install the site's toolchain. | |
| The deploy's stages ride out a dropped npm connection as CI does. Each core, edge and front job keeps npm's downloads between runs (actions/cache on ~/.npm, keyed on the lockfile), and the deploy tool's install reads them first (npm ci --prefer-offline), so a registry connection reset mid-install no longer fails a stage, as it failed the web stage of Deploy 201. | 492 | * --prefer-offline reads npm's cache first, which the deploy's jobs keep |
| 493 | * between runs, so a registry connection dropped mid-install rarely | |
| 494 | * fails a stage. | |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 495 | */ |
| 496 | export function npmCiArgs(units, npm) { | |
| 497 | const workspaces = new Set(); | |
| 498 | for (const unit of units) { | |
| 499 | if (!unit.pkg) continue; | |
| 500 | const stack = [unit.pkg]; | |
| 501 | while (stack.length) { | |
| 502 | const name = stack.pop(); | |
| 503 | if (workspaces.has(name)) continue; | |
| 504 | workspaces.add(name); | |
| 505 | stack.push(...(npm.get(name)?.deps ?? [])); | |
| 506 | } | |
| 507 | } | |
| The deploy's stages ride out a dropped npm connection as CI does. Each core, edge and front job keeps npm's downloads between runs (actions/cache on ~/.npm, keyed on the lockfile), and the deploy tool's install reads them first (npm ci --prefer-offline), so a registry connection reset mid-install no longer fails a stage, as it failed the web stage of Deploy 201. | 508 | const base = ["ci", "--prefer-offline", "--no-audit", "--no-fund"]; |
| The runner base keeps its old comment, so the pushed base image still matches its folder | 509 | if (!workspaces.size) return [...base, "--workspaces=false"]; |
| 510 | return [...base, "--include-workspace-root", ...[...workspaces].sort().flatMap((name) => ["-w", name])]; | |
| 511 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.