| 1 | // The deploy manifest (deploy/stack.jsonc) and what it implies: each |
| 2 | // unit's Wrangler config, the shared crates and packages it is built from, |
| 3 | // and which units a set of changed files touches. Pure apart from reading |
| 4 | // files and `cargo metadata`, so the tests can drive it. |
| 5 | |
| 6 | import { execFileSync } from "node:child_process"; |
| 7 | import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; |
| 8 | import { dirname, join, relative } from "node:path"; |
| 9 | import { fileURLToPath } from "node:url"; |
| 10 | |
| 11 | export const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../.."); |
| 12 | export const STACK_FILE = "deploy/stack.jsonc"; |
| 13 | export const KINDS = ["rust-worker", "ts-worker", "react-router", "astro"]; |
| 14 | export const SELF_HOST = ["run", "off", "separate", "none"]; |
| 15 | |
| 16 | /** Strips comments and trailing commas from JSONC. Strings are respected. */ |
| 17 | export function parseJsonc(text) { |
| 18 | let result = ""; |
| 19 | let inString = false; |
| 20 | for (let i = 0; i < text.length; i++) { |
| 21 | const char = text[i]; |
| 22 | if (inString) { |
| 23 | result += char; |
| 24 | if (char === "\\") result += text[++i]; |
| 25 | else if (char === '"') inString = false; |
| 26 | } else if (char === '"') { |
| 27 | inString = true; |
| 28 | result += char; |
| 29 | } else if (char === "/" && text[i + 1] === "/") { |
| 30 | while (i < text.length && text[i] !== "\n") i++; |
| 31 | result += "\n"; |
| 32 | } else if (char === "/" && text[i + 1] === "*") { |
| 33 | i = text.indexOf("*/", i + 2) + 1; |
| 34 | } else { |
| 35 | result += char; |
| 36 | } |
| 37 | } |
| 38 | return JSON.parse(result.replace(/,(\s*[}\]])/g, "$1")); |
| 39 | } |
| 40 | |
| 41 | const readJsonc = (path) => parseJsonc(readFileSync(path, "utf8")); |
| 42 | const posix = (path) => path.replaceAll("\\", "/"); |
| 43 | |
| 44 | /** |
| 45 | * The manifest, each unit with its Wrangler config beside it. |
| 46 | * Units keep the manifest's order. |
| 47 | */ |
| 48 | export function loadStack(root = ROOT) { |
| 49 | const raw = readJsonc(join(root, STACK_FILE)); |
| 50 | const units = Object.entries(raw.units).map(([id, unit]) => { |
| 51 | const configPath = join(root, unit.path, "wrangler.jsonc"); |
| 52 | const config = existsSync(configPath) ? readJsonc(configPath) : null; |
| 53 | return { |
| 54 | id, |
| 55 | secrets: [], |
| 56 | setup: [], |
| 57 | inputs: [], |
| 58 | ...unit, |
| 59 | config, |
| 60 | bindsTo: (config?.services ?? []).map((binding) => binding.service), |
| 61 | }; |
| 62 | }); |
| 63 | return { stages: raw.stages, resources: raw.resources ?? {}, units }; |
| 64 | } |
| 65 | |
| 66 | /** The deployable stages (every stage but migrations), in order. */ |
| 67 | export const codeStages = (stack) => stack.stages.filter((stage) => stage !== "migrations"); |
| 68 | |
| 69 | /** |
| 70 | * Workspace crates: name -> { dir, deps: [names of workspace crates it |
| 71 | * depends on as a normal or build dependency] }. From `cargo metadata |
| 72 | * --no-deps`, which reads only the workspace's manifests. |
| 73 | */ |
| 74 | export function cargoWorkspace(root = ROOT, metadata = null) { |
| 75 | const meta = |
| 76 | metadata ?? |
| 77 | JSON.parse( |
| 78 | execFileSync("cargo", ["metadata", "--no-deps", "--format-version", "1", "--offline"], { |
| 79 | cwd: root, |
| 80 | encoding: "utf8", |
| 81 | maxBuffer: 64 * 1024 * 1024, |
| 82 | }), |
| 83 | ); |
| 84 | const crates = new Map(); |
| 85 | for (const pkg of meta.packages) { |
| 86 | crates.set(pkg.name, { |
| 87 | dir: posix(relative(meta.workspace_root ?? root, dirname(pkg.manifest_path))), |
| 88 | deps: [], |
| 89 | raw: pkg, |
| 90 | }); |
| 91 | } |
| 92 | for (const crate of crates.values()) { |
| 93 | crate.deps = crate.raw.dependencies |
| 94 | // Dev-dependencies are not in what deploys. |
| 95 | .filter((dep) => dep.kind !== "dev" && dep.path) |
| 96 | .map((dep) => dep.name) |
| 97 | .filter((name) => crates.has(name)); |
| 98 | delete crate.raw; |
| 99 | } |
| 100 | return crates; |
| 101 | } |
| 102 | |
| 103 | /** |
| 104 | * npm workspace packages: name -> { dir, deps: [workspace package names] }. |
| 105 | * dependencies and devDependencies both count: a build reads either. |
| 106 | */ |
| 107 | export function npmWorkspace(root = ROOT) { |
| 108 | const rootPkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); |
| 109 | const dirs = []; |
| 110 | for (const pattern of rootPkg.workspaces ?? []) { |
| 111 | if (pattern.endsWith("/*")) { |
| 112 | const parent = pattern.slice(0, -2); |
| 113 | if (!existsSync(join(root, parent))) continue; |
| 114 | for (const name of readdirSync(join(root, parent)).sort()) { |
| 115 | if (existsSync(join(root, parent, name, "package.json"))) dirs.push(`${parent}/${name}`); |
| 116 | } |
| 117 | } else if (existsSync(join(root, pattern, "package.json"))) { |
| 118 | dirs.push(pattern); |
| 119 | } |
| 120 | } |
| 121 | const packages = new Map(); |
| 122 | const declared = new Map(); |
| 123 | for (const dir of dirs) { |
| 124 | const pkg = JSON.parse(readFileSync(join(root, dir, "package.json"), "utf8")); |
| 125 | packages.set(pkg.name, { dir, deps: [] }); |
| 126 | declared.set(pkg.name, Object.keys({ ...pkg.dependencies, ...pkg.devDependencies })); |
| 127 | } |
| 128 | for (const [name, pkg] of packages) pkg.deps = declared.get(name).filter((dep) => packages.has(dep)); |
| 129 | return packages; |
| 130 | } |
| 131 | |
| 132 | /** Every name reachable from `start` through `graph` (start excluded). */ |
| 133 | function closure(graph, start) { |
| 134 | const seen = new Set(); |
| 135 | const stack = [...(graph.get(start)?.deps ?? [])]; |
| 136 | while (stack.length) { |
| 137 | const name = stack.pop(); |
| 138 | if (seen.has(name)) continue; |
| 139 | seen.add(name); |
| 140 | stack.push(...(graph.get(name)?.deps ?? [])); |
| 141 | } |
| 142 | return [...seen]; |
| 143 | } |
| 144 | |
| 145 | /** Files at the root every unit of a kind is built with. */ |
| 146 | export function globalInputs(kind) { |
| 147 | const inputs = ["package.json"]; |
| 148 | if (kind === "rust-worker") inputs.push("Cargo.toml", "Cargo.lock", "scripts/build-rust-worker.mjs"); |
| 149 | // A Rust worker's JavaScript is a small shim worker-build bundles itself, |
| 150 | // so the npm lockfile only changes what npm-built units ship. |
| 151 | else inputs.push("package-lock.json", "tsconfig.base.json"); |
| 152 | return inputs; |
| 153 | } |
| 154 | |
| 155 | /** |
| 156 | * Adds to each unit what it is built from: |
| 157 | * crate / pkg: its own crate or package name, when it has one; |
| 158 | * dependsOn: folders of the shared crates and packages it uses; |
| 159 | * inputs: the manifest's own inputs plus the root files its kind uses; |
| 160 | * image: for a Containers image, the folders and files it is built from. |
| 161 | */ |
| 162 | export function resolveStack(stack, { cargo, npm }) { |
| 163 | const crateByDir = new Map([...cargo].map(([name, crate]) => [crate.dir, name])); |
| 164 | const pkgByDir = new Map([...npm].map(([name, pkg]) => [pkg.dir, name])); |
| 165 | for (const unit of stack.units) { |
| 166 | const crate = crateByDir.get(unit.path) ?? null; |
| 167 | const pkg = pkgByDir.get(unit.path) ?? null; |
| 168 | const dirs = new Set(); |
| 169 | if (crate) for (const name of closure(cargo, crate)) dirs.add(cargo.get(name).dir); |
| 170 | if (pkg) for (const name of closure(npm, pkg)) dirs.add(npm.get(name).dir); |
| 171 | dirs.delete(unit.path); |
| 172 | unit.crate = crate; |
| 173 | unit.pkg = pkg; |
| 174 | unit.dependsOn = [...dirs].sort(); |
| 175 | unit.inputs = [...new Set([...(unit.inputs ?? []), ...globalInputs(unit.kind)])].sort(); |
| 176 | if (unit.image) { |
| 177 | const name = unit.image.crate; |
| 178 | const crates = name && cargo.has(name) ? [name, ...closure(cargo, name)] : []; |
| 179 | unit.image = { |
| 180 | ...unit.image, |
| 181 | dirs: crates.map((c) => cargo.get(c).dir).sort(), |
| 182 | files: [unit.image.dockerfile, "Cargo.toml", "Cargo.lock"], |
| 183 | }; |
| 184 | for (const dir of unit.image.dirs) if (dir !== unit.path) unit.dependsOn.push(dir); |
| 185 | unit.dependsOn = [...new Set(unit.dependsOn)].sort(); |
| 186 | unit.inputs = [...new Set([...unit.inputs, "Cargo.toml", "Cargo.lock"])].sort(); |
| 187 | } |
| 188 | } |
| 189 | return stack; |
| 190 | } |
| 191 | |
| 192 | /** The manifest, resolved against this checkout. */ |
| 193 | export function resolvedStack(root = ROOT) { |
| 194 | return resolveStack(loadStack(root), { cargo: cargoWorkspace(root), npm: npmWorkspace(root) }); |
| 195 | } |
| 196 | |
| 197 | const under = (file, dir) => file === dir || file.startsWith(`${dir}/`); |
| 198 | |
| 199 | /** |
| 200 | * Why a set of changed files (repository-relative, `/`-separated) touches |
| 201 | * a unit: the first file that does, and through what. Null if none does. |
| 202 | */ |
| 203 | export function touches(unit, files) { |
| 204 | for (const file of files) { |
| 205 | if (under(file, unit.path)) return { file, via: "its own folder" }; |
| 206 | } |
| 207 | for (const file of files) { |
| 208 | const dir = unit.dependsOn.find((d) => under(file, d)); |
| 209 | if (dir) return { file, via: dir }; |
| 210 | if (unit.inputs.includes(file)) return { file, via: file }; |
| 211 | } |
| 212 | return null; |
| 213 | } |
| 214 | |
| 215 | /** Whether changed files touch a unit's Containers image. */ |
| 216 | export function touchesImage(unit, files) { |
| 217 | if (!unit.image) return false; |
| 218 | return files.some((file) => unit.image.files.includes(file) || unit.image.dirs.some((dir) => under(file, dir))); |
| 219 | } |
| 220 | |
| 221 | /** Units named on the command line: short names, folders or Worker names. */ |
| 222 | export function pick(stack, names) { |
| 223 | const wanted = names.flatMap((name) => name.split(",")).map((name) => name.trim().replace(/\/$/, "")).filter(Boolean); |
| 224 | const found = []; |
| 225 | for (const name of wanted) { |
| 226 | const unit = stack.units.find((u) => u.id === name || u.path === name || u.worker === name); |
| 227 | if (!unit) throw new Error(`No unit called ${name}. Units: ${stack.units.map((u) => u.id).join(", ")}`); |
| 228 | if (!found.includes(unit)) found.push(unit); |
| 229 | } |
| 230 | return found; |
| 231 | } |
| 232 | |
| 233 | /** Units grouped by stage, in stage order, keeping the manifest's order inside each. */ |
| 234 | export function byStage(stack, units) { |
| 235 | return codeStages(stack) |
| 236 | .map((stage) => ({ stage, units: units.filter((u) => u.stage === stage) })) |
| 237 | .filter((group) => group.units.length); |
| 238 | } |
| 239 | |
| 240 | /** The most Rust workers one CI job builds; more are split across jobs. */ |
| 241 | export const RUST_PER_JOB = 4; |
| 242 | |
| 243 | /** |
| 244 | * How a stage's units are split into CI jobs, so units that share a build |
| 245 | * share a sandbox: Rust workers (one Cargo target, at most RUST_PER_JOB to |
| 246 | * a job, since a sandbox has half a CPU), the TypeScript Workers (cheap), |
| 247 | * each site that runs a framework build, and each unit whose Containers |
| 248 | * image must be rebuilt (`images`: ids), which needs Docker. |
| 249 | */ |
| 250 | export function buildGroups(units, images = []) { |
| 251 | const groups = new Map(); |
| 252 | const add = (key, id) => { |
| 253 | if (!groups.has(key)) groups.set(key, []); |
| 254 | groups.get(key).push(id); |
| 255 | }; |
| 256 | const rust = units.filter((u) => u.kind === "rust-worker"); |
| 257 | const shards = Math.ceil(rust.length / RUST_PER_JOB); |
| 258 | rust.forEach((unit, i) => add(shards > 1 ? `rust-${(i % shards) + 1}` : "rust", unit.id)); |
| 259 | for (const unit of units.filter((u) => u.kind !== "rust-worker")) { |
| 260 | add(images.includes(unit.id) ? `${unit.id}-image` : unit.kind === "ts-worker" ? "ts" : unit.id, unit.id); |
| 261 | } |
| 262 | return [...groups].map(([group, ids]) => ({ group, units: ids.join(","), rust: group.startsWith("rust") })); |
| 263 | } |
| 264 | |
| 265 | /** |
| 266 | * What is wrong with the manifest, as sentences. Empty when it is right. |
| 267 | * `wranglerConfigs`: every wrangler.jsonc in the repository (relative paths). |
| 268 | */ |
| 269 | export function problems(stack, wranglerConfigs = findWranglerConfigs()) { |
| 270 | const out = []; |
| 271 | const stages = codeStages(stack); |
| 272 | if (stack.stages[0] !== "migrations") out.push('The first stage is "migrations".'); |
| 273 | const byWorker = new Map(); |
| 274 | for (const unit of stack.units) { |
| 275 | const where = `Unit ${unit.id}`; |
| 276 | if (!KINDS.includes(unit.kind)) out.push(`${where}: kind is one of ${KINDS.join(", ")}.`); |
| 277 | if (!stages.includes(unit.stage)) out.push(`${where}: stage is one of ${stages.join(", ")}.`); |
| 278 | if (!SELF_HOST.includes(unit.self_host)) out.push(`${where}: self_host is one of ${SELF_HOST.join(", ")}.`); |
| 279 | if (!unit.config) { |
| 280 | out.push(`${where}: ${unit.path}/wrangler.jsonc does not exist.`); |
| 281 | continue; |
| 282 | } |
| 283 | if (unit.config.name !== unit.worker) out.push(`${where}: worker is ${unit.config.name} in its wrangler.jsonc, not ${unit.worker}.`); |
| 284 | byWorker.set(unit.worker, unit); |
| 285 | const dbs = (unit.config.d1_databases ?? []).filter((db) => db.migrations_dir); |
| 286 | const d1 = dbs[0] ? { database: dbs[0].database_name, migrations: dbs[0].migrations_dir } : null; |
| 287 | if (dbs.length > 1) out.push(`${where}: more than one D1 database with migrations; the deploy tool applies one per unit.`); |
| 288 | if (JSON.stringify(d1) !== JSON.stringify(unit.d1 ?? null)) { |
| 289 | out.push(`${where}: d1 is ${JSON.stringify(d1)} in its wrangler.jsonc, not ${JSON.stringify(unit.d1 ?? null)}.`); |
| 290 | } |
| 291 | const building = unit.config.build?.command ?? ""; |
| 292 | if (unit.kind === "rust-worker" && !building.includes("scripts/build-rust-worker.mjs")) { |
| 293 | out.push(`${where}: a Rust worker builds with node ../../scripts/build-rust-worker.mjs, not "${building}".`); |
| 294 | } |
| 295 | if (unit.kind !== "rust-worker" && building) out.push(`${where}: only Rust workers have a build command; ${unit.kind} builds in the deploy tool.`); |
| 296 | for (const id of (unit.config.kv_namespaces ?? []).map((kv) => kv.id)) { |
| 297 | if (!stack.resources.kv?.[id]) out.push(`${where}: KV namespace ${id} has no name under resources.kv.`); |
| 298 | } |
| 299 | const hasImage = (unit.config.containers ?? []).some((c) => !String(c.image).includes("registry")); |
| 300 | if (hasImage !== Boolean(unit.image)) out.push(`${where}: image is set exactly when its wrangler.jsonc builds a Containers image.`); |
| 301 | } |
| 302 | const listed = new Set(stack.units.map((u) => posix(join(u.path, "wrangler.jsonc")))); |
| 303 | for (const config of wranglerConfigs) { |
| 304 | if (!listed.has(config)) out.push(`${config} is not in ${STACK_FILE}: add its unit.`); |
| 305 | } |
| 306 | // Stage order follows bindings: a unit binds only to units that ship in |
| 307 | // its stage or before it. |
| 308 | for (const unit of stack.units) { |
| 309 | for (const target of unit.bindsTo) { |
| 310 | const other = byWorker.get(target); |
| 311 | if (!other) { |
| 312 | out.push(`Unit ${unit.id} binds to ${target}, which no unit deploys.`); |
| 313 | } else if (stages.indexOf(other.stage) > stages.indexOf(unit.stage)) { |
| 314 | out.push(`Unit ${unit.id} (${unit.stage}) binds to ${other.id} (${other.stage}), which ships after it.`); |
| 315 | } |
| 316 | } |
| 317 | } |
| 318 | return out; |
| 319 | } |
| 320 | |
| 321 | const SKIP_DIRS = new Set(["node_modules", "target", ".git", "build", "dist", ".wrangler", ".generated", ".astro"]); |
| 322 | |
| 323 | /** Every wrangler.jsonc or wrangler.toml checked into the repository. */ |
| 324 | export function findWranglerConfigs(root = ROOT) { |
| 325 | const found = []; |
| 326 | const walk = (dir) => { |
| 327 | for (const name of readdirSync(join(root, dir))) { |
| 328 | if (SKIP_DIRS.has(name) || name.startsWith(".")) continue; |
| 329 | const path = dir ? `${dir}/${name}` : name; |
| 330 | if (statSync(join(root, path)).isDirectory()) walk(path); |
| 331 | else if (/^wrangler\.(jsonc?|toml)$/.test(name)) found.push(path); |
| 332 | } |
| 333 | }; |
| 334 | walk(""); |
| 335 | return found.sort(); |
| 336 | } |
| 337 | |
| 338 | /** |
| 339 | * Relative imports in a unit's non-test sources that leave its folder: |
| 340 | * each { file, target }. The manifest must cover each one, through a |
| 341 | * workspace dependency or `inputs`. |
| 342 | */ |
| 343 | export function outsideImports(root, unit) { |
| 344 | const found = []; |
| 345 | const pattern = /(?:from\s+|import\s*\(\s*|import\s+)["'](\.{1,2}\/[^"']+)["']/g; |
| 346 | const walk = (dir) => { |
| 347 | for (const name of readdirSync(join(root, dir))) { |
| 348 | if (SKIP_DIRS.has(name) || name.startsWith(".")) continue; |
| 349 | const path = `${dir}/${name}`; |
| 350 | if (statSync(join(root, path)).isDirectory()) { |
| 351 | walk(path); |
| 352 | continue; |
| 353 | } |
| 354 | if (!/\.(m?[jt]sx?|astro)$/.test(name) || /\.test\.[jt]sx?$/.test(name) || name.endsWith(".d.ts")) continue; |
| 355 | const text = readFileSync(join(root, path), "utf8"); |
| 356 | for (const match of text.matchAll(pattern)) { |
| 357 | const target = posix(join(dirname(path), match[1])); |
| 358 | if (!under(target, unit.path)) found.push({ file: path, target }); |
| 359 | } |
| 360 | } |
| 361 | }; |
| 362 | walk(unit.path); |
| 363 | return found; |
| 364 | } |
| 365 | |
| 366 | /** |
| 367 | * npm ci of only what the units need: Wrangler alone for Rust workers, |
| 368 | * and each npm-built unit's workspace with the packages it uses. A CI job |
| 369 | * that deploys three Rust workers does not install the site's toolchain. |
| 370 | */ |
| 371 | export function npmCiArgs(units, npm) { |
| 372 | const workspaces = new Set(); |
| 373 | for (const unit of units) { |
| 374 | if (!unit.pkg) continue; |
| 375 | const stack = [unit.pkg]; |
| 376 | while (stack.length) { |
| 377 | const name = stack.pop(); |
| 378 | if (workspaces.has(name)) continue; |
| 379 | workspaces.add(name); |
| 380 | stack.push(...(npm.get(name)?.deps ?? [])); |
| 381 | } |
| 382 | } |
| 383 | const base = ["ci", "--no-audit", "--no-fund"]; |
| 384 | if (!workspaces.size) return [...base, "--workspaces=false"]; |
| 385 | return [...base, "--include-workspace-root", ...[...workspaces].sort().flatMap((name) => ["-w", name])]; |
| 386 | } |