| 1 | #!/usr/bin/env node |
| 2 | // Deploys g1t to Cloudflare from deploy/stack.jsonc: only what changed since |
| 3 | // each Worker's live commit, migrations first, then stage by stage. |
| 4 | // docs.g1t.sh/guides/deploy-to-cloudflare/ is the guide. |
| 5 | // |
| 6 | // node scripts/deploy.mjs plan what would deploy, and why (read-only) |
| 7 | // node scripts/deploy.mjs deploy migrations, then every changed unit |
| 8 | // node scripts/deploy.mjs deploy --only web,api just these (if changed; --force: anyway) |
| 9 | // node scripts/deploy.mjs build --only events build as a deploy would, upload nothing |
| 10 | // node scripts/deploy.mjs migrate pending D1 migrations only |
| 11 | // node scripts/deploy.mjs manifest [--check] the resolved manifest, or its problems |
| 12 | // node scripts/deploy.mjs doctor which units lack their secrets |
| 13 | // node scripts/deploy.mjs install --only a,b npm ci of just what those units need (CI) |
| 14 | // node scripts/deploy.mjs build-base build and push the runner's base image (Docker) |
| 15 | // node scripts/deploy.mjs image make the runner's image for this checkout in the registry (--no-push: build it with Docker) |
| 16 | // |
| 17 | // Flags: --all (every unit, changed or not), --only a,b, --skip a,b, |
| 18 | // --force, --concurrency N (default 4), --stage core (one stage), |
| 19 | // --json (plan), --out FILE (plan), --no-migrations, --allow-dirty, |
| 20 | // --rebuild-image, --rebuild-base, --since REV (Workers with no recorded |
| 21 | // commit are taken to run REV); for build-base and image, --no-push, and |
| 22 | // for build-base, --no-cache. |
| 23 | |
| 24 | import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; |
| 25 | import { tmpdir } from "node:os"; |
| 26 | import { join } from "node:path"; |
| 27 | |
| 28 | import { BINARY, OUT_DIR } from "./build-runner.mjs"; |
| 29 | import { ensureWorkerBuild } from "./build-rust-worker.mjs"; |
| 30 | import { |
| 31 | annotation, |
| 32 | apiAuth, |
| 33 | applyMigrations, |
| 34 | DEPLOY_RETRY_WAITS_MS, |
| 35 | dockerAvailable, |
| 36 | exec, |
| 37 | jsonFrom, |
| 38 | lastLines, |
| 39 | pendingMigrations, |
| 40 | readLive, |
| 41 | retryableDeploy, |
| 42 | versionFrom, |
| 43 | wrangler, |
| 44 | wranglerEnv, |
| 45 | wranglerError, |
| 46 | } from "./deploy/cloudflare.mjs"; |
| 47 | import { |
| 48 | baseInputs, |
| 49 | baseState, |
| 50 | baseTag, |
| 51 | baseVersions, |
| 52 | buildBase, |
| 53 | buildRunnerImage, |
| 54 | diskReport, |
| 55 | dockerHas, |
| 56 | dockerLogin, |
| 57 | imageSize, |
| 58 | makeRunnerImage, |
| 59 | pushImage, |
| 60 | readBaseLock, |
| 61 | registryHas, |
| 62 | removeDeployConfig, |
| 63 | runnerRef, |
| 64 | writeBaseLock, |
| 65 | writeDeployConfig, |
| 66 | } from "./deploy/image.mjs"; |
| 67 | import { decide, git, planJson, pool, table } from "./deploy/plan.mjs"; |
| 68 | import { reportDeployment } from "./deploy/report.mjs"; |
| 69 | import { |
| 70 | ROOT, |
| 71 | byStage, |
| 72 | codeStages, |
| 73 | containersRollout, |
| 74 | findWranglerConfigs, |
| 75 | npmCiArgs, |
| 76 | npmWorkspace, |
| 77 | pick, |
| 78 | problems, |
| 79 | resolvedStack, |
| 80 | waves, |
| 81 | } from "./deploy/stack.mjs"; |
| 82 | import { withDeployWindow } from "./deploy/status-window.mjs"; |
| 83 | import { publishWebAssets, verifyWebAssets } from "./deploy/web-assets.mjs"; |
| 84 | |
| 85 | const USAGE = |
| 86 | "usage: node scripts/deploy.mjs plan|deploy|build|migrate|manifest|doctor|install|build-base|image [--all] [--only a,b] [--skip a,b] [--force] [--rollback] [--concurrency N] [--stage S] [--json]"; |
| 87 | |
| 88 | function parseArgs(argv) { |
| 89 | const opts = { command: argv[0], only: [], skip: [], concurrency: 4, force: false, all: false, json: false }; |
| 90 | for (let i = 1; i < argv.length; i++) { |
| 91 | const arg = argv[i]; |
| 92 | const [flag, inline] = arg.split(/=(.*)/s); |
| 93 | const value = () => inline ?? argv[++i]; |
| 94 | if (flag === "--only") opts.only.push(value()); |
| 95 | else if (flag === "--skip") opts.skip.push(value()); |
| 96 | else if (flag === "--concurrency") opts.concurrency = Number(value()); |
| 97 | else if (flag === "--stage") opts.stage = value(); |
| 98 | else if (flag === "--all") opts.all = true; |
| 99 | else if (flag === "--force") opts.force = true; |
| 100 | else if (flag === "--rollback") opts.rollback = true; |
| 101 | else if (flag === "--json") opts.json = true; |
| 102 | else if (flag === "--check") opts.check = true; |
| 103 | else if (flag === "--no-migrations") opts.noMigrations = true; |
| 104 | else if (flag === "--allow-dirty") opts.allowDirty = true; |
| 105 | else if (flag === "--rebuild-image") opts.rebuildImage = true; |
| 106 | else if (flag === "--rebuild-base") opts.rebuildBase = true; |
| 107 | else if (flag === "--no-push") opts.noPush = true; |
| 108 | else if (flag === "--no-cache") opts.noCache = true; |
| 109 | else if (flag === "--out") opts.out = value(); |
| 110 | else if (flag === "--since") opts.since = value(); |
| 111 | else if (flag === "--github-output") opts.githubOutput = true; |
| 112 | else throw new Error(`unknown flag ${arg}\n${USAGE}`); |
| 113 | } |
| 114 | if (!Number.isInteger(opts.concurrency) || opts.concurrency < 1) throw new Error("--concurrency is a whole number, 1 or more"); |
| 115 | return opts; |
| 116 | } |
| 117 | |
| 118 | /** The units a command works on: --only (or all), less --skip, in --stage. */ |
| 119 | function selected(stack, opts) { |
| 120 | let units = opts.only.length ? pick(stack, opts.only) : [...stack.units]; |
| 121 | const skipped = pick(stack, opts.skip); |
| 122 | units = units.filter((u) => !skipped.includes(u)); |
| 123 | if (opts.stage) { |
| 124 | if (!codeStages(stack).includes(opts.stage)) throw new Error(`--stage is one of ${codeStages(stack).join(", ")}`); |
| 125 | units = units.filter((u) => u.stage === opts.stage); |
| 126 | } |
| 127 | // Manifest order, whatever order they were named in. |
| 128 | return stack.units.filter((u) => units.includes(u)); |
| 129 | } |
| 130 | |
| 131 | const log = (...args) => console.error(...args); |
| 132 | |
| 133 | /** |
| 134 | * The plan for a workflow (.g1t/workflows/deploy.yml): job outputs in |
| 135 | * $GITHUB_OUTPUT, and the plan as a table in $GITHUB_STEP_SUMMARY. |
| 136 | */ |
| 137 | function writeGithubOutputs(data, p) { |
| 138 | const lines = [ |
| 139 | `commit=${data.commit}`, |
| 140 | `migrate=${data.migrations.length > 0}`, |
| 141 | `migrate_units=${data.migrations.map((m) => m.unit).join(",")}`, |
| 142 | `deploying=${data.units |
| 143 | .filter((u) => u.deploy) |
| 144 | .map((u) => u.unit) |
| 145 | .join(",")}`, |
| 146 | ]; |
| 147 | for (const [stage, { jobs }] of Object.entries(data.stages)) { |
| 148 | // A matrix needs one entry; an empty stage's job is skipped by its `if`. |
| 149 | const include = jobs.length ? jobs : [{ group: "none", units: "" }]; |
| 150 | lines.push(`has_${stage}=${jobs.length > 0}`, `${stage}=${JSON.stringify({ include })}`); |
| 151 | } |
| 152 | if (data.migration_errors.length) { |
| 153 | // The units, then why for the first: one cause (a token, say) is usually all of them. |
| 154 | throw new Error(`Could not read pending migrations: ${data.migration_errors.map((e) => e.unit).join(", ")} |
| 155 | ${data.migration_errors[0].error}`); |
| 156 | } |
| 157 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); |
| 158 | else console.log(lines.join("\n")); |
| 159 | if (process.env.GITHUB_STEP_SUMMARY) { |
| 160 | const rows = p.decisions.map( |
| 161 | (d) => `| ${d.unit.id} | ${d.unit.stage} | ${d.deploy ? "deploy" : ""} | ${d.since ? d.since.slice(0, 12) : "?"} | ${d.reason.replaceAll("|", "\\|")} |`, |
| 162 | ); |
| 163 | const pending = data.migrations.map((m) => `- ${m.database}: ${m.pending.join(", ")}`); |
| 164 | appendFileSync( |
| 165 | process.env.GITHUB_STEP_SUMMARY, |
| 166 | [ |
| 167 | `## Deploy plan for ${data.commit.slice(0, 12)}`, |
| 168 | "", |
| 169 | "| unit | stage | action | live | why |", |
| 170 | "| --- | --- | --- | --- | --- |", |
| 171 | ...rows, |
| 172 | "", |
| 173 | pending.length ? "### Pending migrations" : "", |
| 174 | ...pending, |
| 175 | "", |
| 176 | ].join("\n"), |
| 177 | ); |
| 178 | } |
| 179 | } |
| 180 | |
| 181 | const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`; |
| 182 | |
| 183 | /** `fn`, called the first time it is wanted and then remembered. */ |
| 184 | const once = (fn) => { |
| 185 | let answer = null; |
| 186 | return () => (answer ??= fn()); |
| 187 | }; |
| 188 | |
| 189 | /** Reads what each unit runs and what its database is waiting for. */ |
| 190 | async function survey(units, opts) { |
| 191 | const live = {}; |
| 192 | const migrations = {}; |
| 193 | const tasks = [ |
| 194 | ...(opts.noLive ? [] : units).map((unit) => async () => { |
| 195 | live[unit.id] = await readLive(unit); |
| 196 | }), |
| 197 | ...(opts.noMigrations ? [] : units.filter((u) => u.d1)).map((unit) => async () => { |
| 198 | migrations[unit.id] = await pendingMigrations(unit); |
| 199 | }), |
| 200 | ]; |
| 201 | // Through the API every read starts at once (the requests themselves are |
| 202 | // held to API_CONCURRENCY); through Wrangler, a few processes at a time. |
| 203 | await pool(tasks, apiAuth() ? tasks.length : Math.max(8, opts.concurrency * 2), (task) => task()); |
| 204 | return { live, migrations }; |
| 205 | } |
| 206 | |
| 207 | async function plan(stack, opts) { |
| 208 | const units = selected(stack, opts); |
| 209 | const head = git.head(); |
| 210 | const { live, migrations } = await survey(units, opts); |
| 211 | // --since: what a Worker with no recorded commit is taken to run (once, |
| 212 | // to adopt Workers deployed before this tool), for example --since HEAD~3. |
| 213 | if (opts.since) { |
| 214 | const since = git.resolve(opts.since); |
| 215 | for (const unit of units) { |
| 216 | const found = live[unit.id]; |
| 217 | if (found && !found.sha && !found.missing && !found.error) live[unit.id] = { ...found, sha: since, assumed: true }; |
| 218 | } |
| 219 | } |
| 220 | const decisions = decide(units, { live, head, force: opts.force || opts.all, rollback: opts.rollback }); |
| 221 | return { head, units, live, migrations, decisions }; |
| 222 | } |
| 223 | |
| 224 | function buildPlan(stack, opts) { |
| 225 | const units = selected(stack, opts); |
| 226 | return { |
| 227 | head: git.head(), |
| 228 | units, |
| 229 | live: {}, |
| 230 | migrations: {}, |
| 231 | decisions: units.map((unit) => ({ unit, deploy: true, reason: "build", since: null, files: [], image: false })), |
| 232 | }; |
| 233 | } |
| 234 | |
| 235 | function printPlan(stack, { head, decisions, migrations, live }) { |
| 236 | console.log(`Deploying ${head.slice(0, 12)} (${git.subject()})\n`); |
| 237 | const rows = decisions.map((d) => [ |
| 238 | d.unit.id, |
| 239 | d.unit.stage, |
| 240 | d.deploy ? "deploy" : "-", |
| 241 | d.since ? d.since.slice(0, 12) + (live[d.unit.id]?.assumed ? "*" : "") : "?", |
| 242 | d.unit.d1 ? (migrations[d.unit.id]?.error ? "error" : String(migrations[d.unit.id]?.pending?.length ?? "-")) : "", |
| 243 | d.reason + (d.image ? "; image rebuilds" : ""), |
| 244 | ]); |
| 245 | console.log(table(rows, ["unit", "stage", "action", "live", "migrations", "why"])); |
| 246 | if (decisions.some((d) => live[d.unit.id]?.assumed)) console.log("* taken from --since: no commit was recorded for it"); |
| 247 | const pending = Object.entries(migrations).filter(([, m]) => m.pending?.length); |
| 248 | if (pending.length) { |
| 249 | console.log("\nPending migrations:"); |
| 250 | for (const [id, m] of pending) console.log(` ${stack.units.find((u) => u.id === id).d1.database}: ${m.pending.join(", ")}`); |
| 251 | } |
| 252 | for (const [id, m] of Object.entries(migrations).filter(([, m]) => m.error)) { |
| 253 | console.log(`\nCould not list ${id}'s migrations:\n${m.error}`); |
| 254 | } |
| 255 | const deploying = decisions.filter((d) => d.deploy).map((d) => d.unit); |
| 256 | console.log( |
| 257 | deploying.length |
| 258 | ? `\n${deploying.length} to deploy: ${byStage(stack, deploying) |
| 259 | .map((g) => `${g.stage} (${g.units.map((u) => u.id).join(", ")})`) |
| 260 | .join(" -> ")}` |
| 261 | : "\nNothing to deploy.", |
| 262 | ); |
| 263 | } |
| 264 | |
| 265 | /** Output of one unit, prefixed, to the terminal and a log file. */ |
| 266 | function unitLogger(id) { |
| 267 | const dir = join(tmpdir(), "g1t-deploy"); |
| 268 | mkdirSync(dir, { recursive: true }); |
| 269 | const file = join(dir, `${id}.log`); |
| 270 | const lines = []; |
| 271 | return { |
| 272 | file, |
| 273 | line: (text) => { |
| 274 | lines.push(text); |
| 275 | if (text.trim()) log(`[${id}] ${text}`); |
| 276 | }, |
| 277 | save: () => writeFileSync(file, `${lines.join("\n")}\n`), |
| 278 | }; |
| 279 | } |
| 280 | |
| 281 | /** |
| 282 | * The runner's image for this checkout, by registry reference: already in |
| 283 | * the registry (built by an earlier deploy, `deploy.mjs image`, or on |
| 284 | * another machine), or made here. Pushing, it is made in the registry |
| 285 | * (scripts/deploy/registry.mjs: the base's layers and one with the |
| 286 | * binary, nothing pulled), which needs no Docker and no room for the |
| 287 | * base; if that fails and Docker answers, it is built and pushed with |
| 288 | * Docker. A dry run builds it with Docker and pushes nothing. `docker` is |
| 289 | * asked only when needed: asking starts a job's Docker Engine. Returns |
| 290 | * { ref, note }. |
| 291 | */ |
| 292 | async function runnerImage(unit, { dryRun, docker, rebuildImage, rebuildBase }, out) { |
| 293 | let base = baseState(unit); |
| 294 | if (!base.current || rebuildBase) { |
| 295 | if (!rebuildBase) { |
| 296 | throw new Error( |
| 297 | `${unit.image.base.context} has changed since ${unit.image.base.lock} was written${base.lock ? "" : " (the base has never been built)"}. Build and push the base, and commit ${unit.image.base.lock}: node scripts/deploy.mjs build-base`, |
| 298 | ); |
| 299 | } |
| 300 | if (!(await docker())) throw new Error("--rebuild-base needs Docker."); |
| 301 | await newBase(unit, { push: !dryRun, onLine: out.line }); |
| 302 | base = baseState(unit); |
| 303 | } |
| 304 | if (!base.pushed && !dryRun) throw new Error(`${unit.image.base.lock} records a base that was never pushed: node scripts/deploy.mjs build-base`); |
| 305 | const ref = runnerRef(unit); |
| 306 | const tag = ref.split(":").pop(); |
| 307 | if (!rebuildImage) { |
| 308 | if (dryRun && (await docker()) && (await dockerHas(ref))) return { ref, note: `image ${tag} already built here` }; |
| 309 | if (!dryRun) { |
| 310 | try { |
| 311 | if (await registryHas(ref)) return { ref, note: `image ${tag} already in the registry` }; |
| 312 | } catch (error) { |
| 313 | out.line(`could not ask the registry for ${tag}: ${error.message ?? error}`); |
| 314 | } |
| 315 | } |
| 316 | } |
| 317 | if (dryRun && !(await docker())) { |
| 318 | throw new Error(`its image ${tag} is not built here, and Docker does not answer here: a dry run builds it with Docker.`); |
| 319 | } |
| 320 | const started = Date.now(); |
| 321 | const binary = await exec(process.execPath, [join(ROOT, "scripts/build-runner.mjs")], { onLine: out.line }); |
| 322 | if (binary.code !== 0) throw new Error(`the runner binary did not build:\n${lastLines(binary.out)}`); |
| 323 | if (!dryRun) { |
| 324 | try { |
| 325 | await makeRunnerImage({ ref, base: base.ref, binary: BINARY, onLine: out.line }); |
| 326 | return { ref, note: `image ${tag} made in the registry in ${seconds(Date.now() - started)}` }; |
| 327 | } catch (error) { |
| 328 | if (!(await docker())) { |
| 329 | throw new Error(`its image ${tag} could not be made in the registry (${error.message ?? error}), and Docker does not answer here to build it instead.`); |
| 330 | } |
| 331 | out.line(`could not make ${tag} in the registry (${error.message ?? error}); building it with Docker instead`); |
| 332 | } |
| 333 | } |
| 334 | // With Docker: the base is pulled and unpacked on this disk, so what the |
| 335 | // disk held is logged before and after, and when it fails. |
| 336 | await diskReport(out.line, "before the build"); |
| 337 | try { |
| 338 | if (!dryRun) await dockerLogin({ push: true }); |
| 339 | await buildRunnerImage(unit, { ref, base: base.ref, binaryDir: OUT_DIR, onLine: out.line }); |
| 340 | if (!dryRun) await pushImage(ref, { onLine: out.line }); |
| 341 | } finally { |
| 342 | await diskReport(out.line, "after the build"); |
| 343 | } |
| 344 | return { ref, note: `image ${tag} ${dryRun ? "built" : "built and pushed"} with Docker in ${seconds(Date.now() - started)}` }; |
| 345 | } |
| 346 | |
| 347 | /** Builds the base (and pushes it unless `push` is false), and writes its lock. */ |
| 348 | async function newBase(unit, { push = true, noCache = false, onLine = log } = {}) { |
| 349 | const started = Date.now(); |
| 350 | const inputs = baseInputs(unit); |
| 351 | const tag = baseTag(inputs); |
| 352 | const previous = readBaseLock(unit); |
| 353 | // Logged in, the base it replaces is pulled as cache. |
| 354 | if (push || previous?.pushed) { |
| 355 | try { |
| 356 | await dockerLogin({ push }); |
| 357 | } catch (error) { |
| 358 | if (push) throw error; |
| 359 | onLine(`not logged in to the registry, so no cache from it: ${error.message ?? error}`); |
| 360 | } |
| 361 | } |
| 362 | const ref = await buildBase(unit, { tag, previous: previous?.pushed ? previous.image : null, onLine, noCache }); |
| 363 | const built = Date.now(); |
| 364 | const [size, versions] = await Promise.all([imageSize(ref), baseVersions(ref)]); |
| 365 | const digest = push ? await pushImage(ref, { onLine }) : null; |
| 366 | const lock = { image: ref, digest, pushed: push, inputs, built_at: new Date().toISOString(), size_bytes: size, versions }; |
| 367 | writeBaseLock(unit, lock); |
| 368 | onLine( |
| 369 | `base ${tag}: built in ${seconds(built - started)}${push ? `, pushed in ${seconds(Date.now() - built)}` : ""}, ${(size / 1e9).toFixed(2)} GB unpacked`, |
| 370 | ); |
| 371 | return lock; |
| 372 | } |
| 373 | |
| 374 | /** Builds (and unless `dryRun`, deploys) one unit. */ |
| 375 | async function ship(unit, decision, { head, subject, dirty, dryRun, docker, rebuildImage, rebuildBase }) { |
| 376 | const started = Date.now(); |
| 377 | const out = unitLogger(unit.id); |
| 378 | const cwd = join(ROOT, unit.path); |
| 379 | const result = { unit: unit.id, stage: unit.stage, ok: false, version: null, ms: 0, note: "" }; |
| 380 | try { |
| 381 | if (unit.kind === "react-router" || unit.kind === "astro") { |
| 382 | // One command string: Node warns about arguments passed beside shell: true. |
| 383 | const built = await exec("npm run build", [], { cwd, onLine: out.line, shell: true, env: wranglerEnv() }); |
| 384 | if (built.code !== 0) throw new Error(`npm run build failed:\n${lastLines(built.out)}`); |
| 385 | } |
| 386 | const webRelease = unit.id === "web" && !dryRun ? await publishWebAssets(cwd, { onLine: out.line, seedLive: !decision.missing }) : null; |
| 387 | const args = ["deploy"]; |
| 388 | if (dryRun) { |
| 389 | args.push("--dry-run", "--outdir", join(tmpdir(), "g1t-deploy", "dist", unit.id)); |
| 390 | } else { |
| 391 | const { message, tag } = annotation(head, subject); |
| 392 | args.push("--message", dirty ? message.replace(/^g1t-deploy/, "g1t-deploy-dirty") : message, "--tag", tag); |
| 393 | } |
| 394 | if (unit.image) { |
| 395 | // Wrangler is given the image by reference, so it builds nothing. |
| 396 | const image = await runnerImage(unit, { dryRun, docker, rebuildImage, rebuildBase }, out); |
| 397 | args.push("--config", writeDeployConfig(unit, image.ref)); |
| 398 | args.push("--containers-rollout", containersRollout(unit, decision, { rebuildImage, rebuildBase })); |
| 399 | result.note = image.note; |
| 400 | } |
| 401 | // Tried again after a wait when Cloudflare, not what is deployed, is |
| 402 | // why it failed: deploying the same version again is harmless. |
| 403 | let deployed = await wrangler(args, { cwd, onLine: out.line }); |
| 404 | for (const wait of DEPLOY_RETRY_WAITS_MS) { |
| 405 | if (deployed.code === 0 || dryRun || !retryableDeploy(deployed.out)) break; |
| 406 | out.line(`wrangler deploy failed; trying again in ${wait / 1000}s:\n${wranglerError(deployed.out)}`); |
| 407 | await new Promise((resolve) => setTimeout(resolve, wait)); |
| 408 | deployed = await wrangler(args, { cwd, onLine: out.line }); |
| 409 | } |
| 410 | if (deployed.code !== 0) throw new Error(`wrangler deploy failed:\n${wranglerError(deployed.out)}`); |
| 411 | result.version = dryRun ? "(dry run)" : versionFrom(deployed.out); |
| 412 | if (webRelease) { |
| 413 | try { |
| 414 | await verifyWebAssets(webRelease, fetch, { onLine: out.line }); |
| 415 | } catch (error) { |
| 416 | throw new Error(`Worker deployed, but web cutover verification failed: ${error.message}`); |
| 417 | } |
| 418 | out.line("Web cutover verified: current and previous build assets respond"); |
| 419 | } |
| 420 | result.ok = true; |
| 421 | if (unit.image && !dryRun) await reportContainers(unit, cwd, out); |
| 422 | } catch (error) { |
| 423 | result.note = String(error.message ?? error); |
| 424 | } finally { |
| 425 | if (unit.image) removeDeployConfig(unit); |
| 426 | } |
| 427 | result.ms = Date.now() - started; |
| 428 | out.save(); |
| 429 | if (!result.ok) result.note += `\n full log: ${out.file}`; |
| 430 | return result; |
| 431 | } |
| 432 | |
| 433 | /** |
| 434 | * What Cloudflare runs for a unit's container applications after a deploy: |
| 435 | * each one's instance type, size, version and rollout, from `wrangler |
| 436 | * containers list`, so a deploy log says whether new settings took. Never |
| 437 | * fails the deploy. |
| 438 | */ |
| 439 | async function reportContainers(unit, cwd, out) { |
| 440 | try { |
| 441 | const listed = await wrangler(["containers", "list"], { cwd }); |
| 442 | const text = listed.out.slice(listed.out.indexOf("[")); |
| 443 | const apps = JSON.parse(text.slice(0, text.lastIndexOf("]") + 1)); |
| 444 | for (const app of apps.filter((a) => String(a.name ?? "").startsWith(`${unit.worker}-`))) { |
| 445 | const c = app.configuration ?? {}; |
| 446 | const disk = c.disk?.size_mb ?? c.disk_mb ?? c.disk?.size; |
| 447 | const size = [c.instance_type, c.vcpu != null && `${c.vcpu} vCPU`, c.memory_mib != null && `${c.memory_mib} MiB`, disk != null && `${disk} MB disk`] |
| 448 | .filter(Boolean) |
| 449 | .join(", "); |
| 450 | const rollout = app.active_rollout_id ? `rollout ${app.active_rollout_id} in progress` : "no rollout in progress"; |
| 451 | out.line( |
| 452 | `container ${app.name}: ${size || JSON.stringify(c).slice(0, 200)}; version ${app.version ?? "?"}; ${app.instances ?? "?"} instances; ${rollout}`, |
| 453 | ); |
| 454 | } |
| 455 | } catch (error) { |
| 456 | out.line(`could not read the container applications as JSON (${error.message ?? error}); wrangler said:`); |
| 457 | const said = await wrangler(["containers", "list"], { cwd }).catch(() => ({ out: "" })); |
| 458 | for (const line of said.out |
| 459 | .split("\n") |
| 460 | .filter((l) => l.trim()) |
| 461 | .slice(0, 40)) |
| 462 | out.line(` ${line}`); |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | async function migrate(stack, units, migrations, opts) { |
| 467 | const due = units.filter((u) => migrations[u.id]?.pending?.length); |
| 468 | const broken = units.filter((u) => migrations[u.id]?.error); |
| 469 | if (broken.length) { |
| 470 | throw new Error(`Could not read pending migrations for ${broken.map((u) => u.id).join(", ")}; nothing was deployed.`); |
| 471 | } |
| 472 | if (!due.length) return []; |
| 473 | log(`== migrations: ${due.map((u) => `${u.d1.database} (${migrations[u.id].pending.length})`).join(", ")}`); |
| 474 | const results = await pool(due, opts.concurrency, async (unit) => { |
| 475 | const started = Date.now(); |
| 476 | const out = unitLogger(`${unit.id}-migrations`); |
| 477 | const applied = await applyMigrations(unit, out.line); |
| 478 | out.save(); |
| 479 | return { |
| 480 | unit: `${unit.id} (D1 ${unit.d1.database})`, |
| 481 | stage: "migrations", |
| 482 | ok: applied.code === 0, |
| 483 | version: `${migrations[unit.id].pending.length} applied`, |
| 484 | ms: Date.now() - started, |
| 485 | note: applied.code === 0 ? "" : `${lastLines(applied.out)}\n full log: ${out.file}`, |
| 486 | }; |
| 487 | }); |
| 488 | return results; |
| 489 | } |
| 490 | |
| 491 | function summary(results) { |
| 492 | const rows = results.map((r) => [ |
| 493 | r.unit, |
| 494 | r.stage, |
| 495 | r.ok ? "ok" : r.skipped ? "not started" : "FAILED", |
| 496 | r.version ?? "", |
| 497 | r.ms ? seconds(r.ms) : "", |
| 498 | r.note.split("\n")[0], |
| 499 | ]); |
| 500 | console.log(`\n${table(rows, ["unit", "stage", "result", "version", "time", "note"])}`); |
| 501 | for (const r of results.filter((r) => !r.ok && !r.skipped)) console.log(`\n${r.unit}: ${r.note}`); |
| 502 | } |
| 503 | |
| 504 | async function deploy(stack, opts, { dryRun = false } = {}) { |
| 505 | const started = Date.now(); |
| 506 | // A build reads nothing from Cloudflare: it builds what it is given. |
| 507 | const p = dryRun ? buildPlan(stack, opts) : await plan(stack, opts); |
| 508 | if (!dryRun) printPlan(stack, p); |
| 509 | const deploying = p.decisions.filter((d) => d.deploy); |
| 510 | const touched = deploying.map((d) => d.unit); |
| 511 | const head = p.head; |
| 512 | |
| 513 | // A deploy names the commit it came from, so a dirty tree would be |
| 514 | // recorded as something it is not. |
| 515 | const dirtyFiles = git.dirty(); |
| 516 | const dirty = deploying.some((d) => |
| 517 | dirtyFiles.some((file) => file.startsWith(`${d.unit.path}/`) || d.unit.dependsOn.some((dir) => file.startsWith(`${dir}/`)) || d.unit.inputs.includes(file)), |
| 518 | ); |
| 519 | if (dirty && !dryRun && !opts.allowDirty) { |
| 520 | throw new Error( |
| 521 | "Uncommitted changes touch what would deploy. Commit them, or pass --allow-dirty (the deploy then records no commit, and the next plan deploys it again).", |
| 522 | ); |
| 523 | } |
| 524 | |
| 525 | const results = []; |
| 526 | if (!dryRun && !opts.noMigrations) { |
| 527 | const migrated = await migrate(stack, p.units, p.migrations, opts); |
| 528 | results.push(...migrated); |
| 529 | if (migrated.some((r) => !r.ok)) { |
| 530 | summary(results); |
| 531 | return false; |
| 532 | } |
| 533 | } |
| 534 | if (!touched.length) { |
| 535 | if (results.length) summary(results); |
| 536 | return true; |
| 537 | } |
| 538 | |
| 539 | if (touched.some((u) => u.kind === "rust-worker")) ensureWorkerBuild(); |
| 540 | const docker = once(dockerAvailable); |
| 541 | const context = { head, subject: git.subject(), dirty, dryRun, docker, rebuildImage: opts.rebuildImage, rebuildBase: opts.rebuildBase }; |
| 542 | // A deploy run by hand shows on the repository's Deployments page; a |
| 543 | // dirty tree is not a commit anyone can look at, so it is left out. |
| 544 | const settle = dryRun || dirty ? null : await reportDeployment({ head, subject: context.subject, units: touched.map((u) => u.id), log }); |
| 545 | |
| 546 | // status.g1t.sh hears the deploy start and finish (STATUS_DEPLOY_TOKEN; |
| 547 | // nothing without it), so the restarts it causes are not drafted as |
| 548 | // incidents. Never in a dry run, and it never fails a deploy. |
| 549 | let failed = false; |
| 550 | await withDeployWindow( |
| 551 | async () => { |
| 552 | for (const { stage, units } of byStage(stack, touched)) { |
| 553 | if (failed) { |
| 554 | for (const unit of units) results.push({ unit: unit.id, stage, ok: false, skipped: true, ms: 0, note: "an earlier stage failed" }); |
| 555 | continue; |
| 556 | } |
| 557 | // A Worker that does not exist yet goes before those bound to it. |
| 558 | for (const wave of waves(units, (unit) => Boolean(deploying.find((d) => d.unit === unit)?.missing))) { |
| 559 | if (failed) { |
| 560 | for (const unit of wave) results.push({ unit: unit.id, stage, ok: false, skipped: true, ms: 0, note: "a Worker it binds to failed to deploy" }); |
| 561 | continue; |
| 562 | } |
| 563 | log(`== ${stage}: ${wave.map((u) => u.id).join(", ")}`); |
| 564 | const shipped = await pool(wave, opts.concurrency, (unit) => |
| 565 | ship( |
| 566 | unit, |
| 567 | deploying.find((d) => d.unit === unit), |
| 568 | context, |
| 569 | ), |
| 570 | ); |
| 571 | results.push(...shipped); |
| 572 | failed = shipped.some((r) => !r.ok); |
| 573 | } |
| 574 | } |
| 575 | }, |
| 576 | { id: head ?? null, dryRun, log }, |
| 577 | ); |
| 578 | summary(results); |
| 579 | await settle?.(!failed); |
| 580 | console.log(`\n${failed ? "Failed" : dryRun ? "Built" : "Deployed"} in ${seconds(Date.now() - started)}.`); |
| 581 | return !failed; |
| 582 | } |
| 583 | |
| 584 | async function doctor(stack, opts) { |
| 585 | const units = selected(stack, opts); |
| 586 | const rows = await pool(units, 8, async (unit) => { |
| 587 | if (!unit.secrets.length) return [unit.id, "", "", ""]; |
| 588 | const found = await wrangler(["secret", "list", "--name", unit.worker, "--format", "json"], { cwd: join(ROOT, unit.path) }); |
| 589 | if (found.code !== 0) return [unit.id, unit.secrets.join(" "), "?", "could not read"]; |
| 590 | const have = new Set(jsonFrom(found.out).map((s) => s.name)); |
| 591 | const missing = unit.secrets.filter((name) => !have.has(name)); |
| 592 | return [unit.id, unit.secrets.join(" "), missing.join(" "), missing.length ? "missing" : "ok"]; |
| 593 | }); |
| 594 | console.log(table(rows, ["unit", "secrets", "missing", "result"])); |
| 595 | return rows.every((r) => r[3] !== "missing"); |
| 596 | } |
| 597 | |
| 598 | async function install(stack, opts) { |
| 599 | const units = selected(stack, opts); |
| 600 | const args = npmCiArgs(units, npmWorkspace()); |
| 601 | log(`npm ${args.join(" ")}`); |
| 602 | // Through npm-ci.mjs, which runs it again when a connection drops. |
| 603 | const done = await exec(`node scripts/deploy/npm-ci.mjs ${args.slice(1).join(" ")}`, [], { cwd: ROOT, shell: true, onLine: (line) => log(line) }); |
| 604 | return done.code === 0; |
| 605 | } |
| 606 | |
| 607 | function manifest(stack, opts) { |
| 608 | const found = problems(stack, findWranglerConfigs()); |
| 609 | if (opts.check) { |
| 610 | for (const problem of found) console.log(`- ${problem}`); |
| 611 | console.log(found.length ? `\n${found.length} problems in deploy/stack.jsonc.` : "deploy/stack.jsonc is consistent with every wrangler.jsonc."); |
| 612 | return !found.length; |
| 613 | } |
| 614 | const out = stack.units.map(({ config, ...unit }) => ({ |
| 615 | ...unit, |
| 616 | queues: { |
| 617 | produces: (config?.queues?.producers ?? []).map((q) => q.queue), |
| 618 | consumes: (config?.queues?.consumers ?? []).map((q) => q.queue), |
| 619 | dead_letter: [...new Set((config?.queues?.consumers ?? []).map((q) => q.dead_letter_queue).filter(Boolean))], |
| 620 | }, |
| 621 | kv: (config?.kv_namespaces ?? []).map((kv) => stack.resources.kv?.[kv.id] ?? kv.id), |
| 622 | r2: (config?.r2_buckets ?? []).map((b) => b.bucket_name), |
| 623 | vectorize: (config?.vectorize ?? []).map((v) => v.index_name), |
| 624 | dispatch_namespaces: (config?.dispatch_namespaces ?? []).map((d) => d.namespace), |
| 625 | routes: (config?.routes ?? []).map((r) => r.pattern), |
| 626 | })); |
| 627 | if (opts.json) console.log(JSON.stringify({ stages: stack.stages, units: out }, null, 2)); |
| 628 | else |
| 629 | console.log( |
| 630 | table( |
| 631 | out.map((u) => [u.id, u.stage, u.kind, u.worker, u.d1?.database ?? "", u.dependsOn.join(" ")]), |
| 632 | ["unit", "stage", "kind", "worker", "d1", "built from (besides its folder)"], |
| 633 | ), |
| 634 | ); |
| 635 | return true; |
| 636 | } |
| 637 | |
| 638 | /** The unit with a Containers image (the runner), or the one named. */ |
| 639 | function imageUnit(stack, opts) { |
| 640 | const units = (opts.only.length ? pick(stack, opts.only) : stack.units).filter((u) => u.image?.base); |
| 641 | if (units.length !== 1) throw new Error("Name the unit with a Containers image: --only runner"); |
| 642 | return units[0]; |
| 643 | } |
| 644 | |
| 645 | async function main() { |
| 646 | const opts = parseArgs(process.argv.slice(2)); |
| 647 | const stack = resolvedStack(); |
| 648 | switch (opts.command) { |
| 649 | case "plan": { |
| 650 | const p = await plan(stack, opts); |
| 651 | const data = planJson(stack, p.decisions, p.migrations, p.head); |
| 652 | if (opts.out) writeFileSync(opts.out, `${JSON.stringify(data)}\n`); |
| 653 | if (opts.githubOutput) writeGithubOutputs(data, p); |
| 654 | if (opts.json) console.log(JSON.stringify(data, null, 2)); |
| 655 | else if (!opts.githubOutput || process.env.GITHUB_OUTPUT) printPlan(stack, p); |
| 656 | return true; |
| 657 | } |
| 658 | case "deploy": |
| 659 | return deploy(stack, opts); |
| 660 | case "build": |
| 661 | return deploy(stack, { ...opts, force: true }, { dryRun: true }); |
| 662 | case "migrate": { |
| 663 | const units = selected(stack, opts); |
| 664 | const { migrations } = await survey( |
| 665 | units.filter((u) => u.d1), |
| 666 | { ...opts, noMigrations: false, noLive: true }, |
| 667 | ); |
| 668 | const results = await migrate(stack, units, migrations, opts); |
| 669 | if (results.length) summary(results); |
| 670 | else console.log("No migrations to apply."); |
| 671 | return results.every((r) => r.ok); |
| 672 | } |
| 673 | case "manifest": |
| 674 | return manifest(stack, opts); |
| 675 | case "doctor": |
| 676 | return doctor(stack, opts); |
| 677 | case "install": |
| 678 | return install(stack, opts); |
| 679 | case "build-base": { |
| 680 | const unit = imageUnit(stack, opts); |
| 681 | if (!(await dockerAvailable())) throw new Error("build-base needs Docker."); |
| 682 | const lock = await newBase(unit, { push: !opts.noPush, noCache: opts.noCache }); |
| 683 | console.log(JSON.stringify(lock, null, 2)); |
| 684 | if (lock.pushed) console.log(`\nCommit ${unit.image.base.lock}: the next deploy builds the runner's image on this base.`); |
| 685 | return true; |
| 686 | } |
| 687 | case "image": { |
| 688 | const unit = imageUnit(stack, opts); |
| 689 | const out = unitLogger(`${unit.id}-image`); |
| 690 | const docker = once(dockerAvailable); |
| 691 | const image = await runnerImage(unit, { dryRun: Boolean(opts.noPush), docker, rebuildImage: opts.rebuildImage, rebuildBase: opts.rebuildBase }, out); |
| 692 | out.save(); |
| 693 | console.log(`${image.ref}\n${image.note}`); |
| 694 | return true; |
| 695 | } |
| 696 | default: |
| 697 | console.error(USAGE); |
| 698 | return false; |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | main().then( |
| 703 | (ok) => process.exit(ok ? 0 : 1), |
| 704 | (error) => { |
| 705 | console.error(`\n${error.message ?? error}`); |
| 706 | process.exit(1); |
| 707 | }, |
| 708 | ); |