| 1 | #!/usr/bin/env node |
| 2 | // Deploys g1t to Cloudflare from deploy/stack.jsonc: only what changed since |
| 3 | // each Worker's live commit, migrations first, then stage by stage. |
| 4 | // docs.g1t.sh/guides/deploy-to-cloudflare/ is the guide. |
| 5 | // |
| 6 | // node scripts/deploy.mjs plan what would deploy, and why (read-only) |
| 7 | // node scripts/deploy.mjs deploy migrations, then every changed unit |
| 8 | // node scripts/deploy.mjs deploy --only web,api just these (if changed; --force: anyway) |
| 9 | // node scripts/deploy.mjs build --only events build as a deploy would, upload nothing |
| 10 | // node scripts/deploy.mjs outputs --only events what a build leaves for the deploy, one path a line |
| 11 | // node scripts/deploy.mjs migrate pending D1 migrations only |
| 12 | // node scripts/deploy.mjs manifest [--check] the resolved manifest, or its problems |
| 13 | // node scripts/deploy.mjs doctor which units lack their secrets |
| 14 | // node scripts/deploy.mjs install --only a,b npm ci of just what those units need (CI) |
| 15 | // node scripts/deploy.mjs build-base build and push the runner's base image (Docker) |
| 16 | // node scripts/deploy.mjs image make the runner's image for this checkout in the registry (--no-push: build it with Docker) |
| 17 | // |
| 18 | // Flags: --all (every unit, changed or not), --only a,b, --skip a,b, |
| 19 | // --force, --concurrency N (default 4), --stage core (one stage), |
| 20 | // --json (plan), --out FILE (plan), --no-migrations, --allow-dirty, |
| 21 | // --rebuild-image, --rebuild-base, --since REV (Workers with no recorded |
| 22 | // commit are taken to run REV), --prebuilt (deploy what `build` left in |
| 23 | // each unit's outputs, building nothing), --urgent (the runner's |
| 24 | // sandboxes and edges are replaced now, stopping jobs mid-run: for a fix |
| 25 | // that cannot wait for them to finish); for build-base and image, |
| 26 | // --no-push, and for build-base, --no-cache. |
| 27 | |
| 28 | import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; |
| 29 | import { tmpdir } from "node:os"; |
| 30 | import { join } from "node:path"; |
| 31 | |
| 32 | import { BINARY, OUT_DIR } from "./build-runner.mjs"; |
| 33 | import { ensureWorkerBuild } from "./build-rust-worker.mjs"; |
| 34 | import { |
| 35 | annotation, |
| 36 | apiAuth, |
| 37 | applyMigrations, |
| 38 | DEPLOY_RETRY_WAITS_MS, |
| 39 | dockerAvailable, |
| 40 | exec, |
| 41 | jsonFrom, |
| 42 | lastLines, |
| 43 | pendingMigrations, |
| 44 | readLive, |
| 45 | retryableDeploy, |
| 46 | versionFrom, |
| 47 | wrangler, |
| 48 | wranglerEnv, |
| 49 | wranglerError, |
| 50 | } from "./deploy/cloudflare.mjs"; |
| 51 | import { |
| 52 | baseInputs, |
| 53 | baseState, |
| 54 | baseTag, |
| 55 | baseVersions, |
| 56 | buildBase, |
| 57 | buildRunnerImage, |
| 58 | diskReport, |
| 59 | dockerHas, |
| 60 | dockerLogin, |
| 61 | imageSize, |
| 62 | makeRunnerImage, |
| 63 | pushImage, |
| 64 | readBaseLock, |
| 65 | registryHas, |
| 66 | removeDeployConfig, |
| 67 | runnerRef, |
| 68 | writeBaseLock, |
| 69 | writeDeployConfig, |
| 70 | } from "./deploy/image.mjs"; |
| 71 | import { decide, git, planJson, pool, table } from "./deploy/plan.mjs"; |
| 72 | import { reportDeployment } from "./deploy/report.mjs"; |
| 73 | import { |
| 74 | ROOT, |
| 75 | buildOutputs, |
| 76 | byStage, |
| 77 | codeStages, |
| 78 | containersRollout, |
| 79 | findWranglerConfigs, |
| 80 | npmCiArgs, |
| 81 | npmWorkspace, |
| 82 | pick, |
| 83 | problems, |
| 84 | resolvedStack, |
| 85 | waves, |
| 86 | } from "./deploy/stack.mjs"; |
| 87 | import { withDeployWindow } from "./deploy/status-window.mjs"; |
| 88 | import { publishWebAssets, verifyWebAssets } from "./deploy/web-assets.mjs"; |
| 89 | |
| 90 | const USAGE = |
| 91 | "usage: node scripts/deploy.mjs plan|deploy|build|outputs|migrate|manifest|doctor|install|build-base|image [--all] [--only a,b] [--skip a,b] [--force] [--rollback] [--concurrency N] [--stage S] [--urgent] [--json]"; |
| 92 | |
| 93 | function parseArgs(argv) { |
| 94 | const opts = { command: argv[0], only: [], skip: [], concurrency: 4, force: false, all: false, json: false }; |
| 95 | for (let i = 1; i < argv.length; i++) { |
| 96 | const arg = argv[i]; |
| 97 | const [flag, inline] = arg.split(/=(.*)/s); |
| 98 | const value = () => inline ?? argv[++i]; |
| 99 | if (flag === "--only") opts.only.push(value()); |
| 100 | else if (flag === "--skip") opts.skip.push(value()); |
| 101 | else if (flag === "--concurrency") opts.concurrency = Number(value()); |
| 102 | else if (flag === "--stage") opts.stage = value(); |
| 103 | else if (flag === "--all") opts.all = true; |
| 104 | else if (flag === "--force") opts.force = true; |
| 105 | else if (flag === "--rollback") opts.rollback = true; |
| 106 | else if (flag === "--json") opts.json = true; |
| 107 | else if (flag === "--check") opts.check = true; |
| 108 | else if (flag === "--no-migrations") opts.noMigrations = true; |
| 109 | else if (flag === "--allow-dirty") opts.allowDirty = true; |
| 110 | else if (flag === "--rebuild-image") opts.rebuildImage = true; |
| 111 | else if (flag === "--rebuild-base") opts.rebuildBase = true; |
| 112 | else if (flag === "--no-push") opts.noPush = true; |
| 113 | else if (flag === "--no-cache") opts.noCache = true; |
| 114 | else if (flag === "--out") opts.out = value(); |
| 115 | else if (flag === "--since") opts.since = value(); |
| 116 | else if (flag === "--github-output") opts.githubOutput = true; |
| 117 | else if (flag === "--prebuilt") opts.prebuilt = true; |
| 118 | else if (flag === "--urgent") opts.urgent = true; |
| 119 | else throw new Error(`unknown flag ${arg}\n${USAGE}`); |
| 120 | } |
| 121 | if (!Number.isInteger(opts.concurrency) || opts.concurrency < 1) throw new Error("--concurrency is a whole number, 1 or more"); |
| 122 | return opts; |
| 123 | } |
| 124 | |
| 125 | /** The units a command works on: --only (or all), less --skip, in --stage. */ |
| 126 | function selected(stack, opts) { |
| 127 | let units = opts.only.length ? pick(stack, opts.only) : [...stack.units]; |
| 128 | const skipped = pick(stack, opts.skip); |
| 129 | units = units.filter((u) => !skipped.includes(u)); |
| 130 | if (opts.stage) { |
| 131 | if (!codeStages(stack).includes(opts.stage)) throw new Error(`--stage is one of ${codeStages(stack).join(", ")}`); |
| 132 | units = units.filter((u) => u.stage === opts.stage); |
| 133 | } |
| 134 | // Manifest order, whatever order they were named in. |
| 135 | return stack.units.filter((u) => units.includes(u)); |
| 136 | } |
| 137 | |
| 138 | const log = (...args) => console.error(...args); |
| 139 | |
| 140 | /** |
| 141 | * The plan for a workflow (.g1t/workflows/deploy.yml): job outputs in |
| 142 | * $GITHUB_OUTPUT, and the plan as a table in $GITHUB_STEP_SUMMARY. |
| 143 | */ |
| 144 | function writeGithubOutputs(data, p) { |
| 145 | const lines = [ |
| 146 | `commit=${data.commit}`, |
| 147 | `migrate=${data.migrations.length > 0}`, |
| 148 | `migrate_units=${data.migrations.map((m) => m.unit).join(",")}`, |
| 149 | `deploying=${data.units |
| 150 | .filter((u) => u.deploy) |
| 151 | .map((u) => u.unit) |
| 152 | .join(",")}`, |
| 153 | ]; |
| 154 | // Each job names its stage, which names the artifact its build is kept in. |
| 155 | const builds = []; |
| 156 | for (const [stage, { jobs }] of Object.entries(data.stages)) { |
| 157 | const staged = jobs.map((job) => ({ ...job, stage })); |
| 158 | builds.push(...staged.filter((job) => job.prebuilt)); |
| 159 | // A matrix needs one entry; an empty stage's job is skipped by its `if`. |
| 160 | const include = staged.length ? staged : [{ group: "none", units: "", stage }]; |
| 161 | lines.push(`has_${stage}=${staged.length > 0}`, `${stage}=${JSON.stringify({ include })}`); |
| 162 | } |
| 163 | // Every stage's builds, made at once before any stage deploys. |
| 164 | lines.push( |
| 165 | `has_builds=${builds.length > 0}`, |
| 166 | `builds=${JSON.stringify({ include: builds.length ? builds : [{ group: "none", units: "", stage: "none" }] })}`, |
| 167 | ); |
| 168 | if (data.migration_errors.length) { |
| 169 | // The units, then why for the first: one cause (a token, say) is usually all of them. |
| 170 | throw new Error(`Could not read pending migrations: ${data.migration_errors.map((e) => e.unit).join(", ")} |
| 171 | ${data.migration_errors[0].error}`); |
| 172 | } |
| 173 | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); |
| 174 | else console.log(lines.join("\n")); |
| 175 | if (process.env.GITHUB_STEP_SUMMARY) { |
| 176 | const rows = p.decisions.map( |
| 177 | (d) => `| ${d.unit.id} | ${d.unit.stage} | ${d.deploy ? "deploy" : ""} | ${d.since ? d.since.slice(0, 12) : "?"} | ${d.reason.replaceAll("|", "\\|")} |`, |
| 178 | ); |
| 179 | const pending = data.migrations.map((m) => `- ${m.database}: ${m.pending.join(", ")}`); |
| 180 | appendFileSync( |
| 181 | process.env.GITHUB_STEP_SUMMARY, |
| 182 | [ |
| 183 | `## Deploy plan for ${data.commit.slice(0, 12)}`, |
| 184 | "", |
| 185 | "| unit | stage | action | live | why |", |
| 186 | "| --- | --- | --- | --- | --- |", |
| 187 | ...rows, |
| 188 | "", |
| 189 | pending.length ? "### Pending migrations" : "", |
| 190 | ...pending, |
| 191 | "", |
| 192 | ].join("\n"), |
| 193 | ); |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`; |
| 198 | |
| 199 | /** `fn`, called the first time it is wanted and then remembered. */ |
| 200 | const once = (fn) => { |
| 201 | let answer = null; |
| 202 | return () => (answer ??= fn()); |
| 203 | }; |
| 204 | |
| 205 | /** Reads what each unit runs and what its database is waiting for. */ |
| 206 | async function survey(units, opts) { |
| 207 | const live = {}; |
| 208 | const migrations = {}; |
| 209 | const tasks = [ |
| 210 | ...(opts.noLive ? [] : units).map((unit) => async () => { |
| 211 | live[unit.id] = await readLive(unit); |
| 212 | }), |
| 213 | ...(opts.noMigrations ? [] : units.filter((u) => u.d1)).map((unit) => async () => { |
| 214 | migrations[unit.id] = await pendingMigrations(unit); |
| 215 | }), |
| 216 | ]; |
| 217 | // Through the API every read starts at once (the requests themselves are |
| 218 | // held to API_CONCURRENCY); through Wrangler, a few processes at a time. |
| 219 | await pool(tasks, apiAuth() ? tasks.length : Math.max(8, opts.concurrency * 2), (task) => task()); |
| 220 | return { live, migrations }; |
| 221 | } |
| 222 | |
| 223 | async function plan(stack, opts) { |
| 224 | const units = selected(stack, opts); |
| 225 | const head = git.head(); |
| 226 | const { live, migrations } = await survey(units, opts); |
| 227 | // --since: what a Worker with no recorded commit is taken to run (once, |
| 228 | // to adopt Workers deployed before this tool), for example --since HEAD~3. |
| 229 | if (opts.since) { |
| 230 | const since = git.resolve(opts.since); |
| 231 | for (const unit of units) { |
| 232 | const found = live[unit.id]; |
| 233 | if (found && !found.sha && !found.missing && !found.error) live[unit.id] = { ...found, sha: since, assumed: true }; |
| 234 | } |
| 235 | } |
| 236 | const decisions = decide(units, { live, head, force: opts.force || opts.all, rollback: opts.rollback }); |
| 237 | return { head, units, live, migrations, decisions }; |
| 238 | } |
| 239 | |
| 240 | function buildPlan(stack, opts) { |
| 241 | const units = selected(stack, opts); |
| 242 | return { |
| 243 | head: git.head(), |
| 244 | units, |
| 245 | live: {}, |
| 246 | migrations: {}, |
| 247 | decisions: units.map((unit) => ({ unit, deploy: true, reason: "build", since: null, files: [], image: false })), |
| 248 | }; |
| 249 | } |
| 250 | |
| 251 | function printPlan(stack, { head, decisions, migrations, live }) { |
| 252 | console.log(`Deploying ${head.slice(0, 12)} (${git.subject()})\n`); |
| 253 | const rows = decisions.map((d) => [ |
| 254 | d.unit.id, |
| 255 | d.unit.stage, |
| 256 | d.deploy ? "deploy" : "-", |
| 257 | d.since ? d.since.slice(0, 12) + (live[d.unit.id]?.assumed ? "*" : "") : "?", |
| 258 | d.unit.d1 ? (migrations[d.unit.id]?.error ? "error" : String(migrations[d.unit.id]?.pending?.length ?? "-")) : "", |
| 259 | d.reason + (d.image ? "; image rebuilds" : ""), |
| 260 | ]); |
| 261 | console.log(table(rows, ["unit", "stage", "action", "live", "migrations", "why"])); |
| 262 | if (decisions.some((d) => live[d.unit.id]?.assumed)) console.log("* taken from --since: no commit was recorded for it"); |
| 263 | const pending = Object.entries(migrations).filter(([, m]) => m.pending?.length); |
| 264 | if (pending.length) { |
| 265 | console.log("\nPending migrations:"); |
| 266 | for (const [id, m] of pending) console.log(` ${stack.units.find((u) => u.id === id).d1.database}: ${m.pending.join(", ")}`); |
| 267 | } |
| 268 | for (const [id, m] of Object.entries(migrations).filter(([, m]) => m.error)) { |
| 269 | console.log(`\nCould not list ${id}'s migrations:\n${m.error}`); |
| 270 | } |
| 271 | const deploying = decisions.filter((d) => d.deploy).map((d) => d.unit); |
| 272 | console.log( |
| 273 | deploying.length |
| 274 | ? `\n${deploying.length} to deploy: ${byStage(stack, deploying) |
| 275 | .map((g) => `${g.stage} (${g.units.map((u) => u.id).join(", ")})`) |
| 276 | .join(" -> ")}` |
| 277 | : "\nNothing to deploy.", |
| 278 | ); |
| 279 | } |
| 280 | |
| 281 | /** Output of one unit, prefixed, to the terminal and a log file. */ |
| 282 | function unitLogger(id) { |
| 283 | const dir = join(tmpdir(), "g1t-deploy"); |
| 284 | mkdirSync(dir, { recursive: true }); |
| 285 | const file = join(dir, `${id}.log`); |
| 286 | const lines = []; |
| 287 | return { |
| 288 | file, |
| 289 | line: (text) => { |
| 290 | lines.push(text); |
| 291 | if (text.trim()) log(`[${id}] ${text}`); |
| 292 | }, |
| 293 | save: () => writeFileSync(file, `${lines.join("\n")}\n`), |
| 294 | }; |
| 295 | } |
| 296 | |
| 297 | /** |
| 298 | * The runner's image for this checkout, by registry reference: already in |
| 299 | * the registry (built by an earlier deploy, `deploy.mjs image`, or on |
| 300 | * another machine), or made here. Pushing, it is made in the registry |
| 301 | * (scripts/deploy/registry.mjs: the base's layers and one with the |
| 302 | * binary, nothing pulled), which needs no Docker and no room for the |
| 303 | * base; if that fails and Docker answers, it is built and pushed with |
| 304 | * Docker. A dry run builds it with Docker and pushes nothing. `docker` is |
| 305 | * asked only when needed: asking starts a job's Docker Engine. Returns |
| 306 | * { ref, note }. |
| 307 | */ |
| 308 | async function runnerImage(unit, { dryRun, docker, rebuildImage, rebuildBase }, out) { |
| 309 | let base = baseState(unit); |
| 310 | if (!base.current || rebuildBase) { |
| 311 | if (!rebuildBase) { |
| 312 | throw new Error( |
| 313 | `${unit.image.base.context} has changed since ${unit.image.base.lock} was written${base.lock ? "" : " (the base has never been built)"}. Build and push the base, and commit ${unit.image.base.lock}: node scripts/deploy.mjs build-base`, |
| 314 | ); |
| 315 | } |
| 316 | if (!(await docker())) throw new Error("--rebuild-base needs Docker."); |
| 317 | await newBase(unit, { push: !dryRun, onLine: out.line }); |
| 318 | base = baseState(unit); |
| 319 | } |
| 320 | if (!base.pushed && !dryRun) throw new Error(`${unit.image.base.lock} records a base that was never pushed: node scripts/deploy.mjs build-base`); |
| 321 | const ref = runnerRef(unit); |
| 322 | const tag = ref.split(":").pop(); |
| 323 | if (!rebuildImage) { |
| 324 | if (dryRun && (await docker()) && (await dockerHas(ref))) return { ref, note: `image ${tag} already built here` }; |
| 325 | if (!dryRun) { |
| 326 | try { |
| 327 | if (await registryHas(ref)) return { ref, note: `image ${tag} already in the registry` }; |
| 328 | } catch (error) { |
| 329 | out.line(`could not ask the registry for ${tag}: ${error.message ?? error}`); |
| 330 | } |
| 331 | } |
| 332 | } |
| 333 | if (dryRun && !(await docker())) { |
| 334 | throw new Error(`its image ${tag} is not built here, and Docker does not answer here: a dry run builds it with Docker.`); |
| 335 | } |
| 336 | const started = Date.now(); |
| 337 | const binary = await exec(process.execPath, [join(ROOT, "scripts/build-runner.mjs")], { onLine: out.line }); |
| 338 | if (binary.code !== 0) throw new Error(`the runner binary did not build:\n${lastLines(binary.out)}`); |
| 339 | if (!dryRun) { |
| 340 | try { |
| 341 | await makeRunnerImage({ ref, base: base.ref, binary: BINARY, onLine: out.line }); |
| 342 | return { ref, note: `image ${tag} made in the registry in ${seconds(Date.now() - started)}` }; |
| 343 | } catch (error) { |
| 344 | if (!(await docker())) { |
| 345 | throw new Error(`its image ${tag} could not be made in the registry (${error.message ?? error}), and Docker does not answer here to build it instead.`); |
| 346 | } |
| 347 | out.line(`could not make ${tag} in the registry (${error.message ?? error}); building it with Docker instead`); |
| 348 | } |
| 349 | } |
| 350 | // With Docker: the base is pulled and unpacked on this disk, so what the |
| 351 | // disk held is logged before and after, and when it fails. |
| 352 | await diskReport(out.line, "before the build"); |
| 353 | try { |
| 354 | if (!dryRun) await dockerLogin({ push: true }); |
| 355 | await buildRunnerImage(unit, { ref, base: base.ref, binaryDir: OUT_DIR, onLine: out.line }); |
| 356 | if (!dryRun) await pushImage(ref, { onLine: out.line }); |
| 357 | } finally { |
| 358 | await diskReport(out.line, "after the build"); |
| 359 | } |
| 360 | return { ref, note: `image ${tag} ${dryRun ? "built" : "built and pushed"} with Docker in ${seconds(Date.now() - started)}` }; |
| 361 | } |
| 362 | |
| 363 | /** Builds the base (and pushes it unless `push` is false), and writes its lock. */ |
| 364 | async function newBase(unit, { push = true, noCache = false, onLine = log } = {}) { |
| 365 | const started = Date.now(); |
| 366 | const inputs = baseInputs(unit); |
| 367 | const tag = baseTag(inputs); |
| 368 | const previous = readBaseLock(unit); |
| 369 | // Logged in, the base it replaces is pulled as cache. |
| 370 | if (push || previous?.pushed) { |
| 371 | try { |
| 372 | await dockerLogin({ push }); |
| 373 | } catch (error) { |
| 374 | if (push) throw error; |
| 375 | onLine(`not logged in to the registry, so no cache from it: ${error.message ?? error}`); |
| 376 | } |
| 377 | } |
| 378 | const ref = await buildBase(unit, { tag, previous: previous?.pushed ? previous.image : null, onLine, noCache }); |
| 379 | const built = Date.now(); |
| 380 | const [size, versions] = await Promise.all([imageSize(ref), baseVersions(ref)]); |
| 381 | const digest = push ? await pushImage(ref, { onLine }) : null; |
| 382 | const lock = { image: ref, digest, pushed: push, inputs, built_at: new Date().toISOString(), size_bytes: size, versions }; |
| 383 | writeBaseLock(unit, lock); |
| 384 | onLine( |
| 385 | `base ${tag}: built in ${seconds(built - started)}${push ? `, pushed in ${seconds(Date.now() - built)}` : ""}, ${(size / 1e9).toFixed(2)} GB unpacked`, |
| 386 | ); |
| 387 | return lock; |
| 388 | } |
| 389 | |
| 390 | /** Builds (and unless `dryRun`, deploys) one unit. */ |
| 391 | async function ship(unit, decision, { head, subject, dirty, dryRun, docker, rebuildImage, rebuildBase, prebuilt, urgent }) { |
| 392 | const started = Date.now(); |
| 393 | const out = unitLogger(unit.id); |
| 394 | const cwd = join(ROOT, unit.path); |
| 395 | const result = { unit: unit.id, stage: unit.stage, ok: false, version: null, ms: 0, note: "" }; |
| 396 | try { |
| 397 | if ((unit.kind === "react-router" || unit.kind === "astro") && !prebuilt) { |
| 398 | // One command string: Node warns about arguments passed beside shell: true. |
| 399 | const built = await exec("npm run build", [], { cwd, onLine: out.line, shell: true, env: wranglerEnv() }); |
| 400 | if (built.code !== 0) throw new Error(`npm run build failed:\n${lastLines(built.out)}`); |
| 401 | } |
| 402 | const webRelease = unit.id === "web" && !dryRun ? await publishWebAssets(cwd, { onLine: out.line, seedLive: !decision.missing }) : null; |
| 403 | const args = ["deploy"]; |
| 404 | if (dryRun) { |
| 405 | args.push("--dry-run", "--outdir", join(tmpdir(), "g1t-deploy", "dist", unit.id)); |
| 406 | } else { |
| 407 | const { message, tag } = annotation(head, subject); |
| 408 | args.push("--message", dirty ? message.replace(/^g1t-deploy/, "g1t-deploy-dirty") : message, "--tag", tag); |
| 409 | } |
| 410 | if (unit.image) { |
| 411 | // Wrangler is given the image by reference, so it builds nothing. |
| 412 | const image = await runnerImage(unit, { dryRun, docker, rebuildImage, rebuildBase }, out); |
| 413 | args.push("--config", writeDeployConfig(unit, image.ref, undefined, { urgent })); |
| 414 | args.push("--containers-rollout", containersRollout(unit, decision, { rebuildImage, rebuildBase, urgent })); |
| 415 | result.note = image.note; |
| 416 | } |
| 417 | // Tried again after a wait when Cloudflare, not what is deployed, is |
| 418 | // why it failed: deploying the same version again is harmless. |
| 419 | let deployed = await wrangler(args, { cwd, onLine: out.line }); |
| 420 | for (const wait of DEPLOY_RETRY_WAITS_MS) { |
| 421 | if (deployed.code === 0 || dryRun || !retryableDeploy(deployed.out)) break; |
| 422 | out.line(`wrangler deploy failed; trying again in ${wait / 1000}s:\n${wranglerError(deployed.out)}`); |
| 423 | await new Promise((resolve) => setTimeout(resolve, wait)); |
| 424 | deployed = await wrangler(args, { cwd, onLine: out.line }); |
| 425 | } |
| 426 | if (deployed.code !== 0) throw new Error(`wrangler deploy failed:\n${wranglerError(deployed.out)}`); |
| 427 | result.version = dryRun ? "(dry run)" : versionFrom(deployed.out); |
| 428 | if (webRelease) { |
| 429 | try { |
| 430 | await verifyWebAssets(webRelease, fetch, { onLine: out.line }); |
| 431 | } catch (error) { |
| 432 | throw new Error(`Worker deployed, but web cutover verification failed: ${error.message}`); |
| 433 | } |
| 434 | out.line("Web cutover verified: current and previous build assets respond"); |
| 435 | } |
| 436 | result.ok = true; |
| 437 | if (unit.image && !dryRun) await reportContainers(unit, cwd, out); |
| 438 | } catch (error) { |
| 439 | result.note = String(error.message ?? error); |
| 440 | } finally { |
| 441 | if (unit.image) removeDeployConfig(unit); |
| 442 | } |
| 443 | result.ms = Date.now() - started; |
| 444 | out.save(); |
| 445 | if (!result.ok) result.note += `\n full log: ${out.file}`; |
| 446 | return result; |
| 447 | } |
| 448 | |
| 449 | /** |
| 450 | * What Cloudflare runs for a unit's container applications after a deploy: |
| 451 | * each one's instance type, size, version and rollout, from `wrangler |
| 452 | * containers list`, so a deploy log says whether new settings took. Never |
| 453 | * fails the deploy. |
| 454 | */ |
| 455 | async function reportContainers(unit, cwd, out) { |
| 456 | try { |
| 457 | const listed = await wrangler(["containers", "list"], { cwd }); |
| 458 | const text = listed.out.slice(listed.out.indexOf("[")); |
| 459 | const apps = JSON.parse(text.slice(0, text.lastIndexOf("]") + 1)); |
| 460 | for (const app of apps.filter((a) => String(a.name ?? "").startsWith(`${unit.worker}-`))) { |
| 461 | const c = app.configuration ?? {}; |
| 462 | const disk = c.disk?.size_mb ?? c.disk_mb ?? c.disk?.size; |
| 463 | const size = [c.instance_type, c.vcpu != null && `${c.vcpu} vCPU`, c.memory_mib != null && `${c.memory_mib} MiB`, disk != null && `${disk} MB disk`] |
| 464 | .filter(Boolean) |
| 465 | .join(", "); |
| 466 | const rollout = app.active_rollout_id ? `rollout ${app.active_rollout_id} in progress` : "no rollout in progress"; |
| 467 | out.line( |
| 468 | `container ${app.name}: ${size || JSON.stringify(c).slice(0, 200)}; version ${app.version ?? "?"}; ${app.instances ?? "?"} instances; ${rollout}`, |
| 469 | ); |
| 470 | } |
| 471 | } catch (error) { |
| 472 | out.line(`could not read the container applications as JSON (${error.message ?? error}); wrangler said:`); |
| 473 | const said = await wrangler(["containers", "list"], { cwd }).catch(() => ({ out: "" })); |
| 474 | for (const line of said.out |
| 475 | .split("\n") |
| 476 | .filter((l) => l.trim()) |
| 477 | .slice(0, 40)) |
| 478 | out.line(` ${line}`); |
| 479 | } |
| 480 | } |
| 481 | |
| 482 | async function migrate(stack, units, migrations, opts) { |
| 483 | const due = units.filter((u) => migrations[u.id]?.pending?.length); |
| 484 | const broken = units.filter((u) => migrations[u.id]?.error); |
| 485 | if (broken.length) { |
| 486 | throw new Error(`Could not read pending migrations for ${broken.map((u) => u.id).join(", ")}; nothing was deployed.`); |
| 487 | } |
| 488 | if (!due.length) return []; |
| 489 | log(`== migrations: ${due.map((u) => `${u.d1.database} (${migrations[u.id].pending.length})`).join(", ")}`); |
| 490 | const results = await pool(due, opts.concurrency, async (unit) => { |
| 491 | const started = Date.now(); |
| 492 | const out = unitLogger(`${unit.id}-migrations`); |
| 493 | const applied = await applyMigrations(unit, out.line); |
| 494 | out.save(); |
| 495 | return { |
| 496 | unit: `${unit.id} (D1 ${unit.d1.database})`, |
| 497 | stage: "migrations", |
| 498 | ok: applied.code === 0, |
| 499 | version: `${migrations[unit.id].pending.length} applied`, |
| 500 | ms: Date.now() - started, |
| 501 | note: applied.code === 0 ? "" : `${lastLines(applied.out)}\n full log: ${out.file}`, |
| 502 | }; |
| 503 | }); |
| 504 | return results; |
| 505 | } |
| 506 | |
| 507 | function summary(results) { |
| 508 | const rows = results.map((r) => [ |
| 509 | r.unit, |
| 510 | r.stage, |
| 511 | r.ok ? "ok" : r.skipped ? "not started" : "FAILED", |
| 512 | r.version ?? "", |
| 513 | r.ms ? seconds(r.ms) : "", |
| 514 | r.note.split("\n")[0], |
| 515 | ]); |
| 516 | console.log(`\n${table(rows, ["unit", "stage", "result", "version", "time", "note"])}`); |
| 517 | for (const r of results.filter((r) => !r.ok && !r.skipped)) console.log(`\n${r.unit}: ${r.note}`); |
| 518 | } |
| 519 | |
| 520 | async function deploy(stack, opts, { dryRun = false } = {}) { |
| 521 | const started = Date.now(); |
| 522 | // A build reads nothing from Cloudflare: it builds what it is given. |
| 523 | const p = dryRun ? buildPlan(stack, opts) : await plan(stack, opts); |
| 524 | if (!dryRun) printPlan(stack, p); |
| 525 | const deploying = p.decisions.filter((d) => d.deploy); |
| 526 | const touched = deploying.map((d) => d.unit); |
| 527 | const head = p.head; |
| 528 | |
| 529 | // A deploy names the commit it came from, so a dirty tree would be |
| 530 | // recorded as something it is not. |
| 531 | // A prebuilt unit's outputs are untracked files the build job left, not |
| 532 | // changes to what is deployed. |
| 533 | const built = opts.prebuilt ? touched.flatMap(buildOutputs) : []; |
| 534 | const dirtyFiles = git.dirty().filter((file) => !built.some((dir) => file.startsWith(`${dir}/`))); |
| 535 | if (opts.prebuilt && !dryRun) { |
| 536 | const missing = built.filter((dir) => !existsSync(join(ROOT, dir))); |
| 537 | if (missing.length) |
| 538 | throw new Error(`--prebuilt, but nothing was built at ${missing.join(", ")}: run \`node scripts/deploy.mjs build\` for these units first.`); |
| 539 | // Each Rust Worker's build command (scripts/build-rust-worker.mjs) reads this and builds nothing. |
| 540 | process.env.G1T_PREBUILT = "1"; |
| 541 | } |
| 542 | const dirty = deploying.some((d) => |
| 543 | dirtyFiles.some((file) => file.startsWith(`${d.unit.path}/`) || d.unit.dependsOn.some((dir) => file.startsWith(`${dir}/`)) || d.unit.inputs.includes(file)), |
| 544 | ); |
| 545 | if (dirty && !dryRun && !opts.allowDirty) { |
| 546 | throw new Error( |
| 547 | "Uncommitted changes touch what would deploy. Commit them, or pass --allow-dirty (the deploy then records no commit, and the next plan deploys it again).", |
| 548 | ); |
| 549 | } |
| 550 | |
| 551 | const results = []; |
| 552 | if (!dryRun && !opts.noMigrations) { |
| 553 | const migrated = await migrate(stack, p.units, p.migrations, opts); |
| 554 | results.push(...migrated); |
| 555 | if (migrated.some((r) => !r.ok)) { |
| 556 | summary(results); |
| 557 | return false; |
| 558 | } |
| 559 | } |
| 560 | if (!touched.length) { |
| 561 | if (results.length) summary(results); |
| 562 | return true; |
| 563 | } |
| 564 | |
| 565 | if (!opts.prebuilt && touched.some((u) => u.kind === "rust-worker")) ensureWorkerBuild(); |
| 566 | const docker = once(dockerAvailable); |
| 567 | const context = { |
| 568 | head, |
| 569 | subject: git.subject(), |
| 570 | dirty, |
| 571 | dryRun, |
| 572 | docker, |
| 573 | rebuildImage: opts.rebuildImage, |
| 574 | rebuildBase: opts.rebuildBase, |
| 575 | prebuilt: Boolean(opts.prebuilt), |
| 576 | urgent: Boolean(opts.urgent), |
| 577 | }; |
| 578 | // A deploy run by hand shows on the repository's Deployments page; a |
| 579 | // dirty tree is not a commit anyone can look at, so it is left out. |
| 580 | const settle = dryRun || dirty ? null : await reportDeployment({ head, subject: context.subject, units: touched.map((u) => u.id), log }); |
| 581 | |
| 582 | // status.g1t.sh hears the deploy start and finish (STATUS_DEPLOY_TOKEN; |
| 583 | // nothing without it), so the restarts it causes are not drafted as |
| 584 | // incidents. Never in a dry run, and it never fails a deploy. |
| 585 | let failed = false; |
| 586 | await withDeployWindow( |
| 587 | async () => { |
| 588 | for (const { stage, units } of byStage(stack, touched)) { |
| 589 | if (failed) { |
| 590 | for (const unit of units) results.push({ unit: unit.id, stage, ok: false, skipped: true, ms: 0, note: "an earlier stage failed" }); |
| 591 | continue; |
| 592 | } |
| 593 | // A Worker that does not exist yet goes before those bound to it. |
| 594 | for (const wave of waves(units, (unit) => Boolean(deploying.find((d) => d.unit === unit)?.missing))) { |
| 595 | if (failed) { |
| 596 | for (const unit of wave) results.push({ unit: unit.id, stage, ok: false, skipped: true, ms: 0, note: "a Worker it binds to failed to deploy" }); |
| 597 | continue; |
| 598 | } |
| 599 | log(`== ${stage}: ${wave.map((u) => u.id).join(", ")}`); |
| 600 | const shipped = await pool(wave, opts.concurrency, (unit) => |
| 601 | ship( |
| 602 | unit, |
| 603 | deploying.find((d) => d.unit === unit), |
| 604 | context, |
| 605 | ), |
| 606 | ); |
| 607 | results.push(...shipped); |
| 608 | failed = shipped.some((r) => !r.ok); |
| 609 | } |
| 610 | } |
| 611 | }, |
| 612 | { id: head ?? null, dryRun, log }, |
| 613 | ); |
| 614 | summary(results); |
| 615 | await settle?.(!failed); |
| 616 | console.log(`\n${failed ? "Failed" : dryRun ? "Built" : "Deployed"} in ${seconds(Date.now() - started)}.`); |
| 617 | return !failed; |
| 618 | } |
| 619 | |
| 620 | async function doctor(stack, opts) { |
| 621 | const units = selected(stack, opts); |
| 622 | const rows = await pool(units, 8, async (unit) => { |
| 623 | if (!unit.secrets.length) return [unit.id, "", "", ""]; |
| 624 | const found = await wrangler(["secret", "list", "--name", unit.worker, "--format", "json"], { cwd: join(ROOT, unit.path) }); |
| 625 | if (found.code !== 0) return [unit.id, unit.secrets.join(" "), "?", "could not read"]; |
| 626 | const have = new Set(jsonFrom(found.out).map((s) => s.name)); |
| 627 | const missing = unit.secrets.filter((name) => !have.has(name)); |
| 628 | return [unit.id, unit.secrets.join(" "), missing.join(" "), missing.length ? "missing" : "ok"]; |
| 629 | }); |
| 630 | console.log(table(rows, ["unit", "secrets", "missing", "result"])); |
| 631 | return rows.every((r) => r[3] !== "missing"); |
| 632 | } |
| 633 | |
| 634 | async function install(stack, opts) { |
| 635 | const units = selected(stack, opts); |
| 636 | const args = npmCiArgs(units, npmWorkspace()); |
| 637 | log(`npm ${args.join(" ")}`); |
| 638 | // Through npm-ci.mjs, which runs it again when a connection drops. |
| 639 | const done = await exec(`node scripts/deploy/npm-ci.mjs ${args.slice(1).join(" ")}`, [], { cwd: ROOT, shell: true, onLine: (line) => log(line) }); |
| 640 | return done.code === 0; |
| 641 | } |
| 642 | |
| 643 | function manifest(stack, opts) { |
| 644 | const found = problems(stack, findWranglerConfigs()); |
| 645 | if (opts.check) { |
| 646 | for (const problem of found) console.log(`- ${problem}`); |
| 647 | console.log(found.length ? `\n${found.length} problems in deploy/stack.jsonc.` : "deploy/stack.jsonc is consistent with every wrangler.jsonc."); |
| 648 | return !found.length; |
| 649 | } |
| 650 | const out = stack.units.map(({ config, ...unit }) => ({ |
| 651 | ...unit, |
| 652 | queues: { |
| 653 | produces: (config?.queues?.producers ?? []).map((q) => q.queue), |
| 654 | consumes: (config?.queues?.consumers ?? []).map((q) => q.queue), |
| 655 | dead_letter: [...new Set((config?.queues?.consumers ?? []).map((q) => q.dead_letter_queue).filter(Boolean))], |
| 656 | }, |
| 657 | kv: (config?.kv_namespaces ?? []).map((kv) => stack.resources.kv?.[kv.id] ?? kv.id), |
| 658 | r2: (config?.r2_buckets ?? []).map((b) => b.bucket_name), |
| 659 | vectorize: (config?.vectorize ?? []).map((v) => v.index_name), |
| 660 | dispatch_namespaces: (config?.dispatch_namespaces ?? []).map((d) => d.namespace), |
| 661 | routes: (config?.routes ?? []).map((r) => r.pattern), |
| 662 | })); |
| 663 | if (opts.json) console.log(JSON.stringify({ stages: stack.stages, units: out }, null, 2)); |
| 664 | else |
| 665 | console.log( |
| 666 | table( |
| 667 | out.map((u) => [u.id, u.stage, u.kind, u.worker, u.d1?.database ?? "", u.dependsOn.join(" ")]), |
| 668 | ["unit", "stage", "kind", "worker", "d1", "built from (besides its folder)"], |
| 669 | ), |
| 670 | ); |
| 671 | return true; |
| 672 | } |
| 673 | |
| 674 | /** The unit with a Containers image (the runner), or the one named. */ |
| 675 | function imageUnit(stack, opts) { |
| 676 | const units = (opts.only.length ? pick(stack, opts.only) : stack.units).filter((u) => u.image?.base); |
| 677 | if (units.length !== 1) throw new Error("Name the unit with a Containers image: --only runner"); |
| 678 | return units[0]; |
| 679 | } |
| 680 | |
| 681 | async function main() { |
| 682 | const opts = parseArgs(process.argv.slice(2)); |
| 683 | const stack = resolvedStack(); |
| 684 | switch (opts.command) { |
| 685 | case "plan": { |
| 686 | const p = await plan(stack, opts); |
| 687 | const data = planJson(stack, p.decisions, p.migrations, p.head); |
| 688 | if (opts.out) writeFileSync(opts.out, `${JSON.stringify(data)}\n`); |
| 689 | if (opts.githubOutput) writeGithubOutputs(data, p); |
| 690 | if (opts.json) console.log(JSON.stringify(data, null, 2)); |
| 691 | else if (!opts.githubOutput || process.env.GITHUB_OUTPUT) printPlan(stack, p); |
| 692 | return true; |
| 693 | } |
| 694 | case "deploy": |
| 695 | return deploy(stack, opts); |
| 696 | case "build": |
| 697 | return deploy(stack, { ...opts, force: true }, { dryRun: true }); |
| 698 | case "outputs": |
| 699 | console.log(selected(stack, opts).flatMap(buildOutputs).join("\n")); |
| 700 | return true; |
| 701 | case "migrate": { |
| 702 | const units = selected(stack, opts); |
| 703 | const { migrations } = await survey( |
| 704 | units.filter((u) => u.d1), |
| 705 | { ...opts, noMigrations: false, noLive: true }, |
| 706 | ); |
| 707 | const results = await migrate(stack, units, migrations, opts); |
| 708 | if (results.length) summary(results); |
| 709 | else console.log("No migrations to apply."); |
| 710 | return results.every((r) => r.ok); |
| 711 | } |
| 712 | case "manifest": |
| 713 | return manifest(stack, opts); |
| 714 | case "doctor": |
| 715 | return doctor(stack, opts); |
| 716 | case "install": |
| 717 | return install(stack, opts); |
| 718 | case "build-base": { |
| 719 | const unit = imageUnit(stack, opts); |
| 720 | if (!(await dockerAvailable())) throw new Error("build-base needs Docker."); |
| 721 | const lock = await newBase(unit, { push: !opts.noPush, noCache: opts.noCache }); |
| 722 | console.log(JSON.stringify(lock, null, 2)); |
| 723 | if (lock.pushed) console.log(`\nCommit ${unit.image.base.lock}: the next deploy builds the runner's image on this base.`); |
| 724 | return true; |
| 725 | } |
| 726 | case "image": { |
| 727 | const unit = imageUnit(stack, opts); |
| 728 | const out = unitLogger(`${unit.id}-image`); |
| 729 | const docker = once(dockerAvailable); |
| 730 | const image = await runnerImage(unit, { dryRun: Boolean(opts.noPush), docker, rebuildImage: opts.rebuildImage, rebuildBase: opts.rebuildBase }, out); |
| 731 | out.save(); |
| 732 | console.log(`${image.ref}\n${image.note}`); |
| 733 | return true; |
| 734 | } |
| 735 | default: |
| 736 | console.error(USAGE); |
| 737 | return false; |
| 738 | } |
| 739 | } |
| 740 | |
| 741 | main().then( |
| 742 | (ok) => process.exit(ok ? 0 : 1), |
| 743 | (error) => { |
| 744 | console.error(`\n${error.message ?? error}`); |
| 745 | process.exit(1); |
| 746 | }, |
| 747 | ); |