Sign in

flagon-io/g1t

public

g1t/scripts/deploy.mjs

423 lines18,566 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Deploys as code: a manifest of every Worker, a deploy tool that ships only what changed in parallel stages, and a g1t Actions workflow1#!/usr/bin/env node
2// Deploys g1t to Cloudflare from deploy/stack.jsonc: only what changed since
3// each Worker's live commit, migrations first, then stage by stage.
4// docs/DEPLOYING.md is the guide.
5//
6// node scripts/deploy.mjs plan what would deploy, and why (read-only)
7// node scripts/deploy.mjs deploy migrations, then every changed unit
8// node scripts/deploy.mjs deploy --only web,api just these (if changed; --force: anyway)
9// node scripts/deploy.mjs build --only events build as a deploy would, upload nothing
10// node scripts/deploy.mjs migrate pending D1 migrations only
11// node scripts/deploy.mjs manifest [--check] the resolved manifest, or its problems
12// node scripts/deploy.mjs doctor which units lack their secrets
13// node scripts/deploy.mjs install --only a,b npm ci of just what those units need (CI)
14//
15// Flags: --all (every unit, changed or not), --only a,b, --skip a,b,
16// --force, --concurrency N (default 4), --stage core (one stage),
17// --json (plan), --out FILE (plan), --no-migrations, --allow-dirty,
18// --rebuild-image, --since REV (Workers with no recorded commit are taken
19// to run REV).
20
21import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
22import { tmpdir } from "node:os";
23import { join } from "node:path";
24
25import { ensureWorkerBuild } from "./build-rust-worker.mjs";
26import {
27 annotation,
28 applyMigrations,
29 dockerAvailable,
30 exec,
31 jsonFrom,
32 lastLines,
33 pendingMigrations,
34 readLive,
35 versionFrom,
36 wrangler,
37 wranglerEnv,
38} from "./deploy/cloudflare.mjs";
39import { decide, git, planJson, pool, table } from "./deploy/plan.mjs";
40import { ROOT, byStage, codeStages, findWranglerConfigs, npmCiArgs, npmWorkspace, pick, problems, resolvedStack } from "./deploy/stack.mjs";
41
42const USAGE = "usage: node scripts/deploy.mjs plan|deploy|build|migrate|manifest|doctor [--all] [--only a,b] [--skip a,b] [--force] [--concurrency N] [--stage S] [--json]";
43
44function parseArgs(argv) {
45 const opts = { command: argv[0], only: [], skip: [], concurrency: 4, force: false, all: false, json: false };
46 for (let i = 1; i < argv.length; i++) {
47 const arg = argv[i];
48 const [flag, inline] = arg.split(/=(.*)/s);
49 const value = () => inline ?? argv[++i];
50 if (flag === "--only") opts.only.push(value());
51 else if (flag === "--skip") opts.skip.push(value());
52 else if (flag === "--concurrency") opts.concurrency = Number(value());
53 else if (flag === "--stage") opts.stage = value();
54 else if (flag === "--all") opts.all = true;
55 else if (flag === "--force") opts.force = true;
56 else if (flag === "--json") opts.json = true;
57 else if (flag === "--check") opts.check = true;
58 else if (flag === "--no-migrations") opts.noMigrations = true;
59 else if (flag === "--allow-dirty") opts.allowDirty = true;
60 else if (flag === "--rebuild-image") opts.rebuildImage = true;
61 else if (flag === "--out") opts.out = value();
62 else if (flag === "--since") opts.since = value();
63 else if (flag === "--github-output") opts.githubOutput = true;
64 else throw new Error(`unknown flag ${arg}\n${USAGE}`);
65 }
66 if (!Number.isInteger(opts.concurrency) || opts.concurrency < 1) throw new Error("--concurrency is a whole number, 1 or more");
67 return opts;
68}
69
70/** The units a command works on: --only (or all), less --skip, in --stage. */
71function selected(stack, opts) {
72 let units = opts.only.length ? pick(stack, opts.only) : [...stack.units];
73 const skipped = pick(stack, opts.skip);
74 units = units.filter((u) => !skipped.includes(u));
75 if (opts.stage) {
76 if (!codeStages(stack).includes(opts.stage)) throw new Error(`--stage is one of ${codeStages(stack).join(", ")}`);
77 units = units.filter((u) => u.stage === opts.stage);
78 }
79 // Manifest order, whatever order they were named in.
80 return stack.units.filter((u) => units.includes(u));
81}
82
83const log = (...args) => console.error(...args);
84
85/**
86 * The plan for a workflow (.g1t/workflows/deploy.yml): job outputs in
87 * $GITHUB_OUTPUT, and the plan as a table in $GITHUB_STEP_SUMMARY.
88 */
89function writeGithubOutputs(data, p) {
90 const lines = [
91 `commit=${data.commit}`,
92 `migrate=${data.migrations.length > 0}`,
93 `migrate_units=${data.migrations.map((m) => m.unit).join(",")}`,
94 `deploying=${data.units.filter((u) => u.deploy).map((u) => u.unit).join(",")}`,
95 ];
96 for (const [stage, { jobs }] of Object.entries(data.stages)) {
97 // A matrix needs one entry; an empty stage's job is skipped by its `if`.
98 const include = jobs.length ? jobs : [{ group: "none", units: "" }];
99 lines.push(`has_${stage}=${jobs.length > 0}`, `${stage}=${JSON.stringify({ include })}`);
100 }
101 if (data.migration_errors.length) throw new Error(`Could not read pending migrations: ${data.migration_errors.map((e) => e.unit).join(", ")}`);
102 if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`);
103 else console.log(lines.join("\n"));
104 if (process.env.GITHUB_STEP_SUMMARY) {
105 const rows = p.decisions.map((d) => `| ${d.unit.id} | ${d.unit.stage} | ${d.deploy ? "deploy" : ""} | ${d.since ? d.since.slice(0, 12) : "?"} | ${d.reason.replaceAll("|", "\\|")} |`);
106 const pending = data.migrations.map((m) => `- ${m.database}: ${m.pending.join(", ")}`);
107 appendFileSync(
108 process.env.GITHUB_STEP_SUMMARY,
109 [`## Deploy plan for ${data.commit.slice(0, 12)}`, "", "| unit | stage | action | live | why |", "| --- | --- | --- | --- | --- |", ...rows, "", pending.length ? "### Pending migrations" : "", ...pending, ""].join("\n"),
110 );
111 }
112}
113
114const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
115
116/** Reads what each unit runs and what its database is waiting for. */
117async function survey(units, opts) {
118 const live = {};
119 const migrations = {};
120 const tasks = [
121 ...(opts.noLive ? [] : units).map((unit) => async () => {
122 live[unit.id] = await readLive(unit);
123 }),
124 ...(opts.noMigrations ? [] : units.filter((u) => u.d1)).map((unit) => async () => {
125 migrations[unit.id] = await pendingMigrations(unit);
126 }),
127 ];
128 await pool(tasks, Math.max(8, opts.concurrency * 2), (task) => task());
129 return { live, migrations };
130}
131
132async function plan(stack, opts) {
133 const units = selected(stack, opts);
134 const head = git.head();
135 const { live, migrations } = await survey(units, opts);
136 // --since: what a Worker with no recorded commit is taken to run (once,
137 // to adopt Workers deployed before this tool), for example --since HEAD~3.
138 if (opts.since) {
139 const since = git.resolve(opts.since);
140 for (const unit of units) {
141 const found = live[unit.id];
142 if (found && !found.sha && !found.missing && !found.error) live[unit.id] = { ...found, sha: since, assumed: true };
143 }
144 }
145 const decisions = decide(units, { live, head, force: opts.force || opts.all });
146 return { head, units, live, migrations, decisions };
147}
148
149function buildPlan(stack, opts) {
150 const units = selected(stack, opts);
151 return {
152 head: git.head(),
153 units,
154 live: {},
155 migrations: {},
156 decisions: units.map((unit) => ({ unit, deploy: true, reason: "build", since: null, files: [], image: false })),
157 };
158}
159
160function printPlan(stack, { head, decisions, migrations, live }) {
161 console.log(`Deploying ${head.slice(0, 12)} (${git.subject()})\n`);
162 const rows = decisions.map((d) => [
163 d.unit.id,
164 d.unit.stage,
165 d.deploy ? "deploy" : "-",
166 d.since ? d.since.slice(0, 12) + (live[d.unit.id]?.assumed ? "*" : "") : "?",
167 d.unit.d1 ? (migrations[d.unit.id]?.error ? "error" : String(migrations[d.unit.id]?.pending?.length ?? "-")) : "",
168 d.reason + (d.image ? "; image rebuilds" : ""),
169 ]);
170 console.log(table(rows, ["unit", "stage", "action", "live", "migrations", "why"]));
171 if (decisions.some((d) => live[d.unit.id]?.assumed)) console.log("* taken from --since: no commit was recorded for it");
172 const pending = Object.entries(migrations).filter(([, m]) => m.pending?.length);
173 if (pending.length) {
174 console.log("\nPending migrations:");
175 for (const [id, m] of pending) console.log(` ${stack.units.find((u) => u.id === id).d1.database}: ${m.pending.join(", ")}`);
176 }
177 for (const [id, m] of Object.entries(migrations).filter(([, m]) => m.error)) {
178 console.log(`\nCould not list ${id}'s migrations:\n${m.error}`);
179 }
180 const deploying = decisions.filter((d) => d.deploy).map((d) => d.unit);
181 console.log(
182 deploying.length
183 ? `\n${deploying.length} to deploy: ${byStage(stack, deploying).map((g) => `${g.stage} (${g.units.map((u) => u.id).join(", ")})`).join(" -> ")}`
184 : "\nNothing to deploy.",
185 );
186}
187
188/** Output of one unit, prefixed, to the terminal and a log file. */
189function unitLogger(id) {
190 const dir = join(tmpdir(), "g1t-deploy");
191 mkdirSync(dir, { recursive: true });
192 const file = join(dir, `${id}.log`);
193 const lines = [];
194 return {
195 file,
196 line: (text) => {
197 lines.push(text);
198 if (text.trim()) log(`[${id}] ${text}`);
199 },
200 save: () => writeFileSync(file, `${lines.join("\n")}\n`),
201 };
202}
203
204/** Builds (and unless `dryRun`, deploys) one unit. */
205async function ship(unit, decision, { head, subject, dirty, dryRun, docker, rebuildImage }) {
206 const started = Date.now();
207 const out = unitLogger(unit.id);
208 const cwd = join(ROOT, unit.path);
209 const result = { unit: unit.id, stage: unit.stage, ok: false, version: null, ms: 0, note: "" };
210 try {
211 if (unit.kind === "react-router" || unit.kind === "astro") {
212 const built = await exec("npm", ["run", "build"], { cwd, onLine: out.line, shell: true, env: wranglerEnv() });
213 if (built.code !== 0) throw new Error(`npm run build failed:\n${lastLines(built.out)}`);
214 }
215 const args = ["deploy"];
216 if (dryRun) {
217 args.push("--dry-run", "--outdir", join(tmpdir(), "g1t-deploy", "dist", unit.id));
218 } else {
219 const { message, tag } = annotation(head, subject);
220 args.push("--message", dirty ? message.replace(/^g1t-deploy/, "g1t-deploy-dirty") : message, "--tag", tag);
221 }
222 if (unit.image) {
223 const build = rebuildImage || decision.image;
224 if (!build) {
225 args.push("--containers-rollout", "none");
226 result.note = "image unchanged: not rebuilt";
227 } else if (!docker) {
228 throw new Error(
229 "its Containers image changed, and Docker is not available here. Deploy it from a machine with Docker: node scripts/deploy.mjs deploy --only " +
230 unit.id,
231 );
232 } else {
233 result.note = "image rebuilt";
234 }
235 }
236 const deployed = await wrangler(args, { cwd, onLine: out.line });
237 if (deployed.code !== 0) throw new Error(`wrangler deploy failed:\n${lastLines(deployed.out)}`);
238 result.version = dryRun ? "(dry run)" : versionFrom(deployed.out);
239 result.ok = true;
240 } catch (error) {
241 result.note = String(error.message ?? error);
242 }
243 result.ms = Date.now() - started;
244 out.save();
245 if (!result.ok) result.note += `\n full log: ${out.file}`;
246 return result;
247}
248
249async function migrate(stack, units, migrations, opts) {
250 const due = units.filter((u) => migrations[u.id]?.pending?.length);
251 const broken = units.filter((u) => migrations[u.id]?.error);
252 if (broken.length) {
253 throw new Error(`Could not read pending migrations for ${broken.map((u) => u.id).join(", ")}; nothing was deployed.`);
254 }
255 if (!due.length) return [];
256 log(`== migrations: ${due.map((u) => `${u.d1.database} (${migrations[u.id].pending.length})`).join(", ")}`);
257 const results = await pool(due, opts.concurrency, async (unit) => {
258 const started = Date.now();
259 const out = unitLogger(`${unit.id}-migrations`);
260 const applied = await applyMigrations(unit, out.line);
261 out.save();
262 return {
263 unit: `${unit.id} (D1 ${unit.d1.database})`,
264 stage: "migrations",
265 ok: applied.code === 0,
266 version: `${migrations[unit.id].pending.length} applied`,
267 ms: Date.now() - started,
268 note: applied.code === 0 ? "" : `${lastLines(applied.out)}\n full log: ${out.file}`,
269 };
270 });
271 return results;
272}
273
274function summary(results) {
275 const rows = results.map((r) => [r.unit, r.stage, r.ok ? "ok" : r.skipped ? "not started" : "FAILED", r.version ?? "", r.ms ? seconds(r.ms) : "", r.note.split("\n")[0]]);
276 console.log(`\n${table(rows, ["unit", "stage", "result", "version", "time", "note"])}`);
277 for (const r of results.filter((r) => !r.ok && !r.skipped)) console.log(`\n${r.unit}: ${r.note}`);
278}
279
280async function deploy(stack, opts, { dryRun = false } = {}) {
281 const started = Date.now();
282 // A build reads nothing from Cloudflare: it builds what it is given.
283 const p = dryRun ? buildPlan(stack, opts) : await plan(stack, opts);
284 if (!dryRun) printPlan(stack, p);
285 const deploying = p.decisions.filter((d) => d.deploy);
286 const touched = deploying.map((d) => d.unit);
287 const head = p.head;
288
289 // A deploy names the commit it came from, so a dirty tree would be
290 // recorded as something it is not.
291 const dirtyFiles = git.dirty();
292 const dirty = deploying.some((d) => dirtyFiles.some((file) => file.startsWith(`${d.unit.path}/`) || d.unit.dependsOn.some((dir) => file.startsWith(`${dir}/`)) || d.unit.inputs.includes(file)));
293 if (dirty && !dryRun && !opts.allowDirty) {
294 throw new Error("Uncommitted changes touch what would deploy. Commit them, or pass --allow-dirty (the deploy then records no commit, and the next plan deploys it again).");
295 }
296
297 const results = [];
298 if (!dryRun && !opts.noMigrations) {
299 const migrated = await migrate(stack, p.units, p.migrations, opts);
300 results.push(...migrated);
301 if (migrated.some((r) => !r.ok)) {
302 summary(results);
303 return false;
304 }
305 }
306 if (!touched.length) {
307 if (results.length) summary(results);
308 return true;
309 }
310
311 if (touched.some((u) => u.kind === "rust-worker")) ensureWorkerBuild();
312 const docker = touched.some((u) => u.image) ? await dockerAvailable() : false;
313 const context = { head, subject: git.subject(), dirty, dryRun, docker, rebuildImage: opts.rebuildImage };
314
315 let failed = false;
316 for (const { stage, units } of byStage(stack, touched)) {
317 if (failed) {
318 for (const unit of units) results.push({ unit: unit.id, stage, ok: false, skipped: true, ms: 0, note: "an earlier stage failed" });
319 continue;
320 }
321 log(`== ${stage}: ${units.map((u) => u.id).join(", ")}`);
322 const shipped = await pool(units, opts.concurrency, (unit) => ship(unit, deploying.find((d) => d.unit === unit), context));
323 results.push(...shipped);
324 failed = shipped.some((r) => !r.ok);
325 }
326 summary(results);
327 console.log(`\n${failed ? "Failed" : dryRun ? "Built" : "Deployed"} in ${seconds(Date.now() - started)}.`);
328 return !failed;
329}
330
331async function doctor(stack, opts) {
332 const units = selected(stack, opts);
333 const rows = await pool(units, 8, async (unit) => {
334 if (!unit.secrets.length) return [unit.id, "", "", ""];
335 const found = await wrangler(["secret", "list", "--name", unit.worker, "--format", "json"], { cwd: join(ROOT, unit.path) });
336 if (found.code !== 0) return [unit.id, unit.secrets.join(" "), "?", "could not read"];
337 const have = new Set(jsonFrom(found.out).map((s) => s.name));
338 const missing = unit.secrets.filter((name) => !have.has(name));
339 return [unit.id, unit.secrets.join(" "), missing.join(" "), missing.length ? "missing" : "ok"];
340 });
341 console.log(table(rows, ["unit", "secrets", "missing", "result"]));
342 return rows.every((r) => r[3] !== "missing");
343}
344
345async function install(stack, opts) {
346 const units = selected(stack, opts);
347 const args = npmCiArgs(units, npmWorkspace());
348 log(`npm ${args.join(" ")}`);
349 const done = await exec("npm", args, { cwd: ROOT, shell: true, onLine: (line) => log(line) });
350 return done.code === 0;
351}
352
353function manifest(stack, opts) {
354 const found = problems(stack, findWranglerConfigs());
355 if (opts.check) {
356 for (const problem of found) console.log(`- ${problem}`);
357 console.log(found.length ? `\n${found.length} problems in deploy/stack.jsonc.` : "deploy/stack.jsonc is consistent with every wrangler.jsonc.");
358 return !found.length;
359 }
360 const out = stack.units.map(({ config, ...unit }) => ({
361 ...unit,
362 queues: { produces: (config?.queues?.producers ?? []).map((q) => q.queue), consumes: (config?.queues?.consumers ?? []).map((q) => q.queue) },
363 kv: (config?.kv_namespaces ?? []).map((kv) => stack.resources.kv?.[kv.id] ?? kv.id),
364 r2: (config?.r2_buckets ?? []).map((b) => b.bucket_name),
365 vectorize: (config?.vectorize ?? []).map((v) => v.index_name),
366 dispatch_namespaces: (config?.dispatch_namespaces ?? []).map((d) => d.namespace),
367 routes: (config?.routes ?? []).map((r) => r.pattern),
368 }));
369 if (opts.json) console.log(JSON.stringify({ stages: stack.stages, units: out }, null, 2));
370 else
371 console.log(
372 table(
373 out.map((u) => [u.id, u.stage, u.kind, u.worker, u.d1?.database ?? "", u.dependsOn.join(" ")]),
374 ["unit", "stage", "kind", "worker", "d1", "built from (besides its folder)"],
375 ),
376 );
377 return true;
378}
379
380async function main() {
381 const opts = parseArgs(process.argv.slice(2));
382 const stack = resolvedStack();
383 switch (opts.command) {
384 case "plan": {
385 const p = await plan(stack, opts);
386 const data = planJson(stack, p.decisions, p.migrations, p.head);
387 if (opts.out) writeFileSync(opts.out, `${JSON.stringify(data)}\n`);
388 if (opts.githubOutput) writeGithubOutputs(data, p);
389 if (opts.json) console.log(JSON.stringify(data, null, 2));
390 else if (!opts.githubOutput || process.env.GITHUB_OUTPUT) printPlan(stack, p);
391 return true;
392 }
393 case "deploy":
394 return deploy(stack, opts);
395 case "build":
396 return deploy(stack, { ...opts, force: true }, { dryRun: true });
397 case "migrate": {
398 const units = selected(stack, opts);
399 const { migrations } = await survey(units.filter((u) => u.d1), { ...opts, noMigrations: false, noLive: true });
400 const results = await migrate(stack, units, migrations, opts);
401 if (results.length) summary(results);
402 else console.log("No migrations to apply.");
403 return results.every((r) => r.ok);
404 }
405 case "manifest":
406 return manifest(stack, opts);
407 case "doctor":
408 return doctor(stack, opts);
409 case "install":
410 return install(stack, opts);
411 default:
412 console.error(USAGE);
413 return false;
414 }
415}
416
417main().then(
418 (ok) => process.exit(ok ? 0 : 1),
419 (error) => {
420 console.error(`\n${error.message ?? error}`);
421 process.exit(1);
422 },
423);