Sign in

flagon-io/g1t

public

g1t/scripts/ops/restore-to-gitstore.mjs

451 lines21,168 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a2013627e5ea4ab13'1#!/usr/bin/env node
2// Rebuilds repositories from the nightly backups into a git store, the cold
3// fallback for an Artifacts outage (docs/ARTIFACTS.md, R12), and afterwards
4// sends back what was pushed to it while it served.
5//
6// The store is deploy/self-host/gitstore: bare repositories under a root,
7// one directory per Artifacts namespace, `<root>/<namespace>/<name>.git`.
8// The repos service reads them through GIT_FALLBACK_URL once a namespace
9// is switched to it (services/repos/src/fallback.rs).
10//
11// node scripts/ops/restore-to-gitstore.mjs index > index.json
12// node scripts/ops/restore-to-gitstore.mjs restore --into /srv/gitstore --bundles /srv/backups
13// node scripts/ops/restore-to-gitstore.mjs restore --gitstore https://gitstore.example # GITSTORE_SECRET
14// node scripts/ops/restore-to-gitstore.mjs changed --into /srv/gitstore
15// node scripts/ops/restore-to-gitstore.mjs reconcile --into /srv/gitstore
16//
17// Commands:
18// index Every repository with a backup, its id and store key, from
19// the g1t-repos database (read-only), as JSON. Keep a recent
20// one on the fallback host: `restore --index` needs no database.
21// restore Each repository's chain, verified as the restore drill
22// verifies it (scripts/ops/backup-restore-drill.mjs), into the
23// store. One already restored from the same last backup is
24// left alone, so a second run only does what changed.
25// changed The restored repositories whose refs moved since they were
26// restored: pushes the fallback took (GIT_FALLBACK_WRITES=allow).
27// reconcile Sends those back to Artifacts. A ref that Artifacts still has
28// as it was backed up is moved to what the fallback has; one
29// that moved on both sides is kept beside it, as
30// refs/fallback/<the rest of its name>, for its owners to merge.
31// Then each one's refs_version is moved in the database, so
32// nothing kept from before is served.
33//
34// Options:
35// --into <root> the git store's root on this machine (GITSTORE_ROOT).
36// --gitstore <url> a git store reached over HTTP instead, with
37// GITSTORE_SECRET; it must not be read-only while
38// restoring. `changed` and `reconcile` need --into.
39// --bundles <dir> read the bucket from a local copy (`backups/<id>/...`,
40// as `rclone copy r2:g1t-backups <dir>` leaves it);
41// without it each object is read through Wrangler.
42// --index <file> the repositories from `index`'s output, not the database.
43// --namespace <name> only repositories in this Artifacts namespace.
44// --repo <id> only this repository (repeatable).
45// --default-namespace the namespace keys without one are in (default g1t).
46// --jobs <n> repositories at once (default 4).
47// --live-root <dir> reconcile into bare repositories here
48// (`<dir>/<namespace>/<name>.git`), for drills and tests,
49// instead of Artifacts.
50// --no-bump reconcile without moving refs_version.
51//
52// Artifacts is reached for `reconcile` with CLOUDFLARE_API_TOKEN (Artifacts
53// edit), or CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL; the database and the
54// bucket through Wrangler, as the restore drill does.
55
56import { randomUUID } from "node:crypto";
57import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
58import { mkdtemp } from "node:fs/promises";
59import { tmpdir } from "node:os";
60import { dirname, join } from "node:path";
61
62import { cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
63import { ROOT } from "../deploy/stack.mjs";
64import { compareRefs, parseRefs, readManifest, restore } from "./backup-restore-drill.mjs";
65
66const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
67const DATABASE = "g1t-repos";
68const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups";
69const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "1e6f2cffa3f445920836e8ebe446bb58";
70const NAME = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,199}$/;
71/** Where refs that moved on both sides are kept. */
72export const CONFLICT_PREFIX = "refs/fallback/";
73
74async function git(args, { cwd, input } = {}) {
75 const { code, out } = await exec("git", args, { cwd, input });
76 if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`);
77 return out.trim();
78}
79
80/** A store key's Artifacts namespace and name: `g1t-us-1/acme--rocket`, or the default's. */
81export function locate(store, defaultNamespace = "g1t") {
82 const at = store.indexOf("/");
83 const [namespace, name] = at >= 0 ? [store.slice(0, at), store.slice(at + 1)] : [defaultNamespace, store];
84 if (!NAME.test(namespace) || !NAME.test(name)) throw new Error(`not a store key: ${store}`);
85 return { namespace, name };
86}
87
88/** Where a repository lives under the git store's root. */
89export function repoDir(root, namespace, name) {
90 return join(root, namespace, `${name}.git`);
91}
92
93/** What the git store keeps beside a repository (gitstore/server.mjs `g1t.json`), with what it was restored from. */
94export function metaFor(target, manifest, now = new Date().toISOString(), existing = {}) {
95 const last = manifest.chain.at(-1);
96 return {
97 id: existing.id ?? randomUUID(),
98 description: existing.description ?? null,
99 createdAt: existing.createdAt ?? now,
100 readOnly: false,
101 source: null,
102 restored: {
103 repo_id: target.id,
104 entry: last.id,
105 backed_up_at: last.created_at,
106 restored_at: now,
107 refs: Object.fromEntries(Object.entries(last.refs).filter(([ref]) => ref !== "HEAD")),
108 },
109 };
110}
111
112function readMeta(dir) {
113 try {
114 return JSON.parse(readFileSync(join(dir, "g1t.json"), "utf8"));
115 } catch {
116 return {};
117 }
118}
119
120/** As the git store configures a repository it makes. */
121async function configure(dir) {
122 await git(["config", "http.receivepack", "true"], { cwd: dir });
123 await git(["config", "receive.denyNonFastForwards", "false"], { cwd: dir });
124 await git(["config", "uploadpack.allowAnySHA1InWant", "true"], { cwd: dir });
125}
126
127/** Every ref of a bare repository but HEAD. */
128async function refsIn(dir) {
129 return parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir }));
130}
131
132/**
133 * What changed in a restored repository since it was restored: the refs
134 * pushed to or deleted from it, each with the restored value (`base`) and
135 * what it has now (`now`), `null` for absent.
136 */
137export function changedSince(restoredRefs, current) {
138 return compareRefs(restoredRefs, current).map(({ ref, want, have }) => ({ ref, base: want, now: have }));
139}
140
141/**
142 * How each changed ref goes back to the live repository, which has `live`
143 * now: `move` (the live ref is still what was backed up, so it takes the
144 * fallback's value, or is deleted), `same` (it has it already), or
145 * `conflict` (it moved too: the fallback's value goes beside it, under
146 * CONFLICT_PREFIX).
147 */
148export function reconcilePlan(changes, live) {
149 return changes.map(({ ref, base, now }) => {
150 const current = live[ref] ?? null;
151 if (current === now) return { ref, action: "same", from: current, to: now };
152 if (current === base) return { ref, action: "move", from: current, to: now };
153 return { ref, action: "conflict", from: current, to: now, kept: now ? CONFLICT_PREFIX + ref.replace(/^refs\//, "") : null };
154 });
155}
156
157// ---------------------------------------------------------------------
158
159async function d1(sql) {
160 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
161 cwd: join(ROOT, "services/repos"),
162 env: wranglerEnv(),
163 });
164 if (code !== 0) throw new Error(out.slice(-600));
165 return jsonFrom(out)[0]?.results ?? [];
166}
167
168const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
169
170/** Every live repository with a backup: id, store key, path. */
171async function indexFromDatabase() {
172 const rows = await d1(`SELECT r.id, coalesce(r.store, r.namespace || '--' || r.name) AS store, r.namespace AS workspace, r.name,
173 r.default_branch, b.last_entry
174 FROM repos r JOIN repo_backups b ON b.repo_id = r.id
175 WHERE r.deleted_at IS NULL AND r.fork_of IS NULL AND b.last_entry IS NOT NULL
176 ORDER BY r.id`);
177 return rows.map((row) => ({ id: row.id, store: row.store, path: `${row.workspace}/${row.name}`, default_branch: row.default_branch }));
178}
179
180/** Without a database: what each manifest in a local copy of the bucket says. */
181function indexFromBundles(bundles) {
182 const base = join(bundles, "backups");
183 if (!existsSync(base)) return [];
184 return readdirSync(base)
185 .filter((id) => existsSync(join(base, id, "manifest.json")))
186 .map((id) => {
187 const manifest = JSON.parse(readFileSync(join(base, id, "manifest.json"), "utf8"));
188 const path = manifest.path ? `${manifest.path.namespace}/${manifest.path.name}` : null;
189 return { id, store: manifest.store_key, path };
190 });
191}
192
193function bucketReader(localCopy) {
194 if (localCopy) return async (key) => join(localCopy, key);
195 return async (key, file) => {
196 const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], {
197 env: wranglerEnv(),
198 });
199 if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
200 return file;
201 };
202}
203
204/** Runs `work` over `items`, `jobs` at a time. */
205async function pool(items, jobs, work) {
206 const results = [];
207 let next = 0;
208 const lanes = Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, async () => {
209 while (next < items.length) {
210 const at = next++;
211 results[at] = await work(items[at]);
212 }
213 });
214 await Promise.all(lanes);
215 return results;
216}
217
218/** The git store's API, over HTTP. */
219function gitstoreApi(url, secret) {
220 const base = url.replace(/\/$/, "");
221 return async (method, path, body) => {
222 const response = await fetch(`${base}/api/repos${path}`, {
223 method,
224 headers: { "x-gitstore-secret": secret, ...(body ? { "content-type": "application/json" } : {}) },
225 body: body ? JSON.stringify(body) : undefined,
226 });
227 const json = await response.json().catch(() => ({}));
228 return { status: response.status, json };
229 };
230}
231
232/**
233 * Restores one repository. Returns what happened: `restored`, `current`
234 * (already restored from the same last backup), or `missing` (no backup).
235 */
236export async function restoreOne(target, { read, into, gitstore, defaultNamespace = "g1t", work }) {
237 const { namespace, name } = locate(target.store, defaultNamespace);
238 const scratch = await mkdtemp(join(work ?? tmpdir(), "g1t-restore-"));
239 try {
240 let manifestFile;
241 try {
242 manifestFile = await read(`backups/${target.id}/manifest.json`, join(scratch, "manifest.json"));
243 } catch {
244 return { id: target.id, namespace, name, result: "missing" };
245 }
246 if (!existsSync(manifestFile)) return { id: target.id, namespace, name, result: "missing" };
247 const manifest = readManifest(readFileSync(manifestFile, "utf8"));
248 const last = manifest.chain.at(-1);
249 if (into) {
250 const dir = repoDir(into, namespace, name);
251 const existing = readMeta(dir);
252 if (existing.restored?.entry === last.id && existing.restored?.repo_id === target.id) {
253 return { id: target.id, namespace, name, result: "current" };
254 }
255 // Built beside it, then put in place, so a reader never sees half.
256 const building = `${dir}.restoring-${process.pid}`;
257 rmSync(building, { recursive: true, force: true });
258 mkdirSync(dirname(dir), { recursive: true });
259 const refs = await restore(manifest, read, building, scratch);
260 await configure(building);
261 writeFileSync(join(building, "g1t.json"), JSON.stringify(metaFor(target, manifest, undefined, existing), null, 2));
262 rmSync(dir, { recursive: true, force: true });
263 renameSync(building, dir);
264 return { id: target.id, namespace, name, result: "restored", refs: Object.keys(refs).length };
265 }
266 // Over HTTP: rebuilt here, then pushed as a mirror.
267 const local = join(scratch, "restored.git");
268 const refs = await restore(manifest, read, local, scratch);
269 const key = `${namespace}/${name}`;
270 const made = await gitstore.api("POST", "", { name: key, defaultBranch: target.default_branch ?? "main" });
271 if (made.status !== 200 && made.json.code !== "ALREADY_EXISTS") throw new Error(`${key}: ${made.json.message ?? made.status}`);
272 const token = await gitstore.api("POST", `/${encodeURIComponent(key)}/tokens`, { scope: "write", ttl: 3600 });
273 if (token.status !== 200) throw new Error(`${key}: ${token.json.message ?? token.status}`);
274 const remote = `${gitstore.url.replace(/\/$/, "")}/git/${key}.git`;
275 await git(["-c", `http.extraHeader=Authorization: Bearer ${token.json.plaintext}`, "push", "--quiet", "--mirror", remote], { cwd: local });
276 return { id: target.id, namespace, name, result: "restored", refs: Object.keys(refs).length };
277 } finally {
278 rmSync(scratch, { recursive: true, force: true });
279 }
280}
281
282/** The restored repositories under `root`, with what each was restored from. */
283function restoredUnder(root) {
284 const out = [];
285 if (!existsSync(root)) return out;
286 for (const namespace of readdirSync(root)) {
287 const dir = join(root, namespace);
288 if (!NAME.test(namespace) || !existsSync(dir)) continue;
289 for (const entry of readdirSync(dir)) {
290 if (!entry.endsWith(".git")) continue;
291 const meta = readMeta(join(dir, entry));
292 if (meta.restored) out.push({ namespace, name: entry.slice(0, -4), dir: join(dir, entry), restored: meta.restored });
293 }
294 }
295 return out;
296}
297
298/** Repositories that took pushes since they were restored. */
299export async function changedRepos(root) {
300 const out = [];
301 for (const repo of restoredUnder(root)) {
302 const changes = changedSince(repo.restored.refs, await refsIn(repo.dir));
303 if (changes.length) out.push({ ...repo, changes });
304 }
305 return out;
306}
307
308/** Where to push a repository back to: Artifacts, or a bare repository under `--live-root`. */
309function liveRemote(liveRoot) {
310 if (liveRoot) return async (namespace, name) => ({ url: repoDir(liveRoot, namespace, name), args: [] });
311 const auth = cloudflareAuth();
312 if (!auth) throw new Error("set CLOUDFLARE_API_TOKEN (Artifacts edit), or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL, or --live-root");
313 const api = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/artifacts/namespaces`;
314 return async (namespace, name) => {
315 const at = `${api}/${namespace}/repos/${encodeURIComponent(name)}`;
316 const info = await (await fetch(at, { headers: auth })).json().catch(() => ({}));
317 const minted = await (
318 await fetch(`${at}/tokens`, { method: "POST", headers: { ...auth, "content-type": "application/json" }, body: JSON.stringify({ scope: "write", ttl: 3600 }) })
319 )
320 .json()
321 .catch(() => ({}));
322 const token = minted.result?.plaintext ?? minted.result?.token;
323 if (!token) throw new Error(`${namespace}/${name}: Artifacts gave no write token (${JSON.stringify(minted.errors ?? minted).slice(0, 300)})`);
324 const url = info.result?.remote ?? `https://${ACCOUNT_ID}.artifacts.cloudflare.net/git/${namespace}/${name}.git`;
325 // The token as Artifacts gives it, `?expires=` and all, as g1t sends it.
326 return { url, args: ["-c", `http.extraHeader=Authorization: Bearer ${token}`] };
327 };
328}
329
330/** Sends one repository's changes back; what was done with each ref. */
331export async function reconcileOne(repo, remoteFor) {
332 const { url, args } = await remoteFor(repo.namespace, repo.name);
333 const live = parseRefs(await git([...args, "ls-remote", url], { cwd: repo.dir }));
334 const plan = reconcilePlan(repo.changes, live);
335 const specs = [];
336 for (const step of plan) {
337 if (step.action === "move") {
338 const lease = `--force-with-lease=${step.ref}:${step.from ?? ""}`;
339 specs.push({ lease, spec: step.to ? `+${step.to}:${step.ref}` : `:${step.ref}` });
340 } else if (step.action === "conflict" && step.kept) {
341 specs.push({ lease: null, spec: `+${step.to}:${step.kept}` });
342 }
343 }
344 if (specs.length) {
345 // Not --atomic: whether Artifacts takes it is not documented (docs/ARTIFACTS.md, Q6).
346 // Each move is leased on the value read above, so one that moved since is refused, not overwritten.
347 const leases = specs.map((one) => one.lease).filter(Boolean);
348 await git([...args, "push", "--quiet", ...leases, url, ...specs.map((one) => one.spec)], { cwd: repo.dir });
349 }
350 return plan;
351}
352
353async function main() {
354 const argv = process.argv.slice(2);
355 const command = argv[0];
356 const option = (name) => {
357 const at = argv.indexOf(name);
358 return at >= 0 ? argv[at + 1] : undefined;
359 };
360 const many = (name) => argv.flatMap((arg, at) => (arg === name && argv[at + 1] ? [argv[at + 1]] : []));
361 const defaultNamespace = option("--default-namespace") ?? "g1t";
362 const into = option("--into");
363
364 if (command === "index") {
365 console.log(JSON.stringify(await indexFromDatabase(), null, 2));
366 return 0;
367 }
368
369 if (command === "restore") {
370 const url = option("--gitstore");
371 if (!into && !url) throw new Error("say where to restore to: --into <root> or --gitstore <url>");
372 const gitstore = url ? { url, api: gitstoreApi(url, process.env.GITSTORE_SECRET ?? "") } : null;
373 const bundles = option("--bundles");
374 let targets = option("--index")
375 ? JSON.parse(readFileSync(option("--index"), "utf8"))
376 : bundles && argv.includes("--offline")
377 ? indexFromBundles(bundles)
378 : await indexFromDatabase();
379 const only = many("--repo");
380 if (only.length) targets = targets.filter((target) => only.includes(target.id));
381 const namespace = option("--namespace");
382 if (namespace) targets = targets.filter((target) => locate(target.store, defaultNamespace).namespace === namespace);
383 const read = bucketReader(bundles);
384 const started = Date.now();
385 const counts = { restored: 0, current: 0, missing: 0, failed: 0 };
386 await pool(targets, Number(option("--jobs") ?? 4), async (target) => {
387 try {
388 const done = await restoreOne(target, { read, into, gitstore, defaultNamespace });
389 counts[done.result] += 1;
390 if (done.result !== "current") console.log(`${done.result.padEnd(8)} ${done.namespace}/${done.name} (${target.path ?? target.id})`);
391 } catch (error) {
392 counts.failed += 1;
393 console.error(`failed ${target.store} (${target.id}): ${error.message}`);
394 }
395 });
396 const seconds = ((Date.now() - started) / 1000).toFixed(0);
397 console.log(
398 `${targets.length} repositories in ${seconds}s: ${counts.restored} restored, ${counts.current} already current, ${counts.missing} without a backup, ${counts.failed} failed`,
399 );
400 return counts.failed ? 1 : 0;
401 }
402
403 if (command === "changed" || command === "reconcile") {
404 if (!into) throw new Error(`${command} reads the git store's root: --into <root>`);
405 const changed = await changedRepos(into);
406 if (command === "changed") {
407 for (const repo of changed) {
408 console.log(`${repo.namespace}/${repo.name} (${repo.restored.repo_id})`);
409 for (const { ref, base, now } of repo.changes) console.log(` ${ref}: ${base ?? "(none)"} -> ${now ?? "(deleted)"}`);
410 }
411 console.log(`${changed.length} repositories took pushes since they were restored`);
412 return 0;
413 }
414 const remoteFor = liveRemote(option("--live-root"));
415 const bumped = [];
416 let conflicts = 0;
417 let failed = 0;
418 for (const repo of changed) {
419 try {
420 const plan = await reconcileOne(repo, remoteFor);
421 bumped.push(repo.restored.repo_id);
422 for (const step of plan) {
423 if (step.action === "conflict") conflicts += 1;
424 const what = step.action === "conflict" ? `moved on both sides; the fallback's kept as ${step.kept ?? "(it deleted it)"}` : step.action;
425 console.log(`${repo.namespace}/${repo.name} ${step.ref}: ${what}`);
426 }
427 } catch (error) {
428 failed += 1;
429 console.error(`${repo.namespace}/${repo.name}: ${error.message}`);
430 }
431 }
432 if (bumped.length && !argv.includes("--no-bump")) {
433 await d1(`UPDATE repos SET refs_version = coalesce(refs_version, 0) + 1 WHERE id IN (${bumped.map(quoted).join(", ")})`);
434 }
435 console.log(`${changed.length} repositories reconciled: ${conflicts} refs kept beside a newer one, ${failed} failed`);
436 return failed ? 1 : conflicts ? 3 : 0;
437 }
438
439 console.error("usage: restore-to-gitstore.mjs index | restore | changed | reconcile (see the top of the file)");
440 return 2;
441}
442
443if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/restore-to-gitstore.mjs")) {
444 main().then(
445 (code) => process.exit(code),
446 (error) => {
447 console.error(`restore: ${error.message}`);
448 process.exit(2);
449 },
450 );
451}

This file's history is long; its oldest lines are credited to the oldest commit read.