Sign in

flagon-io/g1t

public

g1t/scripts/ops/restore-to-gitstore.mjs

451 lines21,168 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Repositories shard across git store namespaces, move between them, and can keep to the EU; a namespace can be served read-only from the self-hosted git store, rebuilt from the nightly backups (#20, #25)1#!/usr/bin/env node
2// Rebuilds repositories from the nightly backups into a git store, the cold
3// fallback for an Artifacts outage (docs/ARTIFACTS.md, R12), and afterwards
4// sends back what was pushed to it while it served.
5//
6// The store is deploy/self-host/gitstore: bare repositories under a root,
7// one directory per Artifacts namespace, `<root>/<namespace>/<name>.git`.
8// The repos service reads them through GIT_FALLBACK_URL once a namespace
9// is switched to it (services/repos/src/fallback.rs).
10//
11// node scripts/ops/restore-to-gitstore.mjs index > index.json
12// node scripts/ops/restore-to-gitstore.mjs restore --into /srv/gitstore --bundles /srv/backups
13// node scripts/ops/restore-to-gitstore.mjs restore --gitstore https://gitstore.example # GITSTORE_SECRET
14// node scripts/ops/restore-to-gitstore.mjs changed --into /srv/gitstore
15// node scripts/ops/restore-to-gitstore.mjs reconcile --into /srv/gitstore
16//
17// Commands:
18// index Every repository with a backup, its id and store key, from
19// the g1t-repos database (read-only), as JSON. Keep a recent
20// one on the fallback host: `restore --index` needs no database.
21// restore Each repository's chain, verified as the restore drill
22// verifies it (scripts/ops/backup-restore-drill.mjs), into the
23// store. One already restored from the same last backup is
24// left alone, so a second run only does what changed.
25// changed The restored repositories whose refs moved since they were
26// restored: pushes the fallback took (GIT_FALLBACK_WRITES=allow).
27// reconcile Sends those back to Artifacts. A ref that Artifacts still has
28// as it was backed up is moved to what the fallback has; one
29// that moved on both sides is kept beside it, as
30// refs/fallback/<the rest of its name>, for its owners to merge.
31// Then each one's refs_version is moved in the database, so
32// nothing kept from before is served.
33//
34// Options:
35// --into <root> the git store's root on this machine (GITSTORE_ROOT).
36// --gitstore <url> a git store reached over HTTP instead, with
37// GITSTORE_SECRET; it must not be read-only while
38// restoring. `changed` and `reconcile` need --into.
39// --bundles <dir> read the bucket from a local copy (`backups/<id>/...`,
40// as `rclone copy r2:g1t-backups <dir>` leaves it);
41// without it each object is read through Wrangler.
42// --index <file> the repositories from `index`'s output, not the database.
43// --namespace <name> only repositories in this Artifacts namespace.
44// --repo <id> only this repository (repeatable).
45// --default-namespace the namespace keys without one are in (default g1t).
46// --jobs <n> repositories at once (default 4).
47// --live-root <dir> reconcile into bare repositories here
48// (`<dir>/<namespace>/<name>.git`), for drills and tests,
49// instead of Artifacts.
50// --no-bump reconcile without moving refs_version.
51//
52// Artifacts is reached for `reconcile` with CLOUDFLARE_API_TOKEN (Artifacts
53// edit), or CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL; the database and the
54// bucket through Wrangler, as the restore drill does.
55
56import { randomUUID } from "node:crypto";
57import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
58import { mkdtemp } from "node:fs/promises";
59import { tmpdir } from "node:os";
60import { dirname, join } from "node:path";
61
62import { cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
63import { ROOT } from "../deploy/stack.mjs";
64import { compareRefs, parseRefs, readManifest, restore } from "./backup-restore-drill.mjs";
65
66const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
67const DATABASE = "g1t-repos";
68const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups";
69const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "1e6f2cffa3f445920836e8ebe446bb58";
70const NAME = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,199}$/;
71/** Where refs that moved on both sides are kept. */
72export const CONFLICT_PREFIX = "refs/fallback/";
73
74async function git(args, { cwd, input } = {}) {
75 const { code, out } = await exec("git", args, { cwd, input });
76 if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`);
77 return out.trim();
78}
79
80/** A store key's Artifacts namespace and name: `g1t-us-1/acme--rocket`, or the default's. */
81export function locate(store, defaultNamespace = "g1t") {
82 const at = store.indexOf("/");
83 const [namespace, name] = at >= 0 ? [store.slice(0, at), store.slice(at + 1)] : [defaultNamespace, store];
84 if (!NAME.test(namespace) || !NAME.test(name)) throw new Error(`not a store key: ${store}`);
85 return { namespace, name };
86}
87
88/** Where a repository lives under the git store's root. */
89export function repoDir(root, namespace, name) {
90 return join(root, namespace, `${name}.git`);
91}
92
93/** What the git store keeps beside a repository (gitstore/server.mjs `g1t.json`), with what it was restored from. */
94export function metaFor(target, manifest, now = new Date().toISOString(), existing = {}) {
95 const last = manifest.chain.at(-1);
96 return {
97 id: existing.id ?? randomUUID(),
98 description: existing.description ?? null,
99 createdAt: existing.createdAt ?? now,
100 readOnly: false,
101 source: null,
102 restored: {
103 repo_id: target.id,
104 entry: last.id,
105 backed_up_at: last.created_at,
106 restored_at: now,
107 refs: Object.fromEntries(Object.entries(last.refs).filter(([ref]) => ref !== "HEAD")),
108 },
109 };
110}
111
112function readMeta(dir) {
113 try {
114 return JSON.parse(readFileSync(join(dir, "g1t.json"), "utf8"));
115 } catch {
116 return {};
117 }
118}
119
120/** As the git store configures a repository it makes. */
121async function configure(dir) {
122 await git(["config", "http.receivepack", "true"], { cwd: dir });
123 await git(["config", "receive.denyNonFastForwards", "false"], { cwd: dir });
124 await git(["config", "uploadpack.allowAnySHA1InWant", "true"], { cwd: dir });
125}
126
127/** Every ref of a bare repository but HEAD. */
128async function refsIn(dir) {
129 return parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir }));
130}
131
132/**
133 * What changed in a restored repository since it was restored: the refs
134 * pushed to or deleted from it, each with the restored value (`base`) and
135 * what it has now (`now`), `null` for absent.
136 */
137export function changedSince(restoredRefs, current) {
138 return compareRefs(restoredRefs, current).map(({ ref, want, have }) => ({ ref, base: want, now: have }));
139}
140
141/**
142 * How each changed ref goes back to the live repository, which has `live`
143 * now: `move` (the live ref is still what was backed up, so it takes the
144 * fallback's value, or is deleted), `same` (it has it already), or
145 * `conflict` (it moved too: the fallback's value goes beside it, under
146 * CONFLICT_PREFIX).
147 */
148export function reconcilePlan(changes, live) {
149 return changes.map(({ ref, base, now }) => {
150 const current = live[ref] ?? null;
151 if (current === now) return { ref, action: "same", from: current, to: now };
152 if (current === base) return { ref, action: "move", from: current, to: now };
153 return { ref, action: "conflict", from: current, to: now, kept: now ? CONFLICT_PREFIX + ref.replace(/^refs\//, "") : null };
154 });
155}
156
157// ---------------------------------------------------------------------
158
159async function d1(sql) {
160 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
161 cwd: join(ROOT, "services/repos"),
162 env: wranglerEnv(),
163 });
164 if (code !== 0) throw new Error(out.slice(-600));
165 return jsonFrom(out)[0]?.results ?? [];
166}
167
168const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
169
170/** Every live repository with a backup: id, store key, path. */
171async function indexFromDatabase() {
172 const rows = await d1(`SELECT r.id, coalesce(r.store, r.namespace || '--' || r.name) AS store, r.namespace AS workspace, r.name,
173 r.default_branch, b.last_entry
174 FROM repos r JOIN repo_backups b ON b.repo_id = r.id
175 WHERE r.deleted_at IS NULL AND r.fork_of IS NULL AND b.last_entry IS NOT NULL
176 ORDER BY r.id`);
177 return rows.map((row) => ({ id: row.id, store: row.store, path: `${row.workspace}/${row.name}`, default_branch: row.default_branch }));
178}
179
180/** Without a database: what each manifest in a local copy of the bucket says. */
181function indexFromBundles(bundles) {
182 const base = join(bundles, "backups");
183 if (!existsSync(base)) return [];
184 return readdirSync(base)
185 .filter((id) => existsSync(join(base, id, "manifest.json")))
186 .map((id) => {
187 const manifest = JSON.parse(readFileSync(join(base, id, "manifest.json"), "utf8"));
188 const path = manifest.path ? `${manifest.path.namespace}/${manifest.path.name}` : null;
189 return { id, store: manifest.store_key, path };
190 });
191}
192
193function bucketReader(localCopy) {
194 if (localCopy) return async (key) => join(localCopy, key);
195 return async (key, file) => {
196 const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], {
197 env: wranglerEnv(),
198 });
199 if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
200 return file;
201 };
202}
203
204/** Runs `work` over `items`, `jobs` at a time. */
205async function pool(items, jobs, work) {
206 const results = [];
207 let next = 0;
208 const lanes = Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, async () => {
209 while (next < items.length) {
210 const at = next++;
211 results[at] = await work(items[at]);
212 }
213 });
214 await Promise.all(lanes);
215 return results;
216}
217
218/** The git store's API, over HTTP. */
219function gitstoreApi(url, secret) {
220 const base = url.replace(/\/$/, "");
221 return async (method, path, body) => {
222 const response = await fetch(`${base}/api/repos${path}`, {
223 method,
224 headers: { "x-gitstore-secret": secret, ...(body ? { "content-type": "application/json" } : {}) },
225 body: body ? JSON.stringify(body) : undefined,
226 });
227 const json = await response.json().catch(() => ({}));
228 return { status: response.status, json };
229 };
230}
231
232/**
233 * Restores one repository. Returns what happened: `restored`, `current`
234 * (already restored from the same last backup), or `missing` (no backup).
235 */
236export async function restoreOne(target, { read, into, gitstore, defaultNamespace = "g1t", work }) {
237 const { namespace, name } = locate(target.store, defaultNamespace);
238 const scratch = await mkdtemp(join(work ?? tmpdir(), "g1t-restore-"));
239 try {
240 let manifestFile;
241 try {
242 manifestFile = await read(`backups/${target.id}/manifest.json`, join(scratch, "manifest.json"));
243 } catch {
244 return { id: target.id, namespace, name, result: "missing" };
245 }
246 if (!existsSync(manifestFile)) return { id: target.id, namespace, name, result: "missing" };
247 const manifest = readManifest(readFileSync(manifestFile, "utf8"));
248 const last = manifest.chain.at(-1);
249 if (into) {
250 const dir = repoDir(into, namespace, name);
251 const existing = readMeta(dir);
252 if (existing.restored?.entry === last.id && existing.restored?.repo_id === target.id) {
253 return { id: target.id, namespace, name, result: "current" };
254 }
255 // Built beside it, then put in place, so a reader never sees half.
256 const building = `${dir}.restoring-${process.pid}`;
257 rmSync(building, { recursive: true, force: true });
258 mkdirSync(dirname(dir), { recursive: true });
259 const refs = await restore(manifest, read, building, scratch);
260 await configure(building);
261 writeFileSync(join(building, "g1t.json"), JSON.stringify(metaFor(target, manifest, undefined, existing), null, 2));
262 rmSync(dir, { recursive: true, force: true });
263 renameSync(building, dir);
264 return { id: target.id, namespace, name, result: "restored", refs: Object.keys(refs).length };
265 }
266 // Over HTTP: rebuilt here, then pushed as a mirror.
267 const local = join(scratch, "restored.git");
268 const refs = await restore(manifest, read, local, scratch);
269 const key = `${namespace}/${name}`;
270 const made = await gitstore.api("POST", "", { name: key, defaultBranch: target.default_branch ?? "main" });
271 if (made.status !== 200 && made.json.code !== "ALREADY_EXISTS") throw new Error(`${key}: ${made.json.message ?? made.status}`);
272 const token = await gitstore.api("POST", `/${encodeURIComponent(key)}/tokens`, { scope: "write", ttl: 3600 });
273 if (token.status !== 200) throw new Error(`${key}: ${token.json.message ?? token.status}`);
274 const remote = `${gitstore.url.replace(/\/$/, "")}/git/${key}.git`;
275 await git(["-c", `http.extraHeader=Authorization: Bearer ${token.json.plaintext}`, "push", "--quiet", "--mirror", remote], { cwd: local });
276 return { id: target.id, namespace, name, result: "restored", refs: Object.keys(refs).length };
277 } finally {
278 rmSync(scratch, { recursive: true, force: true });
279 }
280}
281
282/** The restored repositories under `root`, with what each was restored from. */
283function restoredUnder(root) {
284 const out = [];
285 if (!existsSync(root)) return out;
286 for (const namespace of readdirSync(root)) {
287 const dir = join(root, namespace);
288 if (!NAME.test(namespace) || !existsSync(dir)) continue;
289 for (const entry of readdirSync(dir)) {
290 if (!entry.endsWith(".git")) continue;
291 const meta = readMeta(join(dir, entry));
292 if (meta.restored) out.push({ namespace, name: entry.slice(0, -4), dir: join(dir, entry), restored: meta.restored });
293 }
294 }
295 return out;
296}
297
298/** Repositories that took pushes since they were restored. */
299export async function changedRepos(root) {
300 const out = [];
301 for (const repo of restoredUnder(root)) {
302 const changes = changedSince(repo.restored.refs, await refsIn(repo.dir));
303 if (changes.length) out.push({ ...repo, changes });
304 }
305 return out;
306}
307
308/** Where to push a repository back to: Artifacts, or a bare repository under `--live-root`. */
309function liveRemote(liveRoot) {
310 if (liveRoot) return async (namespace, name) => ({ url: repoDir(liveRoot, namespace, name), args: [] });
311 const auth = cloudflareAuth();
312 if (!auth) throw new Error("set CLOUDFLARE_API_TOKEN (Artifacts edit), or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL, or --live-root");
313 const api = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/artifacts/namespaces`;
314 return async (namespace, name) => {
315 const at = `${api}/${namespace}/repos/${encodeURIComponent(name)}`;
316 const info = await (await fetch(at, { headers: auth })).json().catch(() => ({}));
317 const minted = await (
318 await fetch(`${at}/tokens`, { method: "POST", headers: { ...auth, "content-type": "application/json" }, body: JSON.stringify({ scope: "write", ttl: 3600 }) })
319 )
320 .json()
321 .catch(() => ({}));
322 const token = minted.result?.plaintext ?? minted.result?.token;
323 if (!token) throw new Error(`${namespace}/${name}: Artifacts gave no write token (${JSON.stringify(minted.errors ?? minted).slice(0, 300)})`);
324 const url = info.result?.remote ?? `https://${ACCOUNT_ID}.artifacts.cloudflare.net/git/${namespace}/${name}.git`;
325 // The token as Artifacts gives it, `?expires=` and all, as g1t sends it.
326 return { url, args: ["-c", `http.extraHeader=Authorization: Bearer ${token}`] };
327 };
328}
329
330/** Sends one repository's changes back; what was done with each ref. */
331export async function reconcileOne(repo, remoteFor) {
332 const { url, args } = await remoteFor(repo.namespace, repo.name);
333 const live = parseRefs(await git([...args, "ls-remote", url], { cwd: repo.dir }));
334 const plan = reconcilePlan(repo.changes, live);
335 const specs = [];
336 for (const step of plan) {
337 if (step.action === "move") {
338 const lease = `--force-with-lease=${step.ref}:${step.from ?? ""}`;
339 specs.push({ lease, spec: step.to ? `+${step.to}:${step.ref}` : `:${step.ref}` });
340 } else if (step.action === "conflict" && step.kept) {
341 specs.push({ lease: null, spec: `+${step.to}:${step.kept}` });
342 }
343 }
344 if (specs.length) {
345 // Not --atomic: whether Artifacts takes it is not documented (docs/ARTIFACTS.md, Q6).
346 // Each move is leased on the value read above, so one that moved since is refused, not overwritten.
347 const leases = specs.map((one) => one.lease).filter(Boolean);
348 await git([...args, "push", "--quiet", ...leases, url, ...specs.map((one) => one.spec)], { cwd: repo.dir });
349 }
350 return plan;
351}
352
353async function main() {
354 const argv = process.argv.slice(2);
355 const command = argv[0];
356 const option = (name) => {
357 const at = argv.indexOf(name);
358 return at >= 0 ? argv[at + 1] : undefined;
359 };
360 const many = (name) => argv.flatMap((arg, at) => (arg === name && argv[at + 1] ? [argv[at + 1]] : []));
361 const defaultNamespace = option("--default-namespace") ?? "g1t";
362 const into = option("--into");
363
364 if (command === "index") {
365 console.log(JSON.stringify(await indexFromDatabase(), null, 2));
366 return 0;
367 }
368
369 if (command === "restore") {
370 const url = option("--gitstore");
371 if (!into && !url) throw new Error("say where to restore to: --into <root> or --gitstore <url>");
372 const gitstore = url ? { url, api: gitstoreApi(url, process.env.GITSTORE_SECRET ?? "") } : null;
373 const bundles = option("--bundles");
374 let targets = option("--index")
375 ? JSON.parse(readFileSync(option("--index"), "utf8"))
376 : bundles && argv.includes("--offline")
377 ? indexFromBundles(bundles)
378 : await indexFromDatabase();
379 const only = many("--repo");
380 if (only.length) targets = targets.filter((target) => only.includes(target.id));
381 const namespace = option("--namespace");
382 if (namespace) targets = targets.filter((target) => locate(target.store, defaultNamespace).namespace === namespace);
383 const read = bucketReader(bundles);
384 const started = Date.now();
385 const counts = { restored: 0, current: 0, missing: 0, failed: 0 };
386 await pool(targets, Number(option("--jobs") ?? 4), async (target) => {
387 try {
388 const done = await restoreOne(target, { read, into, gitstore, defaultNamespace });
389 counts[done.result] += 1;
390 if (done.result !== "current") console.log(`${done.result.padEnd(8)} ${done.namespace}/${done.name} (${target.path ?? target.id})`);
391 } catch (error) {
392 counts.failed += 1;
393 console.error(`failed ${target.store} (${target.id}): ${error.message}`);
394 }
395 });
396 const seconds = ((Date.now() - started) / 1000).toFixed(0);
397 console.log(
398 `${targets.length} repositories in ${seconds}s: ${counts.restored} restored, ${counts.current} already current, ${counts.missing} without a backup, ${counts.failed} failed`,
399 );
400 return counts.failed ? 1 : 0;
401 }
402
403 if (command === "changed" || command === "reconcile") {
404 if (!into) throw new Error(`${command} reads the git store's root: --into <root>`);
405 const changed = await changedRepos(into);
406 if (command === "changed") {
407 for (const repo of changed) {
408 console.log(`${repo.namespace}/${repo.name} (${repo.restored.repo_id})`);
409 for (const { ref, base, now } of repo.changes) console.log(` ${ref}: ${base ?? "(none)"} -> ${now ?? "(deleted)"}`);
410 }
411 console.log(`${changed.length} repositories took pushes since they were restored`);
412 return 0;
413 }
414 const remoteFor = liveRemote(option("--live-root"));
415 const bumped = [];
416 let conflicts = 0;
417 let failed = 0;
418 for (const repo of changed) {
419 try {
420 const plan = await reconcileOne(repo, remoteFor);
421 bumped.push(repo.restored.repo_id);
422 for (const step of plan) {
423 if (step.action === "conflict") conflicts += 1;
424 const what = step.action === "conflict" ? `moved on both sides; the fallback's kept as ${step.kept ?? "(it deleted it)"}` : step.action;
425 console.log(`${repo.namespace}/${repo.name} ${step.ref}: ${what}`);
426 }
427 } catch (error) {
428 failed += 1;
429 console.error(`${repo.namespace}/${repo.name}: ${error.message}`);
430 }
431 }
432 if (bumped.length && !argv.includes("--no-bump")) {
433 await d1(`UPDATE repos SET refs_version = coalesce(refs_version, 0) + 1 WHERE id IN (${bumped.map(quoted).join(", ")})`);
434 }
435 console.log(`${changed.length} repositories reconciled: ${conflicts} refs kept beside a newer one, ${failed} failed`);
436 return failed ? 1 : conflicts ? 3 : 0;
437 }
438
439 console.error("usage: restore-to-gitstore.mjs index | restore | changed | reconcile (see the top of the file)");
440 return 2;
441}
442
443if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/restore-to-gitstore.mjs")) {
444 main().then(
445 (code) => process.exit(code),
446 (error) => {
447 console.error(`restore: ${error.message}`);
448 process.exit(2);
449 },
450 );
451}