Sign in

flagon-io/g1t

public

g1t/scripts/ops/backup-restore-drill.mjs

254 lines12,182 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

The backup restore drill: rebuild a repository from its bundle chain and compare every ref1#!/usr/bin/env node
2// The restore drill for nightly backups (docs/ARTIFACTS.md, R11;
3// services/repos/src/backups.rs). Picks a repository, downloads its
4// manifest and bundle chain from the g1t-backups bucket, rebuilds the
5// repository from them in a temporary directory, and compares every ref
6// with the live repository. Exits 1 on any difference, 2 when it could not
7// run.
8//
9// Read-only: SELECTs against the g1t-repos database, reads of the bucket,
10// and `git ls-remote` of the live repository. Nothing is written anywhere
11// but the temporary directory, which is removed unless you pass --keep.
12//
13// node scripts/ops/backup-restore-drill.mjs # a repository unchanged since its last backup
14// node scripts/ops/backup-restore-drill.mjs --repo acme/rocket
15// node scripts/ops/backup-restore-drill.mjs --repo-id repo_... --bundles ./copy --live /srv/git/acme--rocket.git
16//
17// Options:
18// --repo <workspace/name> the repository; default: one picked at random
19// among those whose refs have not moved since
20// their last backup, so any difference is the
21// backup's.
22// --repo-id <id> the repository by id (no database needed with --bundles).
23// --bundles <dir> read the bucket from a local copy (`backups/<id>/...`
24// under it, as `mc mirror` or `rclone copy` leave it)
25// instead of R2, e.g. a self-hosted MinIO's.
26// --live <url or path> the live repository to compare with; default
27// https://g1t.sh/<workspace>/<name>.git.
28// --keep keep the temporary directory, and say where it is.
29//
30// The live repository is read as G1T_USER with G1T_TOKEN (an access token
31// with code:read) when they are set, which private repositories need. The
32// database and the bucket are read through Wrangler, as you are logged in
33// (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN.
34
35import { createHash } from "node:crypto";
36import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
37import { tmpdir } from "node:os";
38import { join } from "node:path";
39
40import { exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
41import { ROOT } from "../deploy/stack.mjs";
42
43const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
44const DATABASE = "g1t-repos";
45const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups";
46const MANIFEST_VERSION = 1;
47const STAGING = "refs/drill-staging";
48
49/** Runs git; resolves with its output, or throws with what it said. */
50async function git(args, { cwd, input } = {}) {
51 const { code, out } = await exec("git", args, { cwd, input });
52 if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`);
53 return out.trim();
54}
55
56/** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */
57export function parseRefs(listing) {
58 const refs = {};
59 for (const line of listing.split(/\r?\n/)) {
60 const match = /^([0-9a-f]{40,64})\s+(\S+)$/.exec(line.trim());
61 if (!match || match[2].endsWith("^{}")) continue;
62 refs[match[2]] = match[1];
63 }
64 return refs;
65}
66
67/** Every ref of the repository in `dir`, and HEAD. */
68async function refsOf(dir) {
69 const refs = parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir }));
70 const head = await git(["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: dir }).catch(() => "");
71 if (head) refs.HEAD = head;
72 return refs;
73}
74
75/** The refs that differ between `want` and `have`: [{ ref, want, have }], `null` for absent. */
76export function compareRefs(want, have) {
77 const names = [...new Set([...Object.keys(want), ...Object.keys(have)])].sort();
78 return names
79 .filter((ref) => want[ref] !== have[ref])
80 .map((ref) => ({ ref, want: want[ref] ?? null, have: have[ref] ?? null }));
81}
82
83/** The branch HEAD should name: one at HEAD's commit, `main` or `master` first. */
84export function headBranch(refs) {
85 if (!refs.HEAD) return null;
86 const branches = Object.keys(refs).filter((ref) => ref.startsWith("refs/heads/") && refs[ref] === refs.HEAD);
87 return ["refs/heads/main", "refs/heads/master"].find((ref) => branches.includes(ref)) ?? branches.sort()[0] ?? null;
88}
89
90/** Whether a manifest can be read by this drill. */
91export function readManifest(text) {
92 const manifest = JSON.parse(text);
93 if (manifest.version !== MANIFEST_VERSION) throw new Error(`manifest version ${manifest.version}; this drill reads ${MANIFEST_VERSION}`);
94 if (!Array.isArray(manifest.chain) || manifest.chain.length === 0) throw new Error("the manifest lists no backups");
95 if (manifest.chain[0].kind !== "full") throw new Error("the chain does not start with a full backup");
96 return manifest;
97}
98
99/**
100 * Rebuilds the repository the manifest's chain describes into `dir`, a
101 * new bare repository: each bundle in order, checked against its size and
102 * SHA-256 and verified by git, fetched without following tags; then every
103 * ref set to what the last entry says, and nothing else kept. `fetchObject`
104 * saves one object of the bucket to a file and resolves with its path.
105 */
106export async function restore(manifest, fetchObject, dir, work) {
107 await git(["init", "--quiet", "--bare", dir]);
108 for (const entry of manifest.chain) {
109 if (!entry.key) continue;
110 const file = await fetchObject(entry.key, join(work, `${entry.id}.bundle`));
111 const size = statSync(file).size;
112 if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`);
113 if (entry.sha256) {
114 const sha256 = createHash("sha256").update(readFileSync(file)).digest("hex");
115 if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`);
116 }
117 await git(["bundle", "verify", "--quiet", file], { cwd: dir });
118 await git(["fetch", "--quiet", "--no-tags", file, `+refs/*:${STAGING}/${entry.id}/*`], { cwd: dir });
119 }
120 const last = manifest.chain.at(-1).refs;
121 const updates = Object.entries(last)
122 .filter(([ref]) => ref !== "HEAD")
123 .map(([ref, hash]) => `update ${ref} ${hash}\n`)
124 .join("");
125 const staged = await git(["for-each-ref", "--format=delete %(refname)", `${STAGING}/`], { cwd: dir });
126 const commands = [updates.trimEnd(), staged].filter(Boolean).join("\n");
127 if (commands) await git(["update-ref", "--stdin"], { cwd: dir, input: `${commands}\n` });
128 const head = headBranch(last);
129 if (head) await git(["symbolic-ref", "HEAD", head], { cwd: dir });
130 // Every object every ref reaches is there.
131 await git(["fsck", "--no-progress", "--connectivity-only"], { cwd: dir });
132 return refsOf(dir);
133}
134
135// ---------------------------------------------------------------------
136
137async function d1(sql) {
138 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
139 if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = "";
140 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
141 cwd: join(ROOT, "services/repos"),
142 env,
143 });
144 if (code !== 0) throw new Error(out.slice(-600));
145 return jsonFrom(out)[0]?.results ?? [];
146}
147
148const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
149
150/** The repository to drill, and whether its refs moved since its last backup. */
151async function pick({ repo, repoId }) {
152 const select = `SELECT r.id, r.namespace, r.name, r.refs_version, b.refs_version AS backed_version,
153 coalesce(r.refs_open_until, 0) > coalesce(b.backed_up_ms, 0) AS opened
154 FROM repo_backups b JOIN repos r ON r.id = b.repo_id
155 WHERE b.last_entry IS NOT NULL AND r.deleted_at IS NULL`;
156 let rows;
157 if (repoId) rows = await d1(`${select} AND r.id = ${quoted(repoId)}`);
158 else if (repo) {
159 const [namespace, name] = repo.toLowerCase().split("/");
160 rows = await d1(`${select} AND r.namespace = ${quoted(namespace)} AND r.name = ${quoted(name)}`);
161 } else {
162 rows = await d1(`${select} AND b.refs_version = r.refs_version AND coalesce(r.refs_open_until, 0) <= coalesce(b.backed_up_ms, 0)
163 ORDER BY random() LIMIT 1`);
164 }
165 const row = rows[0];
166 if (!row) throw new Error(repo || repoId ? `no backup of ${repo ?? repoId}` : "no repository has a backup yet");
167 return {
168 id: row.id,
169 path: `${row.namespace}/${row.name}`,
170 moved: row.refs_version !== row.backed_version || Boolean(row.opened),
171 };
172}
173
174/** Saves one object of the bucket to `file`. */
175function bucketReader(localCopy) {
176 if (localCopy) return async (key) => join(localCopy, key);
177 return async (key, file) => {
178 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
179 if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = "";
180 const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env });
181 if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
182 return file;
183 };
184}
185
186/** The live repository's refs, as a clone would see them. */
187async function liveRefs(live) {
188 const args = [];
189 if (process.env.G1T_TOKEN && /^https?:/.test(live)) {
190 const user = process.env.G1T_USER || "g1t";
191 const basic = Buffer.from(`${user}:${process.env.G1T_TOKEN}`).toString("base64");
192 args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`);
193 }
194 return parseRefs(await git([...args, "ls-remote", live]));
195}
196
197async function main() {
198 const args = process.argv.slice(2);
199 const option = (name) => {
200 const at = args.indexOf(name);
201 return at >= 0 ? args[at + 1] : undefined;
202 };
203 const keep = args.includes("--keep");
204 const localCopy = option("--bundles");
205 const target =
206 localCopy && option("--repo-id")
207 ? { id: option("--repo-id"), path: option("--repo") ?? null, moved: false }
208 : await pick({ repo: option("--repo"), repoId: option("--repo-id") });
209 const live = option("--live") ?? (target.path ? `https://g1t.sh/${target.path}.git` : null);
210 if (!live) throw new Error("say which live repository to compare with: --live, or --repo");
211
212 const work = mkdtempSync(join(tmpdir(), "g1t-drill-"));
213 try {
214 const read = bucketReader(localCopy);
215 const manifest = readManifest(readFileSync(await read(`backups/${target.id}/manifest.json`, join(work, "manifest.json")), "utf8"));
216 const started = Date.now();
217 const restored = await restore(manifest, read, join(work, "restored.git"), work);
218 const seconds = ((Date.now() - started) / 1000).toFixed(1);
219 const last = manifest.chain.at(-1);
220 const bytes = manifest.chain.reduce((sum, entry) => sum + (entry.size ?? 0), 0);
221 console.log(`${target.path ?? target.id}: ${manifest.chain.length} backups (${bytes} bytes), the last ${last.created_at}, restored in ${seconds}s`);
222
223 const fromChain = compareRefs(last.refs, restored);
224 const fromLive = compareRefs(await liveRefs(live), restored);
225 for (const [what, differences] of [
226 ["the manifest", fromChain],
227 ["the live repository", fromLive],
228 ]) {
229 if (differences.length === 0) {
230 console.log(` every ref matches ${what} (${Object.keys(restored).length} refs)`);
231 continue;
232 }
233 console.log(` ${differences.length} refs differ from ${what}:`);
234 for (const { ref, want, have } of differences) console.log(` ${ref}: ${what} ${want ?? "(none)"}, restored ${have ?? "(none)"}`);
235 }
236 if (target.moved && fromLive.length > 0) {
237 console.log(" The repository's refs moved since its last backup, so differences from it may be new work, not a fault.");
238 }
239 if (keep) console.log(` kept: ${work}`);
240 return fromChain.length === 0 && fromLive.length === 0 ? 0 : 1;
241 } finally {
242 if (!keep) rmSync(work, { recursive: true, force: true });
243 }
244}
245
246if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/backup-restore-drill.mjs")) {
247 main().then(
248 (code) => process.exit(code),
249 (error) => {
250 console.error(`drill: ${error.message}`);
251 process.exit(2);
252 },
253 );
254}