Sign in

flagon-io/g1t

public

g1t/scripts/ops/backup-restore-drill.mjs

259 lines12,196 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-ac5b181a013e54348'1#!/usr/bin/env node
2// The restore drill for nightly backups (docs/ARTIFACTS.md, R11;
3// services/repos/src/backups.rs). Picks a repository, downloads its
4// manifest and bundle chain from the g1t-backups bucket, rebuilds the
5// repository from them in a temporary directory, and compares every ref
6// with the live repository. Exits 1 on any difference, 2 when it could not
7// run.
8//
9// Read-only: SELECTs against the g1t-repos database, reads of the bucket,
10// and `git ls-remote` of the live repository. Nothing is written anywhere
11// but the temporary directory, which is removed unless you pass --keep.
12//
13// node scripts/ops/backup-restore-drill.mjs # a repository unchanged since its last backup
14// node scripts/ops/backup-restore-drill.mjs --repo acme/rocket
15// node scripts/ops/backup-restore-drill.mjs --repo-id repo_... --bundles ./copy --live /srv/git/acme--rocket.git
16//
17// Options:
18// --repo <workspace/name> the repository; default: one picked at random
19// among those whose refs have not moved since
20// their last backup, so any difference is the
21// backup's.
22// --repo-id <id> the repository by id (no database needed with --bundles).
23// --bundles <dir> read the bucket from a local copy (`backups/<id>/...`
Docs: the self-hosted object store is RustFS24// under it, as `aws s3 sync` or `rclone copy` leave it)
25// instead of R2, e.g. a self-hosted store's.
Merge branch 'worktree-agent-ac5b181a013e54348'26// --live <url or path> the live repository to compare with; default
27// https://g1t.sh/<workspace>/<name>.git.
28// --keep keep the temporary directory, and say where it is.
29//
30// The live repository is read as G1T_USER with G1T_TOKEN (an access token
31// with code:read) when they are set, which private repositories need. The
32// database and the bucket are read through Wrangler, as you are logged in
33// (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN when that is set.
34
35import { createHash } from "node:crypto";
36import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
37import { tmpdir } from "node:os";
38import { join } from "node:path";
39
40import { exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
41import { ROOT } from "../deploy/stack.mjs";
42
43const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
44const DATABASE = "g1t-repos";
45const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups";
46const MANIFEST_VERSION = 1;
47const STAGING = "refs/drill-staging";
48
49/** Runs git; resolves with its output, or throws with what it said. */
50async function git(args, { cwd, input } = {}) {
51 const { code, out } = await exec("git", args, { cwd, input });
52 if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`);
53 return out.trim();
54}
55
56/** A file's SHA-256, read as a stream: a bundle can be a gigabyte. */
57async function sha256Of(file) {
58 const hash = createHash("sha256");
59 for await (const chunk of createReadStream(file)) hash.update(chunk);
60 return hash.digest("hex");
61}
62
63/** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */
64export function parseRefs(listing) {
65 const refs = {};
66 for (const line of listing.split(/\r?\n/)) {
67 const match = /^([0-9a-f]{40,64})\s+(\S+)$/.exec(line.trim());
68 if (!match || match[2].endsWith("^{}")) continue;
69 refs[match[2]] = match[1];
70 }
71 return refs;
72}
73
74/** Every ref of the repository in `dir`, and HEAD. */
75async function refsOf(dir) {
76 const refs = parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir }));
77 const head = await git(["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: dir }).catch(() => "");
78 if (head) refs.HEAD = head;
79 return refs;
80}
81
82/** The refs that differ between `want` and `have`: [{ ref, want, have }], `null` for absent. */
83export function compareRefs(want, have) {
84 const names = [...new Set([...Object.keys(want), ...Object.keys(have)])].sort();
85 return names
86 .filter((ref) => want[ref] !== have[ref])
87 .map((ref) => ({ ref, want: want[ref] ?? null, have: have[ref] ?? null }));
88}
89
90/** The branch HEAD should name: one at HEAD's commit, `main` or `master` first. */
91export function headBranch(refs) {
92 if (!refs.HEAD) return null;
93 const branches = Object.keys(refs).filter((ref) => ref.startsWith("refs/heads/") && refs[ref] === refs.HEAD);
94 return ["refs/heads/main", "refs/heads/master"].find((ref) => branches.includes(ref)) ?? branches.sort()[0] ?? null;
95}
96
97/** Whether a manifest can be read by this drill. */
98export function readManifest(text) {
99 const manifest = JSON.parse(text);
100 if (manifest.version !== MANIFEST_VERSION) throw new Error(`manifest version ${manifest.version}; this drill reads ${MANIFEST_VERSION}`);
101 if (!Array.isArray(manifest.chain) || manifest.chain.length === 0) throw new Error("the manifest lists no backups");
102 if (manifest.chain[0].kind !== "full") throw new Error("the chain does not start with a full backup");
103 return manifest;
104}
105
106/**
107 * Rebuilds the repository the manifest's chain describes into `dir`, a
108 * new bare repository: each bundle in order, checked against its size and
109 * SHA-256 and verified by git, fetched without following tags; then every
110 * ref set to what the last entry says, and nothing else kept. `fetchObject`
111 * saves one object of the bucket to a file and resolves with its path.
112 */
113export async function restore(manifest, fetchObject, dir, work) {
114 await git(["init", "--quiet", "--bare", dir]);
115 for (const entry of manifest.chain) {
116 if (!entry.key) continue;
117 const file = await fetchObject(entry.key, join(work, `${entry.id}.bundle`));
118 const size = statSync(file).size;
119 if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`);
120 if (entry.sha256) {
121 const sha256 = await sha256Of(file);
122 if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`);
123 }
124 await git(["bundle", "verify", "--quiet", file], { cwd: dir });
125 await git(["fetch", "--quiet", "--no-tags", file, `+refs/*:${STAGING}/${entry.id}/*`], { cwd: dir });
126 }
127 const last = manifest.chain.at(-1).refs;
128 const updates = Object.entries(last)
129 .filter(([ref]) => ref !== "HEAD")
130 .map(([ref, hash]) => `update ${ref} ${hash}\n`)
131 .join("");
132 const staged = await git(["for-each-ref", "--format=delete %(refname)", `${STAGING}/`], { cwd: dir });
133 const commands = [updates.trimEnd(), staged].filter(Boolean).join("\n");
134 if (commands) await git(["update-ref", "--stdin"], { cwd: dir, input: `${commands}\n` });
135 const head = headBranch(last);
136 if (head) await git(["symbolic-ref", "HEAD", head], { cwd: dir });
137 // Every object every ref reaches is there.
138 await git(["fsck", "--no-progress", "--connectivity-only"], { cwd: dir });
139 return refsOf(dir);
140}
141
142// ---------------------------------------------------------------------
143
144async function d1(sql) {
145 const env = wranglerEnv();
146 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
147 cwd: join(ROOT, "services/repos"),
148 env,
149 });
150 if (code !== 0) throw new Error(out.slice(-600));
151 return jsonFrom(out)[0]?.results ?? [];
152}
153
154const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
155
156/** The repository to drill, and whether its refs moved since its last backup. */
157async function pick({ repo, repoId }) {
158 const select = `SELECT r.id, r.namespace, r.name, r.refs_version, b.refs_version AS backed_version,
159 coalesce(r.refs_open_until, 0) > coalesce(b.backed_up_ms, 0) AS opened
160 FROM repo_backups b JOIN repos r ON r.id = b.repo_id
161 WHERE b.last_entry IS NOT NULL AND r.deleted_at IS NULL`;
162 let rows;
163 if (repoId) rows = await d1(`${select} AND r.id = ${quoted(repoId)}`);
164 else if (repo) {
165 const [namespace, name] = repo.toLowerCase().split("/");
166 rows = await d1(`${select} AND r.namespace = ${quoted(namespace)} AND r.name = ${quoted(name)}`);
167 } else {
168 rows = await d1(`${select} AND b.refs_version = r.refs_version AND coalesce(r.refs_open_until, 0) <= coalesce(b.backed_up_ms, 0)
169 ORDER BY random() LIMIT 1`);
170 }
171 const row = rows[0];
172 if (!row) throw new Error(repo || repoId ? `no backup of ${repo ?? repoId}` : "no repository has a backup yet");
173 return {
174 id: row.id,
175 path: `${row.namespace}/${row.name}`,
176 moved: row.refs_version !== row.backed_version || Boolean(row.opened),
177 };
178}
179
180/** Saves one object of the bucket to `file`. */
181function bucketReader(localCopy) {
182 if (localCopy) return async (key) => join(localCopy, key);
183 return async (key, file) => {
184 const env = wranglerEnv();
185 const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env });
186 if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
187 return file;
188 };
189}
190
191/** The live repository's refs, as a clone would see them. */
192async function liveRefs(live) {
193 const args = [];
194 if (process.env.G1T_TOKEN && /^https?:/.test(live)) {
195 const user = process.env.G1T_USER || "g1t";
196 const basic = Buffer.from(`${user}:${process.env.G1T_TOKEN}`).toString("base64");
197 args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`);
198 }
199 return parseRefs(await git([...args, "ls-remote", live]));
200}
201
202async function main() {
203 const args = process.argv.slice(2);
204 const option = (name) => {
205 const at = args.indexOf(name);
206 return at >= 0 ? args[at + 1] : undefined;
207 };
208 const keep = args.includes("--keep");
209 const localCopy = option("--bundles");
210 const target =
211 localCopy && option("--repo-id")
212 ? { id: option("--repo-id"), path: option("--repo") ?? null, moved: false }
213 : await pick({ repo: option("--repo"), repoId: option("--repo-id") });
214 const live = option("--live") ?? (target.path ? `https://g1t.sh/${target.path}.git` : null);
215 if (!live) throw new Error("say which live repository to compare with: --live, or --repo");
216
217 const work = mkdtempSync(join(tmpdir(), "g1t-drill-"));
218 try {
219 const read = bucketReader(localCopy);
220 const manifest = readManifest(readFileSync(await read(`backups/${target.id}/manifest.json`, join(work, "manifest.json")), "utf8"));
221 const started = Date.now();
222 const restored = await restore(manifest, read, join(work, "restored.git"), work);
223 const seconds = ((Date.now() - started) / 1000).toFixed(1);
224 const last = manifest.chain.at(-1);
225 const bytes = manifest.chain.reduce((sum, entry) => sum + (entry.size ?? 0), 0);
226 console.log(`${target.path ?? target.id}: ${manifest.chain.length} backups (${bytes} bytes), the last ${last.created_at}, restored in ${seconds}s`);
227
228 const fromChain = compareRefs(last.refs, restored);
229 const fromLive = compareRefs(await liveRefs(live), restored);
230 for (const [what, differences] of [
231 ["the manifest", fromChain],
232 ["the live repository", fromLive],
233 ]) {
234 if (differences.length === 0) {
235 console.log(` every ref matches ${what} (${Object.keys(restored).length} refs)`);
236 continue;
237 }
238 console.log(` ${differences.length} refs differ from ${what}:`);
239 for (const { ref, want, have } of differences) console.log(` ${ref}: ${what} ${want ?? "(none)"}, restored ${have ?? "(none)"}`);
240 }
241 if (target.moved && fromLive.length > 0) {
242 console.log(" The repository's refs moved since its last backup, so differences from it may be new work, not a fault.");
243 }
244 if (keep) console.log(` kept: ${work}`);
245 return fromChain.length === 0 && fromLive.length === 0 ? 0 : 1;
246 } finally {
247 if (!keep) rmSync(work, { recursive: true, force: true });
248 }
249}
250
251if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/backup-restore-drill.mjs")) {
252 main().then(
253 (code) => process.exit(code),
254 (error) => {
255 console.error(`drill: ${error.message}`);
256 process.exit(2);
257 },
258 );
259}

This file's history is long; its oldest lines are credited to the oldest commit read.