| 1 | + | #!/usr/bin/env node |
| 2 | + | // The restore drill for nightly backups (docs/ARTIFACTS.md, R11; |
| 3 | + | // services/repos/src/backups.rs). Picks a repository, downloads its |
| 4 | + | // manifest and bundle chain from the g1t-backups bucket, rebuilds the |
| 5 | + | // repository from them in a temporary directory, and compares every ref |
| 6 | + | // with the live repository. Exits 1 on any difference, 2 when it could not |
| 7 | + | // run. |
| 8 | + | // |
| 9 | + | // Read-only: SELECTs against the g1t-repos database, reads of the bucket, |
| 10 | + | // and `git ls-remote` of the live repository. Nothing is written anywhere |
| 11 | + | // but the temporary directory, which is removed unless you pass --keep. |
| 12 | + | // |
| 13 | + | // node scripts/ops/backup-restore-drill.mjs # a repository unchanged since its last backup |
| 14 | + | // node scripts/ops/backup-restore-drill.mjs --repo acme/rocket |
| 15 | + | // node scripts/ops/backup-restore-drill.mjs --repo-id repo_... --bundles ./copy --live /srv/git/acme--rocket.git |
| 16 | + | // |
| 17 | + | // Options: |
| 18 | + | // --repo <workspace/name> the repository; default: one picked at random |
| 19 | + | // among those whose refs have not moved since |
| 20 | + | // their last backup, so any difference is the |
| 21 | + | // backup's. |
| 22 | + | // --repo-id <id> the repository by id (no database needed with --bundles). |
| 23 | + | // --bundles <dir> read the bucket from a local copy (`backups/<id>/...` |
| 24 | + | // under it, as `mc mirror` or `rclone copy` leave it) |
| 25 | + | // instead of R2, e.g. a self-hosted MinIO's. |
| 26 | + | // --live <url or path> the live repository to compare with; default |
| 27 | + | // https://g1t.sh/<workspace>/<name>.git. |
| 28 | + | // --keep keep the temporary directory, and say where it is. |
| 29 | + | // |
| 30 | + | // The live repository is read as G1T_USER with G1T_TOKEN (an access token |
| 31 | + | // with code:read) when they are set, which private repositories need. The |
| 32 | + | // database and the bucket are read through Wrangler, as you are logged in |
| 33 | + | // (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN. |
| 34 | + | |
| 35 | + | import { createHash } from "node:crypto"; |
| 36 | + | import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; |
| 37 | + | import { tmpdir } from "node:os"; |
| 38 | + | import { join } from "node:path"; |
| 39 | + | |
| 40 | + | import { exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs"; |
| 41 | + | import { ROOT } from "../deploy/stack.mjs"; |
| 42 | + | |
| 43 | + | const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js"); |
| 44 | + | const DATABASE = "g1t-repos"; |
| 45 | + | const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups"; |
| 46 | + | const MANIFEST_VERSION = 1; |
| 47 | + | const STAGING = "refs/drill-staging"; |
| 48 | + | |
| 49 | + | /** Runs git; resolves with its output, or throws with what it said. */ |
| 50 | + | async function git(args, { cwd, input } = {}) { |
| 51 | + | const { code, out } = await exec("git", args, { cwd, input }); |
| 52 | + | if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`); |
| 53 | + | return out.trim(); |
| 54 | + | } |
| 55 | + | |
| 56 | + | /** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */ |
| 57 | + | export function parseRefs(listing) { |
| 58 | + | const refs = {}; |
| 59 | + | for (const line of listing.split(/\r?\n/)) { |
| 60 | + | const match = /^([0-9a-f]{40,64})\s+(\S+)$/.exec(line.trim()); |
| 61 | + | if (!match || match[2].endsWith("^{}")) continue; |
| 62 | + | refs[match[2]] = match[1]; |
| 63 | + | } |
| 64 | + | return refs; |
| 65 | + | } |
| 66 | + | |
| 67 | + | /** Every ref of the repository in `dir`, and HEAD. */ |
| 68 | + | async function refsOf(dir) { |
| 69 | + | const refs = parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir })); |
| 70 | + | const head = await git(["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: dir }).catch(() => ""); |
| 71 | + | if (head) refs.HEAD = head; |
| 72 | + | return refs; |
| 73 | + | } |
| 74 | + | |
| 75 | + | /** The refs that differ between `want` and `have`: [{ ref, want, have }], `null` for absent. */ |
| 76 | + | export function compareRefs(want, have) { |
| 77 | + | const names = [...new Set([...Object.keys(want), ...Object.keys(have)])].sort(); |
| 78 | + | return names |
| 79 | + | .filter((ref) => want[ref] !== have[ref]) |
| 80 | + | .map((ref) => ({ ref, want: want[ref] ?? null, have: have[ref] ?? null })); |
| 81 | + | } |
| 82 | + | |
| 83 | + | /** The branch HEAD should name: one at HEAD's commit, `main` or `master` first. */ |
| 84 | + | export function headBranch(refs) { |
| 85 | + | if (!refs.HEAD) return null; |
| 86 | + | const branches = Object.keys(refs).filter((ref) => ref.startsWith("refs/heads/") && refs[ref] === refs.HEAD); |
| 87 | + | return ["refs/heads/main", "refs/heads/master"].find((ref) => branches.includes(ref)) ?? branches.sort()[0] ?? null; |
| 88 | + | } |
| 89 | + | |
| 90 | + | /** Whether a manifest can be read by this drill. */ |
| 91 | + | export function readManifest(text) { |
| 92 | + | const manifest = JSON.parse(text); |
| 93 | + | if (manifest.version !== MANIFEST_VERSION) throw new Error(`manifest version ${manifest.version}; this drill reads ${MANIFEST_VERSION}`); |
| 94 | + | if (!Array.isArray(manifest.chain) || manifest.chain.length === 0) throw new Error("the manifest lists no backups"); |
| 95 | + | if (manifest.chain[0].kind !== "full") throw new Error("the chain does not start with a full backup"); |
| 96 | + | return manifest; |
| 97 | + | } |
| 98 | + | |
| 99 | + | /** |
| 100 | + | * Rebuilds the repository the manifest's chain describes into `dir`, a |
| 101 | + | * new bare repository: each bundle in order, checked against its size and |
| 102 | + | * SHA-256 and verified by git, fetched without following tags; then every |
| 103 | + | * ref set to what the last entry says, and nothing else kept. `fetchObject` |
| 104 | + | * saves one object of the bucket to a file and resolves with its path. |
| 105 | + | */ |
| 106 | + | export async function restore(manifest, fetchObject, dir, work) { |
| 107 | + | await git(["init", "--quiet", "--bare", dir]); |
| 108 | + | for (const entry of manifest.chain) { |
| 109 | + | if (!entry.key) continue; |
| 110 | + | const file = await fetchObject(entry.key, join(work, `${entry.id}.bundle`)); |
| 111 | + | const size = statSync(file).size; |
| 112 | + | if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`); |
| 113 | + | if (entry.sha256) { |
| 114 | + | const sha256 = createHash("sha256").update(readFileSync(file)).digest("hex"); |
| 115 | + | if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`); |
| 116 | + | } |
| 117 | + | await git(["bundle", "verify", "--quiet", file], { cwd: dir }); |
| 118 | + | await git(["fetch", "--quiet", "--no-tags", file, `+refs/*:${STAGING}/${entry.id}/*`], { cwd: dir }); |
| 119 | + | } |
| 120 | + | const last = manifest.chain.at(-1).refs; |
| 121 | + | const updates = Object.entries(last) |
| 122 | + | .filter(([ref]) => ref !== "HEAD") |
| 123 | + | .map(([ref, hash]) => `update ${ref} ${hash}\n`) |
| 124 | + | .join(""); |
| 125 | + | const staged = await git(["for-each-ref", "--format=delete %(refname)", `${STAGING}/`], { cwd: dir }); |
| 126 | + | const commands = [updates.trimEnd(), staged].filter(Boolean).join("\n"); |
| 127 | + | if (commands) await git(["update-ref", "--stdin"], { cwd: dir, input: `${commands}\n` }); |
| 128 | + | const head = headBranch(last); |
| 129 | + | if (head) await git(["symbolic-ref", "HEAD", head], { cwd: dir }); |
| 130 | + | // Every object every ref reaches is there. |
| 131 | + | await git(["fsck", "--no-progress", "--connectivity-only"], { cwd: dir }); |
| 132 | + | return refsOf(dir); |
| 133 | + | } |
| 134 | + | |
| 135 | + | // --------------------------------------------------------------------- |
| 136 | + | |
| 137 | + | async function d1(sql) { |
| 138 | + | const env = { ...wranglerEnv({ ...process.env, CI: "true" }) }; |
| 139 | + | if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = ""; |
| 140 | + | const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], { |
| 141 | + | cwd: join(ROOT, "services/repos"), |
| 142 | + | env, |
| 143 | + | }); |
| 144 | + | if (code !== 0) throw new Error(out.slice(-600)); |
| 145 | + | return jsonFrom(out)[0]?.results ?? []; |
| 146 | + | } |
| 147 | + | |
| 148 | + | const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`; |
| 149 | + | |
| 150 | + | /** The repository to drill, and whether its refs moved since its last backup. */ |
| 151 | + | async function pick({ repo, repoId }) { |
| 152 | + | const select = `SELECT r.id, r.namespace, r.name, r.refs_version, b.refs_version AS backed_version, |
| 153 | + | coalesce(r.refs_open_until, 0) > coalesce(b.backed_up_ms, 0) AS opened |
| 154 | + | FROM repo_backups b JOIN repos r ON r.id = b.repo_id |
| 155 | + | WHERE b.last_entry IS NOT NULL AND r.deleted_at IS NULL`; |
| 156 | + | let rows; |
| 157 | + | if (repoId) rows = await d1(`${select} AND r.id = ${quoted(repoId)}`); |
| 158 | + | else if (repo) { |
| 159 | + | const [namespace, name] = repo.toLowerCase().split("/"); |
| 160 | + | rows = await d1(`${select} AND r.namespace = ${quoted(namespace)} AND r.name = ${quoted(name)}`); |
| 161 | + | } else { |
| 162 | + | rows = await d1(`${select} AND b.refs_version = r.refs_version AND coalesce(r.refs_open_until, 0) <= coalesce(b.backed_up_ms, 0) |
| 163 | + | ORDER BY random() LIMIT 1`); |
| 164 | + | } |
| 165 | + | const row = rows[0]; |
| 166 | + | if (!row) throw new Error(repo || repoId ? `no backup of ${repo ?? repoId}` : "no repository has a backup yet"); |
| 167 | + | return { |
| 168 | + | id: row.id, |
| 169 | + | path: `${row.namespace}/${row.name}`, |
| 170 | + | moved: row.refs_version !== row.backed_version || Boolean(row.opened), |
| 171 | + | }; |
| 172 | + | } |
| 173 | + | |
| 174 | + | /** Saves one object of the bucket to `file`. */ |
| 175 | + | function bucketReader(localCopy) { |
| 176 | + | if (localCopy) return async (key) => join(localCopy, key); |
| 177 | + | return async (key, file) => { |
| 178 | + | const env = { ...wranglerEnv({ ...process.env, CI: "true" }) }; |
| 179 | + | if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = ""; |
| 180 | + | const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env }); |
| 181 | + | if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`); |
| 182 | + | return file; |
| 183 | + | }; |
| 184 | + | } |
| 185 | + | |
| 186 | + | /** The live repository's refs, as a clone would see them. */ |
| 187 | + | async function liveRefs(live) { |
| 188 | + | const args = []; |
| 189 | + | if (process.env.G1T_TOKEN && /^https?:/.test(live)) { |
| 190 | + | const user = process.env.G1T_USER || "g1t"; |
| 191 | + | const basic = Buffer.from(`${user}:${process.env.G1T_TOKEN}`).toString("base64"); |
| 192 | + | args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`); |
| 193 | + | } |
| 194 | + | return parseRefs(await git([...args, "ls-remote", live])); |
| 195 | + | } |
| 196 | + | |
| 197 | + | async function main() { |
| 198 | + | const args = process.argv.slice(2); |
| 199 | + | const option = (name) => { |
| 200 | + | const at = args.indexOf(name); |
| 201 | + | return at >= 0 ? args[at + 1] : undefined; |
| 202 | + | }; |
| 203 | + | const keep = args.includes("--keep"); |
| 204 | + | const localCopy = option("--bundles"); |
| 205 | + | const target = |
| 206 | + | localCopy && option("--repo-id") |
| 207 | + | ? { id: option("--repo-id"), path: option("--repo") ?? null, moved: false } |
| 208 | + | : await pick({ repo: option("--repo"), repoId: option("--repo-id") }); |
| 209 | + | const live = option("--live") ?? (target.path ? `https://g1t.sh/${target.path}.git` : null); |
| 210 | + | if (!live) throw new Error("say which live repository to compare with: --live, or --repo"); |
| 211 | + | |
| 212 | + | const work = mkdtempSync(join(tmpdir(), "g1t-drill-")); |
| 213 | + | try { |
| 214 | + | const read = bucketReader(localCopy); |
| 215 | + | const manifest = readManifest(readFileSync(await read(`backups/${target.id}/manifest.json`, join(work, "manifest.json")), "utf8")); |
| 216 | + | const started = Date.now(); |
| 217 | + | const restored = await restore(manifest, read, join(work, "restored.git"), work); |
| 218 | + | const seconds = ((Date.now() - started) / 1000).toFixed(1); |
| 219 | + | const last = manifest.chain.at(-1); |
| 220 | + | const bytes = manifest.chain.reduce((sum, entry) => sum + (entry.size ?? 0), 0); |
| 221 | + | console.log(`${target.path ?? target.id}: ${manifest.chain.length} backups (${bytes} bytes), the last ${last.created_at}, restored in ${seconds}s`); |
| 222 | + | |
| 223 | + | const fromChain = compareRefs(last.refs, restored); |
| 224 | + | const fromLive = compareRefs(await liveRefs(live), restored); |
| 225 | + | for (const [what, differences] of [ |
| 226 | + | ["the manifest", fromChain], |
| 227 | + | ["the live repository", fromLive], |
| 228 | + | ]) { |
| 229 | + | if (differences.length === 0) { |
| 230 | + | console.log(` every ref matches ${what} (${Object.keys(restored).length} refs)`); |
| 231 | + | continue; |
| 232 | + | } |
| 233 | + | console.log(` ${differences.length} refs differ from ${what}:`); |
| 234 | + | for (const { ref, want, have } of differences) console.log(` ${ref}: ${what} ${want ?? "(none)"}, restored ${have ?? "(none)"}`); |
| 235 | + | } |
| 236 | + | if (target.moved && fromLive.length > 0) { |
| 237 | + | console.log(" The repository's refs moved since its last backup, so differences from it may be new work, not a fault."); |
| 238 | + | } |
| 239 | + | if (keep) console.log(` kept: ${work}`); |
| 240 | + | return fromChain.length === 0 && fromLive.length === 0 ? 0 : 1; |
| 241 | + | } finally { |
| 242 | + | if (!keep) rmSync(work, { recursive: true, force: true }); |
| 243 | + | } |
| 244 | + | } |
| 245 | + | |
| 246 | + | if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/backup-restore-drill.mjs")) { |
| 247 | + | main().then( |
| 248 | + | (code) => process.exit(code), |
| 249 | + | (error) => { |
| 250 | + | console.error(`drill: ${error.message}`); |
| 251 | + | process.exit(2); |
| 252 | + | }, |
| 253 | + | ); |
| 254 | + | } |