| 1 | #!/usr/bin/env node |
| 2 | // The restore drill for nightly backups (services/repos/src/backups.rs). |
| 3 | // Picks a repository, downloads its |
| 4 | // manifest and bundle chain from the g1t-backups bucket, rebuilds the |
| 5 | // repository from them in a temporary directory, and compares every ref |
| 6 | // with the live repository. Exits 1 on any difference, 2 when it could not |
| 7 | // run. |
| 8 | // |
| 9 | // Read-only: SELECTs against the g1t-repos database, reads of the bucket, |
| 10 | // and `git ls-remote` of the live repository. Nothing is written anywhere |
| 11 | // but the temporary directory, which is removed unless you pass --keep. |
| 12 | // |
| 13 | // node scripts/ops/backup-restore-drill.mjs # a repository unchanged since its last backup |
| 14 | // node scripts/ops/backup-restore-drill.mjs --repo acme/rocket |
| 15 | // node scripts/ops/backup-restore-drill.mjs --repo-id repo_... --bundles ./copy --live /srv/git/acme--rocket.git |
| 16 | // |
| 17 | // Options: |
| 18 | // --repo <workspace/name> the repository; default: one picked at random |
| 19 | // among those whose refs have not moved since |
| 20 | // their last backup, so any difference is the |
| 21 | // backup's. |
| 22 | // --repo-id <id> the repository by id (no database needed with --bundles). |
| 23 | // --bundles <dir> read the bucket from a local copy (`backups/<id>/...` |
| 24 | // under it, as `aws s3 sync` or `rclone copy` leave it) |
| 25 | // instead of R2, e.g. a self-hosted store's. |
| 26 | // --live <url or path> the live repository to compare with; default |
| 27 | // https://g1t.sh/<workspace>/<name>.git. |
| 28 | // --keep keep the temporary directory, and say where it is. |
| 29 | // |
| 30 | // The live repository is read as G1T_USER with G1T_TOKEN (an access token |
| 31 | // with code:read) when they are set, which private repositories need. The |
| 32 | // database and the bucket are read through Wrangler, as you are logged in |
| 33 | // (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN when that is set. |
| 34 | |
| 35 | import { createHash } from "node:crypto"; |
| 36 | import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; |
| 37 | import { tmpdir } from "node:os"; |
| 38 | import { join } from "node:path"; |
| 39 | |
| 40 | import { exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs"; |
| 41 | import { ROOT } from "../deploy/stack.mjs"; |
| 42 | |
| 43 | const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js"); |
| 44 | const DATABASE = "g1t-repos"; |
| 45 | const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups"; |
| 46 | const MANIFEST_VERSION = 1; |
| 47 | const STAGING = "refs/drill-staging"; |
| 48 | |
| 49 | /** Runs git; resolves with its output, or throws with what it said. */ |
| 50 | async function git(args, { cwd, input } = {}) { |
| 51 | const { code, out } = await exec("git", args, { cwd, input }); |
| 52 | if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`); |
| 53 | return out.trim(); |
| 54 | } |
| 55 | |
| 56 | /** A file's SHA-256, read as a stream: a bundle can be a gigabyte. */ |
| 57 | async function sha256Of(file) { |
| 58 | const hash = createHash("sha256"); |
| 59 | for await (const chunk of createReadStream(file)) hash.update(chunk); |
| 60 | return hash.digest("hex"); |
| 61 | } |
| 62 | |
| 63 | /** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */ |
| 64 | export function parseRefs(listing) { |
| 65 | const refs = {}; |
| 66 | for (const line of listing.split(/\r?\n/)) { |
| 67 | const match = /^([0-9a-f]{40,64})\s+(\S+)$/.exec(line.trim()); |
| 68 | if (!match || match[2].endsWith("^{}")) continue; |
| 69 | refs[match[2]] = match[1]; |
| 70 | } |
| 71 | return refs; |
| 72 | } |
| 73 | |
| 74 | /** Every ref of the repository in `dir`, and HEAD. */ |
| 75 | async function refsOf(dir) { |
| 76 | const refs = parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir })); |
| 77 | const head = await git(["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: dir }).catch(() => ""); |
| 78 | if (head) refs.HEAD = head; |
| 79 | return refs; |
| 80 | } |
| 81 | |
| 82 | /** The refs that differ between `want` and `have`: [{ ref, want, have }], `null` for absent. */ |
| 83 | export function compareRefs(want, have) { |
| 84 | const names = [...new Set([...Object.keys(want), ...Object.keys(have)])].sort(); |
| 85 | return names.filter((ref) => want[ref] !== have[ref]).map((ref) => ({ ref, want: want[ref] ?? null, have: have[ref] ?? null })); |
| 86 | } |
| 87 | |
| 88 | /** The branch HEAD should name: one at HEAD's commit, `main` or `master` first. */ |
| 89 | export function headBranch(refs) { |
| 90 | if (!refs.HEAD) return null; |
| 91 | const branches = Object.keys(refs).filter((ref) => ref.startsWith("refs/heads/") && refs[ref] === refs.HEAD); |
| 92 | return ["refs/heads/main", "refs/heads/master"].find((ref) => branches.includes(ref)) ?? branches.sort()[0] ?? null; |
| 93 | } |
| 94 | |
| 95 | /** Whether a manifest can be read by this drill. */ |
| 96 | export function readManifest(text) { |
| 97 | const manifest = JSON.parse(text); |
| 98 | if (manifest.version !== MANIFEST_VERSION) throw new Error(`manifest version ${manifest.version}; this drill reads ${MANIFEST_VERSION}`); |
| 99 | if (!Array.isArray(manifest.chain) || manifest.chain.length === 0) throw new Error("the manifest lists no backups"); |
| 100 | if (manifest.chain[0].kind !== "full") throw new Error("the chain does not start with a full backup"); |
| 101 | return manifest; |
| 102 | } |
| 103 | |
| 104 | /** |
| 105 | * Rebuilds the repository the manifest's chain describes into `dir`, a |
| 106 | * new bare repository: each bundle in order, checked against its size and |
| 107 | * SHA-256 and verified by git, fetched without following tags; then every |
| 108 | * ref set to what the last entry says, and nothing else kept. `fetchObject` |
| 109 | * saves one object of the bucket to a file and resolves with its path. |
| 110 | */ |
| 111 | export async function restore(manifest, fetchObject, dir, work) { |
| 112 | await git(["init", "--quiet", "--bare", dir]); |
| 113 | for (const entry of manifest.chain) { |
| 114 | if (!entry.key) continue; |
| 115 | const file = await fetchObject(entry.key, join(work, `${entry.id}.bundle`)); |
| 116 | const size = statSync(file).size; |
| 117 | if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`); |
| 118 | if (entry.sha256) { |
| 119 | const sha256 = await sha256Of(file); |
| 120 | if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`); |
| 121 | } |
| 122 | await git(["bundle", "verify", "--quiet", file], { cwd: dir }); |
| 123 | await git(["fetch", "--quiet", "--no-tags", file, `+refs/*:${STAGING}/${entry.id}/*`], { cwd: dir }); |
| 124 | } |
| 125 | const last = manifest.chain.at(-1).refs; |
| 126 | const updates = Object.entries(last) |
| 127 | .filter(([ref]) => ref !== "HEAD") |
| 128 | .map(([ref, hash]) => `update ${ref} ${hash}\n`) |
| 129 | .join(""); |
| 130 | const staged = await git(["for-each-ref", "--format=delete %(refname)", `${STAGING}/`], { cwd: dir }); |
| 131 | const commands = [updates.trimEnd(), staged].filter(Boolean).join("\n"); |
| 132 | if (commands) await git(["update-ref", "--stdin"], { cwd: dir, input: `${commands}\n` }); |
| 133 | const head = headBranch(last); |
| 134 | if (head) await git(["symbolic-ref", "HEAD", head], { cwd: dir }); |
| 135 | // Every object every ref reaches is there. |
| 136 | await git(["fsck", "--no-progress", "--connectivity-only"], { cwd: dir }); |
| 137 | return refsOf(dir); |
| 138 | } |
| 139 | |
| 140 | // --------------------------------------------------------------------- |
| 141 | |
| 142 | async function d1(sql) { |
| 143 | const env = wranglerEnv(); |
| 144 | const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], { |
| 145 | cwd: join(ROOT, "services/repos"), |
| 146 | env, |
| 147 | }); |
| 148 | if (code !== 0) throw new Error(out.slice(-600)); |
| 149 | return jsonFrom(out)[0]?.results ?? []; |
| 150 | } |
| 151 | |
| 152 | const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`; |
| 153 | |
| 154 | /** The repository to drill, and whether its refs moved since its last backup. */ |
| 155 | async function pick({ repo, repoId }) { |
| 156 | const select = `SELECT r.id, r.namespace, r.name, r.refs_version, b.refs_version AS backed_version, |
| 157 | coalesce(r.refs_open_until, 0) > coalesce(b.backed_up_ms, 0) AS opened |
| 158 | FROM repo_backups b JOIN repos r ON r.id = b.repo_id |
| 159 | WHERE b.last_entry IS NOT NULL AND r.deleted_at IS NULL`; |
| 160 | let rows; |
| 161 | if (repoId) rows = await d1(`${select} AND r.id = ${quoted(repoId)}`); |
| 162 | else if (repo) { |
| 163 | const [namespace, name] = repo.toLowerCase().split("/"); |
| 164 | rows = await d1(`${select} AND r.namespace = ${quoted(namespace)} AND r.name = ${quoted(name)}`); |
| 165 | } else { |
| 166 | rows = await d1(`${select} AND b.refs_version = r.refs_version AND coalesce(r.refs_open_until, 0) <= coalesce(b.backed_up_ms, 0) |
| 167 | ORDER BY random() LIMIT 1`); |
| 168 | } |
| 169 | const row = rows[0]; |
| 170 | if (!row) throw new Error(repo || repoId ? `no backup of ${repo ?? repoId}` : "no repository has a backup yet"); |
| 171 | return { |
| 172 | id: row.id, |
| 173 | path: `${row.namespace}/${row.name}`, |
| 174 | moved: row.refs_version !== row.backed_version || Boolean(row.opened), |
| 175 | }; |
| 176 | } |
| 177 | |
| 178 | /** Saves one object of the bucket to `file`. */ |
| 179 | function bucketReader(localCopy) { |
| 180 | if (localCopy) return async (key) => join(localCopy, key); |
| 181 | return async (key, file) => { |
| 182 | const env = wranglerEnv(); |
| 183 | const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env }); |
| 184 | if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`); |
| 185 | return file; |
| 186 | }; |
| 187 | } |
| 188 | |
| 189 | /** The live repository's refs, as a clone would see them. */ |
| 190 | async function liveRefs(live) { |
| 191 | const args = []; |
| 192 | if (process.env.G1T_TOKEN && /^https?:/.test(live)) { |
| 193 | const user = process.env.G1T_USER || "g1t"; |
| 194 | const basic = Buffer.from(`${user}:${process.env.G1T_TOKEN}`).toString("base64"); |
| 195 | args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`); |
| 196 | } |
| 197 | return parseRefs(await git([...args, "ls-remote", live])); |
| 198 | } |
| 199 | |
| 200 | async function main() { |
| 201 | const args = process.argv.slice(2); |
| 202 | const option = (name) => { |
| 203 | const at = args.indexOf(name); |
| 204 | return at >= 0 ? args[at + 1] : undefined; |
| 205 | }; |
| 206 | const keep = args.includes("--keep"); |
| 207 | const localCopy = option("--bundles"); |
| 208 | const target = |
| 209 | localCopy && option("--repo-id") |
| 210 | ? { id: option("--repo-id"), path: option("--repo") ?? null, moved: false } |
| 211 | : await pick({ repo: option("--repo"), repoId: option("--repo-id") }); |
| 212 | const live = option("--live") ?? (target.path ? `https://g1t.sh/${target.path}.git` : null); |
| 213 | if (!live) throw new Error("say which live repository to compare with: --live, or --repo"); |
| 214 | |
| 215 | const work = mkdtempSync(join(tmpdir(), "g1t-drill-")); |
| 216 | try { |
| 217 | const read = bucketReader(localCopy); |
| 218 | const manifest = readManifest(readFileSync(await read(`backups/${target.id}/manifest.json`, join(work, "manifest.json")), "utf8")); |
| 219 | const started = Date.now(); |
| 220 | const restored = await restore(manifest, read, join(work, "restored.git"), work); |
| 221 | const seconds = ((Date.now() - started) / 1000).toFixed(1); |
| 222 | const last = manifest.chain.at(-1); |
| 223 | const bytes = manifest.chain.reduce((sum, entry) => sum + (entry.size ?? 0), 0); |
| 224 | console.log(`${target.path ?? target.id}: ${manifest.chain.length} backups (${bytes} bytes), the last ${last.created_at}, restored in ${seconds}s`); |
| 225 | |
| 226 | const fromChain = compareRefs(last.refs, restored); |
| 227 | const fromLive = compareRefs(await liveRefs(live), restored); |
| 228 | for (const [what, differences] of [ |
| 229 | ["the manifest", fromChain], |
| 230 | ["the live repository", fromLive], |
| 231 | ]) { |
| 232 | if (differences.length === 0) { |
| 233 | console.log(` every ref matches ${what} (${Object.keys(restored).length} refs)`); |
| 234 | continue; |
| 235 | } |
| 236 | console.log(` ${differences.length} refs differ from ${what}:`); |
| 237 | for (const { ref, want, have } of differences) console.log(` ${ref}: ${what} ${want ?? "(none)"}, restored ${have ?? "(none)"}`); |
| 238 | } |
| 239 | if (target.moved && fromLive.length > 0) { |
| 240 | console.log(" The repository's refs moved since its last backup, so differences from it may be new work, not a fault."); |
| 241 | } |
| 242 | if (keep) console.log(` kept: ${work}`); |
| 243 | return fromChain.length === 0 && fromLive.length === 0 ? 0 : 1; |
| 244 | } finally { |
| 245 | if (!keep) rmSync(work, { recursive: true, force: true }); |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/backup-restore-drill.mjs")) { |
| 250 | main().then( |
| 251 | (code) => process.exit(code), |
| 252 | (error) => { |
| 253 | console.error(`drill: ${error.message}`); |
| 254 | process.exit(2); |
| 255 | }, |
| 256 | ); |
| 257 | } |