Sign in

flagon-io/g1t

public

g1t/scripts/ops/restore-to-gitstore.test.mjs

223 lines11,676 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a2013627e5ea4ab13'1// The fallback path end to end (docs/ARTIFACTS.md, R12): bundles cut as the
2// runner cuts them, restored into a git store's root, served by the git
3// store itself (deploy/self-host/gitstore/server.mjs, read-only), pushed to
4// while it serves, and reconciled back into the "live" repository.
5
6import assert from "node:assert/strict";
7import { execFileSync, spawn, spawnSync } from "node:child_process";
8import { createHash } from "node:crypto";
9import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
10import { tmpdir } from "node:os";
11import { join } from "node:path";
12import { test } from "node:test";
13import { fileURLToPath } from "node:url";
14
15import { parseRefs } from "./backup-restore-drill.mjs";
16import { CONFLICT_PREFIX, changedSince, locate, metaFor, reconcilePlan, repoDir } from "./restore-to-gitstore.mjs";
17
18const TOOL = fileURLToPath(new URL("./restore-to-gitstore.mjs", import.meta.url));
19const SERVER = fileURLToPath(new URL("../../deploy/self-host/gitstore/server.mjs", import.meta.url));
20const git = (cwd, ...args) => execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
21const tool = (...args) => spawnSync(process.execPath, [TOOL, ...args], { encoding: "utf8" });
22
23function commit(dir, file, text) {
24 writeFileSync(join(dir, file), text);
25 git(dir, "add", "--all");
26 git(dir, "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "--quiet", "-m", text);
27}
28
29function refsOf(dir) {
30 const refs = parseRefs(git(dir, "for-each-ref", "--format=%(objectname) %(refname)"));
31 refs.HEAD = git(dir, "rev-parse", "HEAD");
32 return refs;
33}
34
35/** A bucket holding one full backup of `origin` as `repo_1`, as the runner and repos service leave it. */
36function backUp(root, origin, id, storeKey) {
37 const mirror = join(root, `${id}-mirror.git`);
38 git(root, "clone", "--mirror", "--quiet", origin, mirror);
39 const dir = join(root, "bucket", "backups", id);
40 mkdirSync(dir, { recursive: true });
41 const bundle = join(dir, "1-full.bundle");
42 git(mirror, "bundle", "create", "--quiet", bundle, "--all");
43 const entry = {
44 id: "20261006T025300Z",
45 kind: "full",
46 key: `backups/${id}/1-full.bundle`,
47 created_at: "2026-10-06T02:53:00.000Z",
48 refs_version: 1,
49 refs: refsOf(mirror),
50 prerequisites: [],
51 size: statSync(bundle).size,
52 sha256: createHash("sha256").update(readFileSync(bundle)).digest("hex"),
53 };
54 const manifest = { version: 1, repo_id: id, store_key: storeKey, path: { namespace: "acme", name: "rocket" }, updated_at: "", chain: [entry], previous: [] };
55 writeFileSync(join(dir, "manifest.json"), JSON.stringify(manifest));
56 return join(root, "bucket");
57}
58
59/** The git store, serving `root`, until `stop()`. */
60async function serve(root, { readOnly }) {
61 const port = 41000 + Math.floor(Math.random() * 2000);
62 const secret = "0123456789abcdef0123456789abcdef";
63 const child = spawn(process.execPath, [SERVER], {
64 env: { ...process.env, GITSTORE_ROOT: root, GITSTORE_PORT: String(port), GITSTORE_SECRET: secret, GITSTORE_URL: `http://127.0.0.1:${port}`, GITSTORE_READ_ONLY: readOnly ? "1" : "" },
65 stdio: "ignore",
66 });
67 const url = `http://127.0.0.1:${port}`;
68 for (let tries = 0; tries < 100; tries++) {
69 try {
70 if ((await fetch(`${url}/healthz`)).ok) break;
71 } catch {}
72 await new Promise((resolve) => setTimeout(resolve, 100));
73 }
74 const api = async (method, path, body) => {
75 const response = await fetch(`${url}/api/repos${path}`, {
76 method,
77 headers: { "x-gitstore-secret": secret, ...(body ? { "content-type": "application/json" } : {}) },
78 body: body ? JSON.stringify(body) : undefined,
79 });
80 return { status: response.status, json: await response.json().catch(() => ({})) };
81 };
82 return { url, secret, api, stop: () => child.kill() };
83}
84
85test("keys, plans and what a restore records", () => {
86 assert.deepEqual(locate("acme--rocket"), { namespace: "g1t", name: "acme--rocket" });
87 assert.deepEqual(locate("g1t-us-1/pulls--pul_1"), { namespace: "g1t-us-1", name: "pulls--pul_1" });
88 assert.throws(() => locate("../x"), /not a store key/);
89 assert.equal(repoDir("/srv", "g1t", "acme--rocket").replaceAll("\\", "/"), "/srv/g1t/acme--rocket.git");
90 const a = "a".repeat(40);
91 const b = "b".repeat(40);
92 const c = "c".repeat(40);
93 const changes = changedSince({ "refs/heads/main": a, "refs/heads/old": a }, { "refs/heads/main": b, "refs/heads/new": c });
94 assert.deepEqual(changes, [
95 { ref: "refs/heads/main", base: a, now: b },
96 { ref: "refs/heads/new", base: null, now: c },
97 { ref: "refs/heads/old", base: a, now: null },
98 ]);
99 // Live still as backed up: moved. Live has it: nothing. Live moved too: kept beside.
100 assert.deepEqual(
101 reconcilePlan(changes, { "refs/heads/main": a, "refs/heads/old": c }).map((step) => [step.ref, step.action, step.kept ?? null]),
102 [
103 ["refs/heads/main", "move", null],
104 ["refs/heads/new", "move", null],
105 ["refs/heads/old", "conflict", null],
106 ],
107 );
108 assert.equal(reconcilePlan(changes, { "refs/heads/main": c })[0].kept, `${CONFLICT_PREFIX}heads/main`);
109 assert.equal(reconcilePlan(changes, { "refs/heads/main": b })[0].action, "same");
110 const manifest = { chain: [{ id: "e1", created_at: "t", refs: { HEAD: a, "refs/heads/main": a } }] };
111 const meta = metaFor({ id: "repo_1" }, manifest, "now", { id: "kept", createdAt: "then" });
112 assert.equal(meta.id, "kept");
113 assert.deepEqual(meta.restored.refs, { "refs/heads/main": a });
114 assert.equal(meta.restored.entry, "e1");
115});
116
117test("restored repositories are served read-only, and pushes taken later are reconciled", async () => {
118 const root = mkdtempSync(join(tmpdir(), "g1t-fallback-test-"));
119 let server = null;
120 try {
121 const origin = join(root, "origin");
122 mkdirSync(origin);
123 git(origin, "init", "--quiet", "--initial-branch=main");
124 commit(origin, "a.txt", "one");
125 git(origin, "tag", "-a", "v1", "-m", "v1");
126 const bucket = backUp(root, origin, "repo_1", "g1t-us-1/acme--rocket");
127 const index = join(root, "index.json");
128 writeFileSync(index, JSON.stringify([{ id: "repo_1", store: "g1t-us-1/acme--rocket", path: "acme/rocket" }, { id: "repo_2", store: "acme--gone", path: "acme/gone" }]));
129 const store = join(root, "store");
130
131 const first = tool("restore", "--into", store, "--bundles", bucket, "--index", index);
132 assert.equal(first.status, 0, first.stdout + first.stderr);
133 assert.match(first.stdout, /1 restored, 0 already current, 1 without a backup/);
134 const dir = repoDir(store, "g1t-us-1", "acme--rocket");
135 assert.ok(existsSync(join(dir, "g1t.json")));
136 assert.equal(git(dir, "rev-parse", "refs/heads/main"), git(origin, "rev-parse", "main"));
137 // Again: nothing to do.
138 const again = tool("restore", "--into", store, "--bundles", bucket, "--index", index);
139 assert.match(again.stdout, /0 restored, 1 already current/);
140 // Only a namespace asked for.
141 assert.match(tool("restore", "--into", store, "--bundles", bucket, "--index", index, "--namespace", "g1t-eu").stdout, /^0 repositories/m);
142
143 // Served as the repos service reads it: a namespaced key, a token, git.
144 server = await serve(store, { readOnly: true });
145 const key = encodeURIComponent("g1t-us-1/acme--rocket");
146 const info = await server.api("GET", `/${key}`);
147 assert.equal(info.status, 200);
148 assert.equal(info.json.remote, `${server.url}/git/g1t-us-1/acme--rocket.git`);
149 const read = await server.api("POST", `/${key}/tokens`, { scope: "read", ttl: 600 });
150 assert.equal(read.status, 200);
151 const header = `http.extraHeader=Authorization: Bearer ${read.json.plaintext}`;
152 const clone = join(root, "clone");
153 git(root, "-c", header, "clone", "--quiet", info.json.remote, clone);
154 assert.equal(git(clone, "rev-parse", "HEAD"), git(origin, "rev-parse", "main"));
155 // Read-only: no write token, no new repository.
156 assert.equal((await server.api("POST", `/${key}/tokens`, { scope: "write" })).json.code, "READ_ONLY");
157 assert.equal((await server.api("POST", "", { name: "g1t-us-1/new" })).json.code, "READ_ONLY");
158 assert.equal((await server.api("GET", `/${encodeURIComponent("../etc")}`)).json.code, "INVALID_REPO_NAME");
159 server.stop();
160 server = null;
161
162 // While it served with writes allowed, a push came in.
163 assert.match(tool("changed", "--into", store).stdout, /^0 repositories/m);
164 commit(clone, "b.txt", "pushed to the fallback");
165 git(clone, "push", "--quiet", dir, "HEAD:refs/heads/main", "HEAD:refs/heads/during");
166 const changed = tool("changed", "--into", store);
167 assert.match(changed.stdout, /g1t-us-1\/acme--rocket \(repo_1\)/);
168 assert.match(changed.stdout, /refs\/heads\/during/);
169
170 // The live repository still as backed up: the push goes back as it is.
171 const live = join(root, "live");
172 mkdirSync(join(live, "g1t-us-1"), { recursive: true });
173 git(root, "clone", "--bare", "--quiet", origin, repoDir(live, "g1t-us-1", "acme--rocket"));
174 const reconciled = tool("reconcile", "--into", store, "--live-root", live, "--no-bump");
175 assert.equal(reconciled.status, 0, reconciled.stdout + reconciled.stderr);
176 const liveDir = repoDir(live, "g1t-us-1", "acme--rocket");
177 assert.equal(git(liveDir, "rev-parse", "refs/heads/main"), git(clone, "rev-parse", "HEAD"));
178 assert.equal(git(liveDir, "rev-parse", "refs/heads/during"), git(clone, "rev-parse", "HEAD"));
179
180 // Moved on both sides: the live one stays, the fallback's goes beside it.
181 rmSync(liveDir, { recursive: true, force: true });
182 commit(origin, "c.txt", "pushed to Artifacts before the outage, after the backup");
183 git(root, "clone", "--bare", "--quiet", origin, liveDir);
184 const conflicted = tool("reconcile", "--into", store, "--live-root", live, "--no-bump");
185 assert.equal(conflicted.status, 3, conflicted.stdout + conflicted.stderr);
186 assert.match(conflicted.stdout, /moved on both sides/);
187 assert.equal(git(liveDir, "rev-parse", "refs/heads/main"), git(origin, "rev-parse", "main"));
188 assert.equal(git(liveDir, "rev-parse", "refs/fallback/heads/main"), git(clone, "rev-parse", "HEAD"));
189 } finally {
190 server?.stop();
191 rmSync(root, { recursive: true, force: true });
192 }
193});
194
195test("a git store over HTTP is restored into through its API", async () => {
196 const root = mkdtempSync(join(tmpdir(), "g1t-fallback-http-"));
197 let server = null;
198 try {
199 const origin = join(root, "origin");
200 mkdirSync(origin);
201 git(origin, "init", "--quiet", "--initial-branch=main");
202 commit(origin, "a.txt", "one");
203 git(origin, "branch", "dev");
204 const bucket = backUp(root, origin, "repo_1", "acme--rocket");
205 const index = join(root, "index.json");
206 writeFileSync(index, JSON.stringify([{ id: "repo_1", store: "acme--rocket", default_branch: "main" }]));
207 server = await serve(join(root, "store"), { readOnly: false });
208 const run = spawn(process.execPath, [TOOL, "restore", "--gitstore", server.url, "--bundles", bucket, "--index", index], {
209 env: { ...process.env, GITSTORE_SECRET: server.secret },
210 });
211 let out = "";
212 run.stdout.on("data", (chunk) => (out += chunk));
213 run.stderr.on("data", (chunk) => (out += chunk));
214 const status = await new Promise((resolve) => run.on("close", resolve));
215 assert.equal(status, 0, out);
216 // The default namespace's repositories are kept under it.
217 const dir = repoDir(join(root, "store"), "g1t", "acme--rocket");
218 assert.equal(git(dir, "rev-parse", "refs/heads/dev"), git(origin, "rev-parse", "dev"));
219 } finally {
220 server?.stop();
221 rmSync(root, { recursive: true, force: true });
222 }
223});

This file's history is long; its oldest lines are credited to the oldest commit read.