Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
259 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

The workspace cost alert compares only days that carry their value, not the days before it was kept1#!/usr/bin/env node
The docs folder is gone, and what it held lives where people read it: how a self-hosted g1t runs and how to deploy g1t to Cloudflare are pages on docs.g1t.sh under Run g1t yourself, and speed, rate limits and operating g1t.sh are sections of CONTRIBUTING.md; code that cited a file in docs/ now points to the page or section that covers it, or says what it means itself, and applied migrations and the runner images are left as they were.2// The restore drill for nightly backups (services/repos/src/backups.rs).
3// Picks a repository, downloads its
The workspace cost alert compares only days that carry their value, not the days before it was kept4// manifest and bundle chain from the g1t-backups bucket, rebuilds the
5// repository from them in a temporary directory, and compares every ref
6// with the live repository. Exits 1 on any difference, 2 when it could not
7// run.
8//
9// Read-only: SELECTs against the g1t-repos database, reads of the bucket,
10// and `git ls-remote` of the live repository. Nothing is written anywhere
11// but the temporary directory, which is removed unless you pass --keep.
12//
13// node scripts/ops/backup-restore-drill.mjs # a repository unchanged since its last backup
14// node scripts/ops/backup-restore-drill.mjs --repo acme/rocket
15// node scripts/ops/backup-restore-drill.mjs --repo-id repo_... --bundles ./copy --live /srv/git/acme--rocket.git
16//
17// Options:
18// --repo <workspace/name> the repository; default: one picked at random
19// among those whose refs have not moved since
20// their last backup, so any difference is the
21// backup's.
22// --repo-id <id> the repository by id (no database needed with --bundles).
23// --bundles <dir> read the bucket from a local copy (`backups/<id>/...`
Merge branch 'worktree-agent-af58ac8933b0dd125'24// under it, as `aws s3 sync` or `rclone copy` leave it)
25// instead of R2, e.g. a self-hosted store's.
The workspace cost alert compares only days that carry their value, not the days before it was kept26// --live <url or path> the live repository to compare with; default
27// https://g1t.sh/<workspace>/<name>.git.
28// --keep keep the temporary directory, and say where it is.
29//
30// The live repository is read as G1T_USER with G1T_TOKEN (an access token
31// with code:read) when they are set, which private repositories need. The
32// database and the bucket are read through Wrangler, as you are logged in
33// (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN when that is set.
34
35import { createHash } from "node:crypto";
36import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
37import { tmpdir } from "node:os";
38import { join } from "node:path";
39
40import { exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
41import { ROOT } from "../deploy/stack.mjs";
42
43const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
44const DATABASE = "g1t-repos";
45const BUCKET = process.env.BACKUP_BUCKET || "g1t-backups";
46const MANIFEST_VERSION = 1;
47const STAGING = "refs/drill-staging";
48
49/** Runs git; resolves with its output, or throws with what it said. */
50async function git(args, { cwd, input } = {}) {
51 const { code, out } = await exec("git", args, { cwd, input });
52 if (code !== 0) throw new Error(`git ${args.find((arg) => !arg.startsWith("-")) ?? ""} failed: ${out.trim().slice(-600)}`);
53 return out.trim();
54}
55
56/** A file's SHA-256, read as a stream: a bundle can be a gigabyte. */
57async function sha256Of(file) {
58 const hash = createHash("sha256");
59 for await (const chunk of createReadStream(file)) hash.update(chunk);
60 return hash.digest("hex");
61}
62
63/** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */
64export function parseRefs(listing) {
65 const refs = {};
66 for (const line of listing.split(/\r?\n/)) {
67 const match = /^([0-9a-f]{40,64})\s+(\S+)$/.exec(line.trim());
68 if (!match || match[2].endsWith("^{}")) continue;
69 refs[match[2]] = match[1];
70 }
71 return refs;
72}
73
74/** Every ref of the repository in `dir`, and HEAD. */
75async function refsOf(dir) {
76 const refs = parseRefs(await git(["for-each-ref", "--format=%(objectname) %(refname)"], { cwd: dir }));
77 const head = await git(["rev-parse", "--verify", "--quiet", "HEAD"], { cwd: dir }).catch(() => "");
78 if (head) refs.HEAD = head;
79 return refs;
80}
81
82/** The refs that differ between `want` and `have`: [{ ref, want, have }], `null` for absent. */
83export function compareRefs(want, have) {
84 const names = [...new Set([...Object.keys(want), ...Object.keys(have)])].sort();
85 return names
86 .filter((ref) => want[ref] !== have[ref])
87 .map((ref) => ({ ref, want: want[ref] ?? null, have: have[ref] ?? null }));
88}
89
90/** The branch HEAD should name: one at HEAD's commit, `main` or `master` first. */
91export function headBranch(refs) {
92 if (!refs.HEAD) return null;
93 const branches = Object.keys(refs).filter((ref) => ref.startsWith("refs/heads/") && refs[ref] === refs.HEAD);
94 return ["refs/heads/main", "refs/heads/master"].find((ref) => branches.includes(ref)) ?? branches.sort()[0] ?? null;
95}
96
97/** Whether a manifest can be read by this drill. */
98export function readManifest(text) {
99 const manifest = JSON.parse(text);
100 if (manifest.version !== MANIFEST_VERSION) throw new Error(`manifest version ${manifest.version}; this drill reads ${MANIFEST_VERSION}`);
101 if (!Array.isArray(manifest.chain) || manifest.chain.length === 0) throw new Error("the manifest lists no backups");
102 if (manifest.chain[0].kind !== "full") throw new Error("the chain does not start with a full backup");
103 return manifest;
104}
105
106/**
107 * Rebuilds the repository the manifest's chain describes into `dir`, a
108 * new bare repository: each bundle in order, checked against its size and
109 * SHA-256 and verified by git, fetched without following tags; then every
110 * ref set to what the last entry says, and nothing else kept. `fetchObject`
111 * saves one object of the bucket to a file and resolves with its path.
112 */
113export async function restore(manifest, fetchObject, dir, work) {
114 await git(["init", "--quiet", "--bare", dir]);
115 for (const entry of manifest.chain) {
116 if (!entry.key) continue;
117 const file = await fetchObject(entry.key, join(work, `${entry.id}.bundle`));
118 const size = statSync(file).size;
119 if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`);
120 if (entry.sha256) {
121 const sha256 = await sha256Of(file);
122 if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`);
123 }
124 await git(["bundle", "verify", "--quiet", file], { cwd: dir });
125 await git(["fetch", "--quiet", "--no-tags", file, `+refs/*:${STAGING}/${entry.id}/*`], { cwd: dir });
126 }
127 const last = manifest.chain.at(-1).refs;
128 const updates = Object.entries(last)
129 .filter(([ref]) => ref !== "HEAD")
130 .map(([ref, hash]) => `update ${ref} ${hash}\n`)
131 .join("");
132 const staged = await git(["for-each-ref", "--format=delete %(refname)", `${STAGING}/`], { cwd: dir });
133 const commands = [updates.trimEnd(), staged].filter(Boolean).join("\n");
134 if (commands) await git(["update-ref", "--stdin"], { cwd: dir, input: `${commands}\n` });
135 const head = headBranch(last);
136 if (head) await git(["symbolic-ref", "HEAD", head], { cwd: dir });
137 // Every object every ref reaches is there.
138 await git(["fsck", "--no-progress", "--connectivity-only"], { cwd: dir });
139 return refsOf(dir);
140}
141
142// ---------------------------------------------------------------------
143
144async function d1(sql) {
145 const env = wranglerEnv();
146 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
147 cwd: join(ROOT, "services/repos"),
148 env,
149 });
150 if (code !== 0) throw new Error(out.slice(-600));
151 return jsonFrom(out)[0]?.results ?? [];
152}
153
154const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
155
156/** The repository to drill, and whether its refs moved since its last backup. */
157async function pick({ repo, repoId }) {
158 const select = `SELECT r.id, r.namespace, r.name, r.refs_version, b.refs_version AS backed_version,
159 coalesce(r.refs_open_until, 0) > coalesce(b.backed_up_ms, 0) AS opened
160 FROM repo_backups b JOIN repos r ON r.id = b.repo_id
161 WHERE b.last_entry IS NOT NULL AND r.deleted_at IS NULL`;
162 let rows;
163 if (repoId) rows = await d1(`${select} AND r.id = ${quoted(repoId)}`);
164 else if (repo) {
165 const [namespace, name] = repo.toLowerCase().split("/");
166 rows = await d1(`${select} AND r.namespace = ${quoted(namespace)} AND r.name = ${quoted(name)}`);
167 } else {
168 rows = await d1(`${select} AND b.refs_version = r.refs_version AND coalesce(r.refs_open_until, 0) <= coalesce(b.backed_up_ms, 0)
169 ORDER BY random() LIMIT 1`);
170 }
171 const row = rows[0];
172 if (!row) throw new Error(repo || repoId ? `no backup of ${repo ?? repoId}` : "no repository has a backup yet");
173 return {
174 id: row.id,
175 path: `${row.namespace}/${row.name}`,
176 moved: row.refs_version !== row.backed_version || Boolean(row.opened),
177 };
178}
179
180/** Saves one object of the bucket to `file`. */
181function bucketReader(localCopy) {
182 if (localCopy) return async (key) => join(localCopy, key);
183 return async (key, file) => {
184 const env = wranglerEnv();
185 const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env });
186 if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
187 return file;
188 };
189}
190
191/** The live repository's refs, as a clone would see them. */
192async function liveRefs(live) {
193 const args = [];
194 if (process.env.G1T_TOKEN && /^https?:/.test(live)) {
195 const user = process.env.G1T_USER || "g1t";
196 const basic = Buffer.from(`${user}:${process.env.G1T_TOKEN}`).toString("base64");
197 args.push("-c", `http.extraHeader=Authorization: Basic ${basic}`);
198 }
199 return parseRefs(await git([...args, "ls-remote", live]));
200}
201
202async function main() {
203 const args = process.argv.slice(2);
204 const option = (name) => {
205 const at = args.indexOf(name);
206 return at >= 0 ? args[at + 1] : undefined;
207 };
208 const keep = args.includes("--keep");
209 const localCopy = option("--bundles");
210 const target =
211 localCopy && option("--repo-id")
212 ? { id: option("--repo-id"), path: option("--repo") ?? null, moved: false }
213 : await pick({ repo: option("--repo"), repoId: option("--repo-id") });
214 const live = option("--live") ?? (target.path ? `https://g1t.sh/${target.path}.git` : null);
215 if (!live) throw new Error("say which live repository to compare with: --live, or --repo");
216
217 const work = mkdtempSync(join(tmpdir(), "g1t-drill-"));
218 try {
219 const read = bucketReader(localCopy);
220 const manifest = readManifest(readFileSync(await read(`backups/${target.id}/manifest.json`, join(work, "manifest.json")), "utf8"));
221 const started = Date.now();
222 const restored = await restore(manifest, read, join(work, "restored.git"), work);
223 const seconds = ((Date.now() - started) / 1000).toFixed(1);
224 const last = manifest.chain.at(-1);
225 const bytes = manifest.chain.reduce((sum, entry) => sum + (entry.size ?? 0), 0);
226 console.log(`${target.path ?? target.id}: ${manifest.chain.length} backups (${bytes} bytes), the last ${last.created_at}, restored in ${seconds}s`);
227
228 const fromChain = compareRefs(last.refs, restored);
229 const fromLive = compareRefs(await liveRefs(live), restored);
230 for (const [what, differences] of [
231 ["the manifest", fromChain],
232 ["the live repository", fromLive],
233 ]) {
234 if (differences.length === 0) {
235 console.log(` every ref matches ${what} (${Object.keys(restored).length} refs)`);
236 continue;
237 }
238 console.log(` ${differences.length} refs differ from ${what}:`);
239 for (const { ref, want, have } of differences) console.log(` ${ref}: ${what} ${want ?? "(none)"}, restored ${have ?? "(none)"}`);
240 }
241 if (target.moved && fromLive.length > 0) {
242 console.log(" The repository's refs moved since its last backup, so differences from it may be new work, not a fault.");
243 }
244 if (keep) console.log(` kept: ${work}`);
245 return fromChain.length === 0 && fromLive.length === 0 ? 0 : 1;
246 } finally {
247 if (!keep) rmSync(work, { recursive: true, force: true });
248 }
249}
250
251if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/backup-restore-drill.mjs")) {
252 main().then(
253 (code) => process.exit(code),
254 (error) => {
255 console.error(`drill: ${error.message}`);
256 process.exit(2);
257 },
258 );
259}

This file's history is long; its oldest lines are credited to the oldest commit read.