Sign in

flagon-io/g1t

public

g1t/services/repos/dev/clone-check.mjs

238 lines11,304 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a1b995daa94e4e1b7'1#!/usr/bin/env node
2// Clones a repository through the repos service twice, full and shallow,
3// over protocol v2 and v0, and checks that the second clone of each came
4// from the pack cache (src/pack_cache.rs) and matches the first. Then
5// moves the repository's refs version and checks the next clone misses.
6//
7// Runs everything on this machine: the self-hosted git store, and
8// `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc and
9// dev/stubs.jsonc, state kept in a temporary folder. Build first:
10//
11// cd services/repos && node ../../scripts/build-rust-worker.mjs
12// node dev/clone-check.mjs
13//
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue14// With `--s3`, packs are kept in MinIO instead of local R2, as a
15// self-hosted installation keeps them (PACK_STORE=s3): it starts MinIO in
16// Docker, makes the `g1t-git-packs` bucket with the same expiry rule the
17// compose file gives it, and checks that what was kept is there, whole,
18// with no upload left unfinished.
19//
20// node dev/clone-check.mjs --s3
21//
22// GITSTORE_PORT, REPOS_PORT and MINIO_PORT move it off 8799, 8791 and 9010.
23//
24// Needs node, git (with git-http-backend) and the repository's npm
25// packages; with `--s3`, Docker too.
Merge branch 'worktree-agent-a1b995daa94e4e1b7'26
27import { spawn, spawnSync } from "node:child_process";
28import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
29import { randomBytes } from "node:crypto";
30import { tmpdir } from "node:os";
31import { dirname, join, resolve } from "node:path";
32import { fileURLToPath } from "node:url";
33import { createRequire } from "node:module";
34
35const here = dirname(fileURLToPath(import.meta.url));
36const service = resolve(here, "..");
37const root = resolve(service, "../..");
38const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-"));
39const persist = join(work, "state");
40const SECRET = "dev-gitstore-secret-0123";
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue41// The ports, if something on this machine already has the defaults.
42const STORE_PORT = process.env.GITSTORE_PORT ?? "8799";
43const REPOS_PORT = process.env.REPOS_PORT ?? "8791";
44const STORE = `http://localhost:${STORE_PORT}`;
45const REPOS = `http://localhost:${REPOS_PORT}`;
46// dev/artifacts.jsonc, pointed at this run's git store.
47const ARTIFACTS_CONFIG = join(work, "artifacts.json");
48writeFileSync(
49 ARTIFACTS_CONFIG,
50 JSON.stringify({
51 name: "g1t-artifacts",
52 main: join(root, "deploy/self-host/workers/artifacts/index.js"),
53 compatibility_date: "2026-09-26",
54 vars: { GITSTORE_URL: STORE, GITSTORE_SECRET: SECRET },
55 }),
56);
Merge branch 'worktree-agent-a1b995daa94e4e1b7'57const KEY = "acme--rocket";
58const children = [];
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue59// --s3: packs in MinIO (see the top).
60const S3 = process.argv.includes("--s3");
61const MINIO = "g1t-clone-check-minio";
62const MINIO_PORT = process.env.MINIO_PORT ?? "9010";
63const MINIO_USER = "g1t";
64const MINIO_PASSWORD = "g1t-clone-check-secret";
65const PACKS_BUCKET = "g1t-git-packs";
66/** `mc` inside the MinIO container, against itself. */
67const mc = (args, options = {}) => run("docker", ["exec", MINIO, "mc", ...args], options);
Merge branch 'worktree-agent-a1b995daa94e4e1b7'68// Wrangler from the repository's packages, run with node: no shell to quote for.
69const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js");
70
71function run(command, args, options = {}) {
72 const done = spawnSync(command, args, { encoding: "utf8", ...options });
73 if (done.status !== 0 && !options.allowFail) {
74 throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`);
75 }
76 return done;
77}
78
79const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } });
80
81function start(command, args, options) {
82 const child = spawn(command, args, { ...options });
83 children.push(child);
84 return child;
85}
86
87async function waitFor(url, what) {
88 for (let i = 0; i < 120; i++) {
89 try {
90 const response = await fetch(url);
91 if (response.status < 500) return;
92 } catch {}
93 await new Promise((resolve) => setTimeout(resolve, 500));
94 }
95 throw new Error(`${what} did not start`);
96}
97
98const sql = (command) =>
99 run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], {
100 cwd: service,
101 env: { ...process.env, CI: "1" },
102 });
103
104/** Clones with `args`, and says how the pack was found and what came. */
105function clone(name, args) {
106 const dir = join(work, name);
107 const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" });
108 const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line));
109 const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]);
110 const head = git(["rev-parse", "HEAD"], dir).stdout.trim();
111 const files = git(["ls-tree", "-r", "HEAD"], dir).stdout;
112 const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim();
113 git(["fsck", "--no-progress"], dir);
114 return { pack, head, files, count };
115}
116
117const checks = [];
118function check(what, ok, detail = "") {
119 checks.push({ what, ok });
120 console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`);
121}
122
123function twice(label, args, depth) {
124 const first = clone(`${label}-1`, args);
125 const second = clone(`${label}-2`, args);
126 check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(","));
127 check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(","));
128 check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count);
129 if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count);
130 return second;
131}
132
133try {
134 // The git store, with a repository of a few commits.
135 start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], {
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue136 env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: STORE_PORT, GITSTORE_URL: STORE },
Merge branch 'worktree-agent-a1b995daa94e4e1b7'137 stdio: "inherit",
138 });
139 await waitFor(`${STORE}/healthz`, "the git store");
140 const api = (path, body) =>
141 fetch(`${STORE}/api/repos${path}`, {
142 method: "POST",
143 headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" },
144 body: JSON.stringify(body),
145 }).then((response) => response.json());
146 await api("", { name: KEY, defaultBranch: "main" });
147 const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext;
148 const seed = join(work, "seed");
149 git(["init", "-q", "-b", "main", seed]);
150 for (let i = 1; i <= 5; i++) {
151 writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`);
152 // 12 MB that does not compress, so a pack goes up in multipart parts.
153 if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024));
154 git(["add", "."], seed);
155 git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed);
156 }
157 git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed);
158
159 // The repos service, its database with the repository in it.
160 run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], {
161 cwd: service,
162 env: { ...process.env, CI: "1" },
163 });
164 sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)");
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue165
166 // MinIO, with the bucket and expiry rule deploy/self-host/docker-compose.yml makes.
167 const s3Vars = [];
168 if (S3) {
169 run("docker", ["rm", "-f", MINIO], { allowFail: true });
170 run("docker", [
171 "run", "-d", "--rm", "--name", MINIO, "-p", `${MINIO_PORT}:9000`,
172 "-e", `MINIO_ROOT_USER=${MINIO_USER}`, "-e", `MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}`,
173 process.env.MINIO_IMAGE ?? "pgsty/minio:latest", "server", "/data",
174 ]);
175 await waitFor(`http://localhost:${MINIO_PORT}/minio/health/ready`, "MinIO");
176 mc(["alias", "set", "local", "http://localhost:9000", MINIO_USER, MINIO_PASSWORD]);
177 mc(["mb", "--ignore-existing", `local/${PACKS_BUCKET}`]);
178 mc(["ilm", "rule", "add", "--prefix", "packs/", "--expire-days", "7", `local/${PACKS_BUCKET}`]);
179 for (const [name, value] of Object.entries({
180 PACK_STORE: "s3",
181 PACK_S3_BUCKET: PACKS_BUCKET,
182 S3_ENDPOINT: `http://localhost:${MINIO_PORT}`,
183 S3_REGION: "us-east-1",
184 S3_ACCESS_KEY_ID: MINIO_USER,
185 S3_SECRET_ACCESS_KEY: MINIO_PASSWORD,
186 })) {
187 s3Vars.push("--var", `${name}:${value}`);
188 }
189 }
Merge branch 'worktree-agent-a1b995daa94e4e1b7'190 start(
191 "node",
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue192 [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", ARTIFACTS_CONFIG, "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", REPOS_PORT, ...s3Vars],
Merge branch 'worktree-agent-a1b995daa94e4e1b7'193 { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] },
194 );
195 await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev");
196
197 twice("full, v2", [], 5);
198 twice("shallow, v2", ["--depth=1"], 1);
199 twice("full, v0", ["-c", "protocol.version=0"], 5);
200 twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1);
201
202 // A change to the refs: the next clone goes to the store.
203 sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'");
204 // The service keeps a row it read a moment ago for the same clone's next request.
205 await new Promise((resolve) => setTimeout(resolve, 6000));
206 const after = clone("after-refs", ["--depth=1"]);
207 check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(","));
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue208
209 if (S3) {
210 // Fills finish after git has its answer: give the last one a moment.
211 await new Promise((resolve) => setTimeout(resolve, 3000));
212 const listed = mc(["ls", "--recursive", "--json", `local/${PACKS_BUCKET}/packs/`]).stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
213 const sizes = listed.map((entry) => entry.size);
214 // Nine clones: four kinds twice, each kept once, and one more after the refs moved.
215 check("the packs are in MinIO, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`);
216 check("the full clone's pack went up in parts", sizes.some((size) => size > 5 * 1024 * 1024), `largest ${Math.max(...sizes)}`);
217 const incomplete = mc(["ls", "--recursive", "--incomplete", `local/${PACKS_BUCKET}`]).stdout.trim();
218 check("no upload is left unfinished", incomplete === "", incomplete);
219 const rules = mc(["ilm", "rule", "ls", "--json", `local/${PACKS_BUCKET}`]).stdout;
220 check("the bucket expires packs after 7 days", /"Days":\s*7/.test(rules) && rules.includes("packs/"));
221 }
Merge branch 'worktree-agent-a1b995daa94e4e1b7'222} catch (error) {
223 console.error(error);
224 checks.push({ what: "ran", ok: false });
225} finally {
226 for (const child of children) {
227 if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
228 else child.kill();
229 }
Self-hosting: the clone pack cache on S3 (MinIO, expiring), the API and MCP on their own port with PUBLIC_URL-derived addresses across the site, API and mail, scheduler --once, and smoke.sh covering pull requests from branches and forks and the merge queue230 if (S3) run("docker", ["rm", "-f", MINIO], { allowFail: true });
Merge branch 'worktree-agent-a1b995daa94e4e1b7'231 try {
232 rmSync(work, { recursive: true, force: true });
233 } catch {}
234}
235
236const failed = checks.filter((c) => !c.ok);
237console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`);
238process.exit(failed.length ? 1 : 0);

This file's history is long; its oldest lines are credited to the oldest commit read.