Sign in

flagon-io/g1t

public

g1t/services/repos/dev/clone-check.mjs

261 lines12,867 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a1b995daa94e4e1b7'1#!/usr/bin/env node
2// Clones a repository through the repos service twice, full and shallow,
3// over protocol v2 and v0, and checks that the second clone of each came
4// from the pack cache (src/pack_cache.rs) and matches the first. Then
5// moves the repository's refs version and checks the next clone misses.
6//
7// Runs everything on this machine: the self-hosted git store, and
8// `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc and
9// dev/stubs.jsonc, state kept in a temporary folder. Build first:
10//
11// cd services/repos && node ../../scripts/build-rust-worker.mjs
12// node dev/clone-check.mjs
13//
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI14// With `--s3`, packs are kept in RustFS instead of local R2, as a
15// self-hosted installation keeps them (PACK_STORE=s3): it starts RustFS in
16// Docker, makes the `g1t-git-packs` bucket with the same lifecycle rule the
17// compose file gives it (with the AWS CLI, as the compose file does), and
18// checks that what was kept is there, whole, with no upload left
19// unfinished.
Merge branch 'worktree-agent-aaf03bdceac799c89'20//
21// node dev/clone-check.mjs --s3
22//
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI23// GITSTORE_PORT, REPOS_PORT and S3_PORT move it off 8799, 8791 and 9010.
24// RUSTFS_IMAGE and AWS_CLI_IMAGE choose other images.
Merge branch 'worktree-agent-aaf03bdceac799c89'25//
26// Needs node, git (with git-http-backend) and the repository's npm
27// packages; with `--s3`, Docker too.
Merge branch 'worktree-agent-a1b995daa94e4e1b7'28
29import { spawn, spawnSync } from "node:child_process";
30import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
31import { randomBytes } from "node:crypto";
32import { tmpdir } from "node:os";
33import { dirname, join, resolve } from "node:path";
34import { fileURLToPath } from "node:url";
35import { createRequire } from "node:module";
36
37const here = dirname(fileURLToPath(import.meta.url));
38const service = resolve(here, "..");
39const root = resolve(service, "../..");
40const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-"));
41const persist = join(work, "state");
42const SECRET = "dev-gitstore-secret-0123";
Merge branch 'worktree-agent-aaf03bdceac799c89'43// The ports, if something on this machine already has the defaults.
44const STORE_PORT = process.env.GITSTORE_PORT ?? "8799";
45const REPOS_PORT = process.env.REPOS_PORT ?? "8791";
46const STORE = `http://localhost:${STORE_PORT}`;
47const REPOS = `http://localhost:${REPOS_PORT}`;
48// dev/artifacts.jsonc, pointed at this run's git store.
49const ARTIFACTS_CONFIG = join(work, "artifacts.json");
50writeFileSync(
51 ARTIFACTS_CONFIG,
52 JSON.stringify({
53 name: "g1t-artifacts",
54 main: join(root, "deploy/self-host/workers/artifacts/index.js"),
55 compatibility_date: "2026-09-26",
56 vars: { GITSTORE_URL: STORE, GITSTORE_SECRET: SECRET },
57 }),
58);
Merge branch 'worktree-agent-a1b995daa94e4e1b7'59const KEY = "acme--rocket";
60const children = [];
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI61// --s3: packs in RustFS (see the top). The images are the compose file's.
Merge branch 'worktree-agent-aaf03bdceac799c89'62const S3 = process.argv.includes("--s3");
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI63const STORAGE = "g1t-clone-check-rustfs";
64const S3_PORT = process.env.S3_PORT ?? "9010";
65const S3_USER = "g1t";
66const S3_PASSWORD = "g1t-clone-check-secret";
67const RUSTFS_IMAGE = process.env.RUSTFS_IMAGE ?? "rustfs/rustfs:1.0.1";
68const AWS_CLI_IMAGE = process.env.AWS_CLI_IMAGE ?? "amazon/aws-cli:2.37.10";
Merge branch 'worktree-agent-aaf03bdceac799c89'69const PACKS_BUCKET = "g1t-git-packs";
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI70/** The AWS CLI's `s3api`, in a container on the store's network, against it. */
71const s3api = (args, options = {}) =>
72 run("docker", [
73 "run", "--rm", "--network", `container:${STORAGE}`,
74 "-e", `AWS_ACCESS_KEY_ID=${S3_USER}`, "-e", `AWS_SECRET_ACCESS_KEY=${S3_PASSWORD}`, "-e", "AWS_DEFAULT_REGION=us-east-1",
75 AWS_CLI_IMAGE, "--endpoint-url", "http://localhost:9000", "--output", "json", "s3api", ...args,
76 ], options);
77/** The same, its answer parsed. */
78const s3json = (args) => JSON.parse(s3api(args).stdout.trim() || "{}");
Merge branch 'worktree-agent-a1b995daa94e4e1b7'79// Wrangler from the repository's packages, run with node: no shell to quote for.
80const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js");
81
82function run(command, args, options = {}) {
83 const done = spawnSync(command, args, { encoding: "utf8", ...options });
84 if (done.status !== 0 && !options.allowFail) {
85 throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`);
86 }
87 return done;
88}
89
90const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } });
91
92function start(command, args, options) {
93 const child = spawn(command, args, { ...options });
94 children.push(child);
95 return child;
96}
97
98async function waitFor(url, what) {
99 for (let i = 0; i < 120; i++) {
100 try {
101 const response = await fetch(url);
102 if (response.status < 500) return;
103 } catch {}
104 await new Promise((resolve) => setTimeout(resolve, 500));
105 }
106 throw new Error(`${what} did not start`);
107}
108
109const sql = (command) =>
110 run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], {
111 cwd: service,
112 env: { ...process.env, CI: "1" },
113 });
114
115/** Clones with `args`, and says how the pack was found and what came. */
116function clone(name, args) {
117 const dir = join(work, name);
118 const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" });
119 const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line));
120 const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]);
121 const head = git(["rev-parse", "HEAD"], dir).stdout.trim();
122 const files = git(["ls-tree", "-r", "HEAD"], dir).stdout;
123 const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim();
124 git(["fsck", "--no-progress"], dir);
125 return { pack, head, files, count };
126}
127
128const checks = [];
129function check(what, ok, detail = "") {
130 checks.push({ what, ok });
131 console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`);
132}
133
134function twice(label, args, depth) {
135 const first = clone(`${label}-1`, args);
136 const second = clone(`${label}-2`, args);
137 check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(","));
138 check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(","));
139 check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count);
140 if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count);
141 return second;
142}
143
144try {
145 // The git store, with a repository of a few commits.
146 start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], {
Merge branch 'worktree-agent-aaf03bdceac799c89'147 env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: STORE_PORT, GITSTORE_URL: STORE },
Merge branch 'worktree-agent-a1b995daa94e4e1b7'148 stdio: "inherit",
149 });
150 await waitFor(`${STORE}/healthz`, "the git store");
151 const api = (path, body) =>
152 fetch(`${STORE}/api/repos${path}`, {
153 method: "POST",
154 headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" },
155 body: JSON.stringify(body),
156 }).then((response) => response.json());
157 await api("", { name: KEY, defaultBranch: "main" });
158 const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext;
159 const seed = join(work, "seed");
160 git(["init", "-q", "-b", "main", seed]);
161 for (let i = 1; i <= 5; i++) {
162 writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`);
163 // 12 MB that does not compress, so a pack goes up in multipart parts.
164 if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024));
165 git(["add", "."], seed);
166 git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed);
167 }
168 git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed);
169
170 // The repos service, its database with the repository in it.
171 run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], {
172 cwd: service,
173 env: { ...process.env, CI: "1" },
174 });
175 sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)");
Merge branch 'worktree-agent-aaf03bdceac799c89'176
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI177 // RustFS, with the bucket and lifecycle rule deploy/self-host/docker-compose.yml makes.
Merge branch 'worktree-agent-aaf03bdceac799c89'178 const s3Vars = [];
179 if (S3) {
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI180 run("docker", ["rm", "-f", STORAGE], { allowFail: true });
Merge branch 'worktree-agent-aaf03bdceac799c89'181 run("docker", [
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI182 "run", "-d", "--rm", "--name", STORAGE, "-p", `${S3_PORT}:9000`,
183 "-e", `RUSTFS_ACCESS_KEY=${S3_USER}`, "-e", `RUSTFS_SECRET_KEY=${S3_PASSWORD}`, "-e", "RUSTFS_CONSOLE_ENABLE=false",
184 RUSTFS_IMAGE,
185 ]);
186 await waitFor(`http://localhost:${S3_PORT}/health/ready`, "RustFS");
187 s3api(["create-bucket", "--bucket", PACKS_BUCKET]);
188 s3api([
189 "put-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET, "--lifecycle-configuration",
190 JSON.stringify({
191 Rules: [
192 { ID: "expire-packs", Status: "Enabled", Filter: { Prefix: "packs/" }, Expiration: { Days: 7 } },
193 { ID: "abort-unfinished-uploads", Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },
194 ],
195 }),
Merge branch 'worktree-agent-aaf03bdceac799c89'196 ]);
197 for (const [name, value] of Object.entries({
198 PACK_STORE: "s3",
199 PACK_S3_BUCKET: PACKS_BUCKET,
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI200 S3_ENDPOINT: `http://localhost:${S3_PORT}`,
Merge branch 'worktree-agent-aaf03bdceac799c89'201 S3_REGION: "us-east-1",
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI202 S3_ACCESS_KEY_ID: S3_USER,
203 S3_SECRET_ACCESS_KEY: S3_PASSWORD,
Merge branch 'worktree-agent-aaf03bdceac799c89'204 })) {
205 s3Vars.push("--var", `${name}:${value}`);
206 }
207 }
Merge branch 'worktree-agent-a1b995daa94e4e1b7'208 start(
209 "node",
Merge branch 'worktree-agent-aaf03bdceac799c89'210 [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", ARTIFACTS_CONFIG, "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", REPOS_PORT, ...s3Vars],
Merge branch 'worktree-agent-a1b995daa94e4e1b7'211 { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] },
212 );
213 await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev");
214
215 twice("full, v2", [], 5);
216 twice("shallow, v2", ["--depth=1"], 1);
217 twice("full, v0", ["-c", "protocol.version=0"], 5);
218 twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1);
219
220 // A change to the refs: the next clone goes to the store.
221 sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'");
222 // The service keeps a row it read a moment ago for the same clone's next request.
223 await new Promise((resolve) => setTimeout(resolve, 6000));
224 const after = clone("after-refs", ["--depth=1"]);
225 check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(","));
Merge branch 'worktree-agent-aaf03bdceac799c89'226
227 if (S3) {
228 // Fills finish after git has its answer: give the last one a moment.
229 await new Promise((resolve) => setTimeout(resolve, 3000));
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI230 const listed = s3json(["list-objects-v2", "--bucket", PACKS_BUCKET, "--prefix", "packs/"]).Contents ?? [];
231 const sizes = listed.map((entry) => entry.Size);
Merge branch 'worktree-agent-aaf03bdceac799c89'232 // Nine clones: four kinds twice, each kept once, and one more after the refs moved.
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI233 check("the packs are in RustFS, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`);
Merge branch 'worktree-agent-aaf03bdceac799c89'234 check("the full clone's pack went up in parts", sizes.some((size) => size > 5 * 1024 * 1024), `largest ${Math.max(...sizes)}`);
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI235 // A multipart object's ETag ends in -<number of parts>.
236 check("the large pack is one multipart object", listed.some((entry) => /-\d+"?$/.test(entry.ETag ?? "")), listed.map((entry) => entry.ETag).join(", "));
237 const short = listed.filter((entry) => s3json(["head-object", "--bucket", PACKS_BUCKET, "--key", entry.Key]).ContentLength !== entry.Size);
238 check("every pack reads back whole", short.length === 0, short.map((entry) => entry.Key).join(", "));
239 const incomplete = s3json(["list-multipart-uploads", "--bucket", PACKS_BUCKET]).Uploads ?? [];
240 check("no upload is left unfinished", incomplete.length === 0, incomplete.map((upload) => upload.Key).join(", "));
241 const rules = s3json(["get-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET]).Rules ?? [];
242 check("the bucket expires packs after 7 days", rules.some((rule) => rule.Expiration?.Days === 7 && rule.Filter?.Prefix === "packs/"), JSON.stringify(rules));
243 check("the bucket aborts uploads unfinished after a day", rules.some((rule) => rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1));
Merge branch 'worktree-agent-aaf03bdceac799c89'244 }
Merge branch 'worktree-agent-a1b995daa94e4e1b7'245} catch (error) {
246 console.error(error);
247 checks.push({ what: "ran", ok: false });
248} finally {
249 for (const child of children) {
250 if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
251 else child.kill();
252 }
Self-hosting: RustFS for S3-compatible storage, buckets and the packs' lifecycle rule made with the AWS CLI253 if (S3) run("docker", ["rm", "-f", STORAGE], { allowFail: true });
Merge branch 'worktree-agent-a1b995daa94e4e1b7'254 try {
255 rmSync(work, { recursive: true, force: true });
256 } catch {}
257}
258
259const failed = checks.filter((c) => !c.ok);
260console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`);
261process.exit(failed.length ? 1 : 0);

This file's history is long; its oldest lines are credited to the oldest commit read.