Sign in

flagon-io/g1t

public

Commit

repos/dev: run repos on its own with wrangler dev, and clone through it twice to check the pack cache

clone-check.mjs starts the self-hosted git store and wrangler dev (local D1 and R2, the Artifacts shim, stubbed identity, events, security and billing), then clones full and --depth=1 over protocol v2 and v0, twice each, checks the second came from the pack cache and matches the first, and that a clone after the refs version moves misses.

syntaqxcommitted Parent654886cBrowse files
5 files+247−00/5 viewed
+8−0
1+// The Artifacts binding for running repos on its own (repos.jsonc): the
2+// self-hosted shim, in front of a git store on this machine.
3+{
4+ "name": "g1t-artifacts",
5+ "main": "../../../deploy/self-host/workers/artifacts/index.js",
6+ "compatibility_date": "2026-09-26",
7+ "vars": { "GITSTORE_URL": "http://localhost:8799", "GITSTORE_SECRET": "dev-gitstore-secret-0123" }
8+}
+164−0
1+#!/usr/bin/env node
2+// Clones a repository through the repos service twice, full and shallow,
3+// over protocol v2 and v0, and checks that the second clone of each came
4+// from the pack cache (src/pack_cache.rs) and matches the first. Then
5+// moves the repository's refs version and checks the next clone misses.
6+//
7+// Runs everything on this machine: the self-hosted git store, and
8+// `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc and
9+// dev/stubs.jsonc, state kept in a temporary folder. Build first:
10+//
11+// cd services/repos && node ../../scripts/build-rust-worker.mjs
12+// node dev/clone-check.mjs
13+//
14+// Needs node, git (with git-http-backend) and the repository's npm packages.
15+
16+import { spawn, spawnSync } from "node:child_process";
17+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
18+import { randomBytes } from "node:crypto";
19+import { tmpdir } from "node:os";
20+import { dirname, join, resolve } from "node:path";
21+import { fileURLToPath } from "node:url";
22+import { createRequire } from "node:module";
23+
24+const here = dirname(fileURLToPath(import.meta.url));
25+const service = resolve(here, "..");
26+const root = resolve(service, "../..");
27+const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-"));
28+const persist = join(work, "state");
29+const SECRET = "dev-gitstore-secret-0123";
30+const STORE = "http://localhost:8799";
31+const REPOS = "http://localhost:8791";
32+const KEY = "acme--rocket";
33+const children = [];
34+// Wrangler from the repository's packages, run with node: no shell to quote for.
35+const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js");
36+
37+function run(command, args, options = {}) {
38+ const done = spawnSync(command, args, { encoding: "utf8", ...options });
39+ if (done.status !== 0 && !options.allowFail) {
40+ throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`);
41+ }
42+ return done;
43+}
44+
45+const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } });
46+
47+function start(command, args, options) {
48+ const child = spawn(command, args, { ...options });
49+ children.push(child);
50+ return child;
51+}
52+
53+async function waitFor(url, what) {
54+ for (let i = 0; i < 120; i++) {
55+ try {
56+ const response = await fetch(url);
57+ if (response.status < 500) return;
58+ } catch {}
59+ await new Promise((resolve) => setTimeout(resolve, 500));
60+ }
61+ throw new Error(`${what} did not start`);
62+}
63+
64+const sql = (command) =>
65+ run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], {
66+ cwd: service,
67+ env: { ...process.env, CI: "1" },
68+ });
69+
70+/** Clones with `args`, and says how the pack was found and what came. */
71+function clone(name, args) {
72+ const dir = join(work, name);
73+ const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" });
74+ const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line));
75+ const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]);
76+ const head = git(["rev-parse", "HEAD"], dir).stdout.trim();
77+ const files = git(["ls-tree", "-r", "HEAD"], dir).stdout;
78+ const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim();
79+ git(["fsck", "--no-progress"], dir);
80+ return { pack, head, files, count };
81+}
82+
83+const checks = [];
84+function check(what, ok, detail = "") {
85+ checks.push({ what, ok });
86+ console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`);
87+}
88+
89+function twice(label, args, depth) {
90+ const first = clone(`${label}-1`, args);
91+ const second = clone(`${label}-2`, args);
92+ check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(","));
93+ check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(","));
94+ check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count);
95+ if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count);
96+ return second;
97+}
98+
99+try {
100+ // The git store, with a repository of a few commits.
101+ start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], {
102+ env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: "8799", GITSTORE_URL: STORE },
103+ stdio: "inherit",
104+ });
105+ await waitFor(`${STORE}/healthz`, "the git store");
106+ const api = (path, body) =>
107+ fetch(`${STORE}/api/repos${path}`, {
108+ method: "POST",
109+ headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" },
110+ body: JSON.stringify(body),
111+ }).then((response) => response.json());
112+ await api("", { name: KEY, defaultBranch: "main" });
113+ const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext;
114+ const seed = join(work, "seed");
115+ git(["init", "-q", "-b", "main", seed]);
116+ for (let i = 1; i <= 5; i++) {
117+ writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`);
118+ // 12 MB that does not compress, so a pack goes up in multipart parts.
119+ if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024));
120+ git(["add", "."], seed);
121+ git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed);
122+ }
123+ git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed);
124+
125+ // The repos service, its database with the repository in it.
126+ run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], {
127+ cwd: service,
128+ env: { ...process.env, CI: "1" },
129+ });
130+ sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)");
131+ start(
132+ "node",
133+ [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", "dev/artifacts.jsonc", "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", "8791"],
134+ { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] },
135+ );
136+ await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev");
137+
138+ twice("full, v2", [], 5);
139+ twice("shallow, v2", ["--depth=1"], 1);
140+ twice("full, v0", ["-c", "protocol.version=0"], 5);
141+ twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1);
142+
143+ // A change to the refs: the next clone goes to the store.
144+ sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'");
145+ // The service keeps a row it read a moment ago for the same clone's next request.
146+ await new Promise((resolve) => setTimeout(resolve, 6000));
147+ const after = clone("after-refs", ["--depth=1"]);
148+ check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(","));
149+} catch (error) {
150+ console.error(error);
151+ checks.push({ what: "ran", ok: false });
152+} finally {
153+ for (const child of children) {
154+ if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
155+ else child.kill();
156+ }
157+ try {
158+ rmSync(work, { recursive: true, force: true });
159+ } catch {}
160+}
161+
162+const failed = checks.filter((c) => !c.ok);
163+console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`);
164+process.exit(failed.length ? 1 : 0);
+43−0
1+// The repos service as `wrangler dev` runs it on its own, for git over
2+// HTTPS against a local git store: local D1 and R2 (the clone pack cache,
3+// GIT_PACKS), the Artifacts binding played by the self-hosted shim
4+// (deploy/self-host/workers/artifacts) in front of the self-hosted git store
5+// (deploy/self-host/gitstore/server.mjs), and its other services stubbed
6+// (stubs.js). `node dev/clone-check.mjs` does all of it and clones twice;
7+// by hand, from services/repos:
8+//
9+// node ../../scripts/build-rust-worker.mjs
10+// npx wrangler d1 migrations apply g1t-repos --local -c dev/repos.jsonc
11+// GITSTORE_ROOT=/tmp/g1t-git GITSTORE_SECRET=dev-gitstore-secret-0123 \
12+// GITSTORE_PORT=8799 GITSTORE_URL=http://localhost:8799 \
13+// node ../../deploy/self-host/gitstore/server.mjs &
14+// npx wrangler dev -c dev/repos.jsonc -c dev/artifacts.jsonc -c dev/stubs.jsonc --local --port 8791
15+//
16+// Then make a repository row and its store (clone-check.mjs shows how) and
17+// `git clone http://localhost:8791/acme/rocket.git`.
18+{
19+ "name": "g1t-repos",
20+ "main": "../build/index.js",
21+ "compatibility_date": "2026-09-26",
22+ "d1_databases": [
23+ { "binding": "DB", "database_name": "g1t-repos", "database_id": "local", "migrations_dir": "../migrations" }
24+ ],
25+ "r2_buckets": [{ "binding": "GIT_PACKS", "bucket_name": "g1t-git-packs" }],
26+ "services": [
27+ { "binding": "ARTIFACTS", "service": "g1t-artifacts" },
28+ { "binding": "IDENTITY", "service": "g1t-repos-stubs" },
29+ { "binding": "EVENTS", "service": "g1t-repos-stubs" },
30+ { "binding": "SECURITY", "service": "g1t-repos-stubs" },
31+ { "binding": "BILLING", "service": "g1t-repos-stubs" }
32+ ],
33+ "vars": {
34+ "GIT_OPERATIONS_FREE_CAP": "50000",
35+ "GIT_OPERATIONS_FREE_HOURLY": "100000",
36+ "FREE_PRIVATE_STORAGE_BYTES": "1000000000",
37+ "REPO_STORAGE_LIMIT_BYTES": "950000000",
38+ "LARGE_PUSHES": "unscanned",
39+ "FORK_RETENTION_DAYS": "1",
40+ "ARTIFACTS_NAMESPACES": "{\"ARTIFACTS\":\"g1t\"}",
41+ "ARTIFACTS_NEW_REPOS": ""
42+ }
43+}
+26−0
1+// Stand-ins for identity, events, security and billing, so the repos
2+// service answers anonymous git requests with `wrangler dev` (repos.jsonc).
3+//
4+// - Identity knows nobody: credentials name no one, and no workspace was
5+// renamed. Public repositories can be cloned without signing in.
6+// - Events takes every event and audit entry and logs them.
7+// - Security has allowed no secrets; billing says every workspace is free.
8+
9+export default {
10+ async fetch(request) {
11+ const method = new URL(request.url).pathname.replace(/^\/rpc\//, "");
12+ const args = await request.json().catch(() => ({}));
13+ const json = (value) => Response.json(value);
14+ switch (method) {
15+ case "user_for_git_credentials":
16+ case "resolve_slug":
17+ return json(null);
18+ case "is_free":
19+ case "plan":
20+ return json({ free: true });
21+ default:
22+ console.log(`stub ${method}`, JSON.stringify(args).slice(0, 200));
23+ return json(null);
24+ }
25+ },
26+};
+6−0
1+// The stand-in services for running repos on its own (dev/stubs.js).
2+{
3+ "name": "g1t-repos-stubs",
4+ "main": "stubs.js",
5+ "compatibility_date": "2026-09-26"
6+}