Sign in

flagon-io/g1t

public

g1t/services/repos/dev/clone-check.mjs

164 lines7,547 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

repos/dev: run repos on its own with wrangler dev, and clone through it twice to check the pack cache1#!/usr/bin/env node
2// Clones a repository through the repos service twice, full and shallow,
3// over protocol v2 and v0, and checks that the second clone of each came
4// from the pack cache (src/pack_cache.rs) and matches the first. Then
5// moves the repository's refs version and checks the next clone misses.
6//
7// Runs everything on this machine: the self-hosted git store, and
8// `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc and
9// dev/stubs.jsonc, state kept in a temporary folder. Build first:
10//
11// cd services/repos && node ../../scripts/build-rust-worker.mjs
12// node dev/clone-check.mjs
13//
14// Needs node, git (with git-http-backend) and the repository's npm packages.
15
16import { spawn, spawnSync } from "node:child_process";
17import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
18import { randomBytes } from "node:crypto";
19import { tmpdir } from "node:os";
20import { dirname, join, resolve } from "node:path";
21import { fileURLToPath } from "node:url";
22import { createRequire } from "node:module";
23
24const here = dirname(fileURLToPath(import.meta.url));
25const service = resolve(here, "..");
26const root = resolve(service, "../..");
27const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-"));
28const persist = join(work, "state");
29const SECRET = "dev-gitstore-secret-0123";
30const STORE = "http://localhost:8799";
31const REPOS = "http://localhost:8791";
32const KEY = "acme--rocket";
33const children = [];
34// Wrangler from the repository's packages, run with node: no shell to quote for.
35const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js");
36
37function run(command, args, options = {}) {
38 const done = spawnSync(command, args, { encoding: "utf8", ...options });
39 if (done.status !== 0 && !options.allowFail) {
40 throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`);
41 }
42 return done;
43}
44
45const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } });
46
47function start(command, args, options) {
48 const child = spawn(command, args, { ...options });
49 children.push(child);
50 return child;
51}
52
53async function waitFor(url, what) {
54 for (let i = 0; i < 120; i++) {
55 try {
56 const response = await fetch(url);
57 if (response.status < 500) return;
58 } catch {}
59 await new Promise((resolve) => setTimeout(resolve, 500));
60 }
61 throw new Error(`${what} did not start`);
62}
63
64const sql = (command) =>
65 run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], {
66 cwd: service,
67 env: { ...process.env, CI: "1" },
68 });
69
70/** Clones with `args`, and says how the pack was found and what came. */
71function clone(name, args) {
72 const dir = join(work, name);
73 const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" });
74 const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line));
75 const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]);
76 const head = git(["rev-parse", "HEAD"], dir).stdout.trim();
77 const files = git(["ls-tree", "-r", "HEAD"], dir).stdout;
78 const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim();
79 git(["fsck", "--no-progress"], dir);
80 return { pack, head, files, count };
81}
82
83const checks = [];
84function check(what, ok, detail = "") {
85 checks.push({ what, ok });
86 console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`);
87}
88
89function twice(label, args, depth) {
90 const first = clone(`${label}-1`, args);
91 const second = clone(`${label}-2`, args);
92 check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(","));
93 check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(","));
94 check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count);
95 if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count);
96 return second;
97}
98
99try {
100 // The git store, with a repository of a few commits.
101 start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], {
102 env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: "8799", GITSTORE_URL: STORE },
103 stdio: "inherit",
104 });
105 await waitFor(`${STORE}/healthz`, "the git store");
106 const api = (path, body) =>
107 fetch(`${STORE}/api/repos${path}`, {
108 method: "POST",
109 headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" },
110 body: JSON.stringify(body),
111 }).then((response) => response.json());
112 await api("", { name: KEY, defaultBranch: "main" });
113 const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext;
114 const seed = join(work, "seed");
115 git(["init", "-q", "-b", "main", seed]);
116 for (let i = 1; i <= 5; i++) {
117 writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`);
118 // 12 MB that does not compress, so a pack goes up in multipart parts.
119 if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024));
120 git(["add", "."], seed);
121 git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed);
122 }
123 git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed);
124
125 // The repos service, its database with the repository in it.
126 run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], {
127 cwd: service,
128 env: { ...process.env, CI: "1" },
129 });
130 sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)");
131 start(
132 "node",
133 [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", "dev/artifacts.jsonc", "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", "8791"],
134 { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] },
135 );
136 await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev");
137
138 twice("full, v2", [], 5);
139 twice("shallow, v2", ["--depth=1"], 1);
140 twice("full, v0", ["-c", "protocol.version=0"], 5);
141 twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1);
142
143 // A change to the refs: the next clone goes to the store.
144 sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'");
145 // The service keeps a row it read a moment ago for the same clone's next request.
146 await new Promise((resolve) => setTimeout(resolve, 6000));
147 const after = clone("after-refs", ["--depth=1"]);
148 check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(","));
149} catch (error) {
150 console.error(error);
151 checks.push({ what: "ran", ok: false });
152} finally {
153 for (const child of children) {
154 if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" });
155 else child.kill();
156 }
157 try {
158 rmSync(work, { recursive: true, force: true });
159 } catch {}
160}
161
162const failed = checks.filter((c) => !c.ok);
163console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`);
164process.exit(failed.length ? 1 : 0);