| 1 | #!/usr/bin/env node |
| 2 | // Clones a repository through the repos service twice, full and shallow, |
| 3 | // over protocol v2 and v0, and checks that the second clone of each came |
| 4 | // from the pack cache (src/pack_cache.rs) and matches the first. Then |
| 5 | // moves the repository's refs version and checks the next clone misses. |
| 6 | // |
| 7 | // Runs everything on this machine: the self-hosted git store, and |
| 8 | // `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc and |
| 9 | // dev/stubs.jsonc, state kept in a temporary folder. Build first: |
| 10 | // |
| 11 | // cd services/repos && node ../../scripts/build-rust-worker.mjs |
| 12 | // node dev/clone-check.mjs |
| 13 | // |
| 14 | // With `--s3`, packs are kept in MinIO instead of local R2, as a |
| 15 | // self-hosted installation keeps them (PACK_STORE=s3): it starts MinIO in |
| 16 | // Docker, makes the `g1t-git-packs` bucket with the same expiry rule the |
| 17 | // compose file gives it, and checks that what was kept is there, whole, |
| 18 | // with no upload left unfinished. |
| 19 | // |
| 20 | // node dev/clone-check.mjs --s3 |
| 21 | // |
| 22 | // GITSTORE_PORT, REPOS_PORT and MINIO_PORT move it off 8799, 8791 and 9010. |
| 23 | // |
| 24 | // Needs node, git (with git-http-backend) and the repository's npm |
| 25 | // packages; with `--s3`, Docker too. |
| 26 | |
| 27 | import { spawn, spawnSync } from "node:child_process"; |
| 28 | import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 29 | import { randomBytes } from "node:crypto"; |
| 30 | import { tmpdir } from "node:os"; |
| 31 | import { dirname, join, resolve } from "node:path"; |
| 32 | import { fileURLToPath } from "node:url"; |
| 33 | import { createRequire } from "node:module"; |
| 34 | |
| 35 | const here = dirname(fileURLToPath(import.meta.url)); |
| 36 | const service = resolve(here, ".."); |
| 37 | const root = resolve(service, "../.."); |
| 38 | const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-")); |
| 39 | const persist = join(work, "state"); |
| 40 | const SECRET = "dev-gitstore-secret-0123"; |
| 41 | // The ports, if something on this machine already has the defaults. |
| 42 | const STORE_PORT = process.env.GITSTORE_PORT ?? "8799"; |
| 43 | const REPOS_PORT = process.env.REPOS_PORT ?? "8791"; |
| 44 | const STORE = `http://localhost:${STORE_PORT}`; |
| 45 | const REPOS = `http://localhost:${REPOS_PORT}`; |
| 46 | // dev/artifacts.jsonc, pointed at this run's git store. |
| 47 | const ARTIFACTS_CONFIG = join(work, "artifacts.json"); |
| 48 | writeFileSync( |
| 49 | ARTIFACTS_CONFIG, |
| 50 | JSON.stringify({ |
| 51 | name: "g1t-artifacts", |
| 52 | main: join(root, "deploy/self-host/workers/artifacts/index.js"), |
| 53 | compatibility_date: "2026-09-26", |
| 54 | vars: { GITSTORE_URL: STORE, GITSTORE_SECRET: SECRET }, |
| 55 | }), |
| 56 | ); |
| 57 | const KEY = "acme--rocket"; |
| 58 | const children = []; |
| 59 | // --s3: packs in MinIO (see the top). |
| 60 | const S3 = process.argv.includes("--s3"); |
| 61 | const MINIO = "g1t-clone-check-minio"; |
| 62 | const MINIO_PORT = process.env.MINIO_PORT ?? "9010"; |
| 63 | const MINIO_USER = "g1t"; |
| 64 | const MINIO_PASSWORD = "g1t-clone-check-secret"; |
| 65 | const PACKS_BUCKET = "g1t-git-packs"; |
| 66 | /** `mc` inside the MinIO container, against itself. */ |
| 67 | const mc = (args, options = {}) => run("docker", ["exec", MINIO, "mc", ...args], options); |
| 68 | // Wrangler from the repository's packages, run with node: no shell to quote for. |
| 69 | const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js"); |
| 70 | |
| 71 | function run(command, args, options = {}) { |
| 72 | const done = spawnSync(command, args, { encoding: "utf8", ...options }); |
| 73 | if (done.status !== 0 && !options.allowFail) { |
| 74 | throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`); |
| 75 | } |
| 76 | return done; |
| 77 | } |
| 78 | |
| 79 | const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } }); |
| 80 | |
| 81 | function start(command, args, options) { |
| 82 | const child = spawn(command, args, { ...options }); |
| 83 | children.push(child); |
| 84 | return child; |
| 85 | } |
| 86 | |
| 87 | async function waitFor(url, what) { |
| 88 | for (let i = 0; i < 120; i++) { |
| 89 | try { |
| 90 | const response = await fetch(url); |
| 91 | if (response.status < 500) return; |
| 92 | } catch {} |
| 93 | await new Promise((resolve) => setTimeout(resolve, 500)); |
| 94 | } |
| 95 | throw new Error(`${what} did not start`); |
| 96 | } |
| 97 | |
| 98 | const sql = (command) => |
| 99 | run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], { |
| 100 | cwd: service, |
| 101 | env: { ...process.env, CI: "1" }, |
| 102 | }); |
| 103 | |
| 104 | /** Clones with `args`, and says how the pack was found and what came. */ |
| 105 | function clone(name, args) { |
| 106 | const dir = join(work, name); |
| 107 | const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" }); |
| 108 | const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line)); |
| 109 | const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]); |
| 110 | const head = git(["rev-parse", "HEAD"], dir).stdout.trim(); |
| 111 | const files = git(["ls-tree", "-r", "HEAD"], dir).stdout; |
| 112 | const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim(); |
| 113 | git(["fsck", "--no-progress"], dir); |
| 114 | return { pack, head, files, count }; |
| 115 | } |
| 116 | |
| 117 | const checks = []; |
| 118 | function check(what, ok, detail = "") { |
| 119 | checks.push({ what, ok }); |
| 120 | console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`); |
| 121 | } |
| 122 | |
| 123 | function twice(label, args, depth) { |
| 124 | const first = clone(`${label}-1`, args); |
| 125 | const second = clone(`${label}-2`, args); |
| 126 | check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(",")); |
| 127 | check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(",")); |
| 128 | check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count); |
| 129 | if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count); |
| 130 | return second; |
| 131 | } |
| 132 | |
| 133 | try { |
| 134 | // The git store, with a repository of a few commits. |
| 135 | start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], { |
| 136 | env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: STORE_PORT, GITSTORE_URL: STORE }, |
| 137 | stdio: "inherit", |
| 138 | }); |
| 139 | await waitFor(`${STORE}/healthz`, "the git store"); |
| 140 | const api = (path, body) => |
| 141 | fetch(`${STORE}/api/repos${path}`, { |
| 142 | method: "POST", |
| 143 | headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" }, |
| 144 | body: JSON.stringify(body), |
| 145 | }).then((response) => response.json()); |
| 146 | await api("", { name: KEY, defaultBranch: "main" }); |
| 147 | const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext; |
| 148 | const seed = join(work, "seed"); |
| 149 | git(["init", "-q", "-b", "main", seed]); |
| 150 | for (let i = 1; i <= 5; i++) { |
| 151 | writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`); |
| 152 | // 12 MB that does not compress, so a pack goes up in multipart parts. |
| 153 | if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024)); |
| 154 | git(["add", "."], seed); |
| 155 | git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed); |
| 156 | } |
| 157 | git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed); |
| 158 | |
| 159 | // The repos service, its database with the repository in it. |
| 160 | run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], { |
| 161 | cwd: service, |
| 162 | env: { ...process.env, CI: "1" }, |
| 163 | }); |
| 164 | sql("INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)"); |
| 165 | |
| 166 | // MinIO, with the bucket and expiry rule deploy/self-host/docker-compose.yml makes. |
| 167 | const s3Vars = []; |
| 168 | if (S3) { |
| 169 | run("docker", ["rm", "-f", MINIO], { allowFail: true }); |
| 170 | run("docker", [ |
| 171 | "run", "-d", "--rm", "--name", MINIO, "-p", `${MINIO_PORT}:9000`, |
| 172 | "-e", `MINIO_ROOT_USER=${MINIO_USER}`, "-e", `MINIO_ROOT_PASSWORD=${MINIO_PASSWORD}`, |
| 173 | process.env.MINIO_IMAGE ?? "pgsty/minio:latest", "server", "/data", |
| 174 | ]); |
| 175 | await waitFor(`http://localhost:${MINIO_PORT}/minio/health/ready`, "MinIO"); |
| 176 | mc(["alias", "set", "local", "http://localhost:9000", MINIO_USER, MINIO_PASSWORD]); |
| 177 | mc(["mb", "--ignore-existing", `local/${PACKS_BUCKET}`]); |
| 178 | mc(["ilm", "rule", "add", "--prefix", "packs/", "--expire-days", "7", `local/${PACKS_BUCKET}`]); |
| 179 | for (const [name, value] of Object.entries({ |
| 180 | PACK_STORE: "s3", |
| 181 | PACK_S3_BUCKET: PACKS_BUCKET, |
| 182 | S3_ENDPOINT: `http://localhost:${MINIO_PORT}`, |
| 183 | S3_REGION: "us-east-1", |
| 184 | S3_ACCESS_KEY_ID: MINIO_USER, |
| 185 | S3_SECRET_ACCESS_KEY: MINIO_PASSWORD, |
| 186 | })) { |
| 187 | s3Vars.push("--var", `${name}:${value}`); |
| 188 | } |
| 189 | } |
| 190 | start( |
| 191 | "node", |
| 192 | [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", ARTIFACTS_CONFIG, "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", REPOS_PORT, ...s3Vars], |
| 193 | { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] }, |
| 194 | ); |
| 195 | await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev"); |
| 196 | |
| 197 | twice("full, v2", [], 5); |
| 198 | twice("shallow, v2", ["--depth=1"], 1); |
| 199 | twice("full, v0", ["-c", "protocol.version=0"], 5); |
| 200 | twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1); |
| 201 | |
| 202 | // A change to the refs: the next clone goes to the store. |
| 203 | sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'"); |
| 204 | // The service keeps a row it read a moment ago for the same clone's next request. |
| 205 | await new Promise((resolve) => setTimeout(resolve, 6000)); |
| 206 | const after = clone("after-refs", ["--depth=1"]); |
| 207 | check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(",")); |
| 208 | |
| 209 | if (S3) { |
| 210 | // Fills finish after git has its answer: give the last one a moment. |
| 211 | await new Promise((resolve) => setTimeout(resolve, 3000)); |
| 212 | const listed = mc(["ls", "--recursive", "--json", `local/${PACKS_BUCKET}/packs/`]).stdout.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)); |
| 213 | const sizes = listed.map((entry) => entry.size); |
| 214 | // Nine clones: four kinds twice, each kept once, and one more after the refs moved. |
| 215 | check("the packs are in MinIO, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`); |
| 216 | check("the full clone's pack went up in parts", sizes.some((size) => size > 5 * 1024 * 1024), `largest ${Math.max(...sizes)}`); |
| 217 | const incomplete = mc(["ls", "--recursive", "--incomplete", `local/${PACKS_BUCKET}`]).stdout.trim(); |
| 218 | check("no upload is left unfinished", incomplete === "", incomplete); |
| 219 | const rules = mc(["ilm", "rule", "ls", "--json", `local/${PACKS_BUCKET}`]).stdout; |
| 220 | check("the bucket expires packs after 7 days", /"Days":\s*7/.test(rules) && rules.includes("packs/")); |
| 221 | } |
| 222 | } catch (error) { |
| 223 | console.error(error); |
| 224 | checks.push({ what: "ran", ok: false }); |
| 225 | } finally { |
| 226 | for (const child of children) { |
| 227 | if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" }); |
| 228 | else child.kill(); |
| 229 | } |
| 230 | if (S3) run("docker", ["rm", "-f", MINIO], { allowFail: true }); |
| 231 | try { |
| 232 | rmSync(work, { recursive: true, force: true }); |
| 233 | } catch {} |
| 234 | } |
| 235 | |
| 236 | const failed = checks.filter((c) => !c.ok); |
| 237 | console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`); |
| 238 | process.exit(failed.length ? 1 : 0); |