| 1 | #!/usr/bin/env node |
| 2 | // Clones a repository through the repos service twice, full and shallow, |
| 3 | // over protocol v2 and v0, and checks that the second clone of each came |
| 4 | // from the pack cache (src/pack_cache.rs) and matches the first. Then |
| 5 | // moves the repository's refs version and checks the next clone misses. |
| 6 | // |
| 7 | // Runs everything on this machine: the self-hosted git store, and |
| 8 | // `wrangler dev` with dev/repos.jsonc, dev/gitstore.jsonc and |
| 9 | // dev/stubs.jsonc, state kept in a temporary folder. Build first: |
| 10 | // |
| 11 | // cd services/repos && node ../../scripts/build-rust-worker.mjs |
| 12 | // node dev/clone-check.mjs |
| 13 | // |
| 14 | // With `--s3`, packs are kept in RustFS instead of local R2, as a |
| 15 | // self-hosted installation keeps them (PACK_STORE=s3): it starts RustFS in |
| 16 | // Docker, makes the `g1t-git-packs` bucket with the same lifecycle rule the |
| 17 | // compose file gives it (with the AWS CLI, as the compose file does), and |
| 18 | // checks that what was kept is there, whole, with no upload left |
| 19 | // unfinished. |
| 20 | // |
| 21 | // node dev/clone-check.mjs --s3 |
| 22 | // |
| 23 | // GITSTORE_PORT, REPOS_PORT and S3_PORT move it off 8799, 8791 and 9010. |
| 24 | // RUSTFS_IMAGE and AWS_CLI_IMAGE choose other images. |
| 25 | // |
| 26 | // Needs node, git (with git-http-backend) and the repository's npm |
| 27 | // packages; with `--s3`, Docker too. |
| 28 | |
| 29 | import { spawn, spawnSync } from "node:child_process"; |
| 30 | import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 31 | import { randomBytes } from "node:crypto"; |
| 32 | import { tmpdir } from "node:os"; |
| 33 | import { dirname, join, resolve } from "node:path"; |
| 34 | import { fileURLToPath } from "node:url"; |
| 35 | import { createRequire } from "node:module"; |
| 36 | |
| 37 | const here = dirname(fileURLToPath(import.meta.url)); |
| 38 | const service = resolve(here, ".."); |
| 39 | const root = resolve(service, "../.."); |
| 40 | const work = mkdtempSync(join(tmpdir(), "g1t-clone-check-")); |
| 41 | const persist = join(work, "state"); |
| 42 | const SECRET = "dev-gitstore-secret-0123"; |
| 43 | // The ports, if something on this machine already has the defaults. |
| 44 | const STORE_PORT = process.env.GITSTORE_PORT ?? "8799"; |
| 45 | const REPOS_PORT = process.env.REPOS_PORT ?? "8791"; |
| 46 | const STORE = `http://localhost:${STORE_PORT}`; |
| 47 | const REPOS = `http://localhost:${REPOS_PORT}`; |
| 48 | // dev/gitstore.jsonc, pointed at this run's git store. |
| 49 | const GITSTORE_CONFIG = join(work, "gitstore.json"); |
| 50 | writeFileSync( |
| 51 | GITSTORE_CONFIG, |
| 52 | JSON.stringify({ |
| 53 | name: "g1t-gitstore", |
| 54 | main: join(root, "deploy/self-host/workers/gitstore/index.js"), |
| 55 | compatibility_date: "2026-09-26", |
| 56 | vars: { GITSTORE_URL: STORE, GITSTORE_SECRET: SECRET }, |
| 57 | }), |
| 58 | ); |
| 59 | const KEY = "acme--rocket"; |
| 60 | const children = []; |
| 61 | // --s3: packs in RustFS (see the top). The images are the compose file's. |
| 62 | const S3 = process.argv.includes("--s3"); |
| 63 | const STORAGE = "g1t-clone-check-rustfs"; |
| 64 | const S3_PORT = process.env.S3_PORT ?? "9010"; |
| 65 | const S3_USER = "g1t"; |
| 66 | const S3_PASSWORD = "g1t-clone-check-secret"; |
| 67 | const RUSTFS_IMAGE = process.env.RUSTFS_IMAGE ?? "rustfs/rustfs:1.0.1"; |
| 68 | const AWS_CLI_IMAGE = process.env.AWS_CLI_IMAGE ?? "amazon/aws-cli:2.37.10"; |
| 69 | const PACKS_BUCKET = "g1t-git-packs"; |
| 70 | /** The AWS CLI's `s3api`, in a container on the store's network, against it. */ |
| 71 | const s3api = (args, options = {}) => |
| 72 | run( |
| 73 | "docker", |
| 74 | [ |
| 75 | "run", |
| 76 | "--rm", |
| 77 | "--network", |
| 78 | `container:${STORAGE}`, |
| 79 | "-e", |
| 80 | `AWS_ACCESS_KEY_ID=${S3_USER}`, |
| 81 | "-e", |
| 82 | `AWS_SECRET_ACCESS_KEY=${S3_PASSWORD}`, |
| 83 | "-e", |
| 84 | "AWS_DEFAULT_REGION=us-east-1", |
| 85 | AWS_CLI_IMAGE, |
| 86 | "--endpoint-url", |
| 87 | "http://localhost:9000", |
| 88 | "--output", |
| 89 | "json", |
| 90 | "s3api", |
| 91 | ...args, |
| 92 | ], |
| 93 | options, |
| 94 | ); |
| 95 | /** The same, its answer parsed. */ |
| 96 | const s3json = (args) => JSON.parse(s3api(args).stdout.trim() || "{}"); |
| 97 | // Wrangler from the repository's packages, run with node: no shell to quote for. |
| 98 | const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js"); |
| 99 | |
| 100 | function run(command, args, options = {}) { |
| 101 | const done = spawnSync(command, args, { encoding: "utf8", ...options }); |
| 102 | if (done.status !== 0 && !options.allowFail) { |
| 103 | throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`); |
| 104 | } |
| 105 | return done; |
| 106 | } |
| 107 | |
| 108 | const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } }); |
| 109 | |
| 110 | function start(command, args, options) { |
| 111 | const child = spawn(command, args, { ...options }); |
| 112 | children.push(child); |
| 113 | return child; |
| 114 | } |
| 115 | |
| 116 | async function waitFor(url, what) { |
| 117 | for (let i = 0; i < 120; i++) { |
| 118 | try { |
| 119 | const response = await fetch(url); |
| 120 | if (response.status < 500) return; |
| 121 | } catch {} |
| 122 | await new Promise((resolve) => setTimeout(resolve, 500)); |
| 123 | } |
| 124 | throw new Error(`${what} did not start`); |
| 125 | } |
| 126 | |
| 127 | const sql = (command) => |
| 128 | run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], { |
| 129 | cwd: service, |
| 130 | env: { ...process.env, CI: "1" }, |
| 131 | }); |
| 132 | |
| 133 | /** Clones with `args`, and says how the pack was found and what came. */ |
| 134 | function clone(name, args) { |
| 135 | const dir = join(work, name); |
| 136 | const done = git(["clone", ...args, `${REPOS}/acme/rocket.git`, dir], work, { GIT_TRACE_CURL: "1", GIT_TRACE_CURL_NO_DATA: "1" }); |
| 137 | const timings = done.stderr.split("\n").filter((line) => /server-timing:/i.test(line) && /pack;desc=/.test(line)); |
| 138 | const pack = timings.map((line) => /pack;desc=(\w+)/.exec(line)[1]); |
| 139 | const head = git(["rev-parse", "HEAD"], dir).stdout.trim(); |
| 140 | const files = git(["ls-tree", "-r", "HEAD"], dir).stdout; |
| 141 | const count = git(["rev-list", "--count", "HEAD"], dir).stdout.trim(); |
| 142 | git(["fsck", "--no-progress"], dir); |
| 143 | return { pack, head, files, count }; |
| 144 | } |
| 145 | |
| 146 | const checks = []; |
| 147 | function check(what, ok, detail = "") { |
| 148 | checks.push({ what, ok }); |
| 149 | console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`); |
| 150 | } |
| 151 | |
| 152 | function twice(label, args, depth) { |
| 153 | const first = clone(`${label}-1`, args); |
| 154 | const second = clone(`${label}-2`, args); |
| 155 | check(`${label}: the first clone misses`, first.pack.includes("miss"), first.pack.join(",")); |
| 156 | check(`${label}: the second clone hits`, second.pack.includes("hit"), second.pack.join(",")); |
| 157 | check(`${label}: both clones are the same`, first.head === second.head && first.files === second.files && first.count === second.count); |
| 158 | if (depth) check(`${label}: ${depth} commit(s) of history`, second.count === String(depth), second.count); |
| 159 | return second; |
| 160 | } |
| 161 | |
| 162 | try { |
| 163 | // The git store, with a repository of a few commits. |
| 164 | start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], { |
| 165 | env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: STORE_PORT, GITSTORE_URL: STORE }, |
| 166 | stdio: "inherit", |
| 167 | }); |
| 168 | await waitFor(`${STORE}/healthz`, "the git store"); |
| 169 | const api = (path, body) => |
| 170 | fetch(`${STORE}/api/repos${path}`, { |
| 171 | method: "POST", |
| 172 | headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" }, |
| 173 | body: JSON.stringify(body), |
| 174 | }).then((response) => response.json()); |
| 175 | await api("", { name: KEY, defaultBranch: "main" }); |
| 176 | const token = (await api(`/${KEY}/tokens`, { scope: "write" })).plaintext; |
| 177 | const seed = join(work, "seed"); |
| 178 | git(["init", "-q", "-b", "main", seed]); |
| 179 | for (let i = 1; i <= 5; i++) { |
| 180 | writeFileSync(join(seed, `file-${i}.txt`), `${"line\n".repeat(200 * i)}${i}\n`); |
| 181 | // 12 MB that does not compress, so a pack goes up in multipart parts. |
| 182 | if (i === 5) writeFileSync(join(seed, "noise.bin"), randomBytes(12 * 1024 * 1024)); |
| 183 | git(["add", "."], seed); |
| 184 | git(["-c", "user.name=dev", "-c", "user.email=dev@example.com", "commit", "-q", "-m", `commit ${i}`], seed); |
| 185 | } |
| 186 | git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/${KEY}.git`, "main"], seed); |
| 187 | |
| 188 | // The repos service, its database with the repository in it. |
| 189 | run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], { |
| 190 | cwd: service, |
| 191 | env: { ...process.env, CI: "1" }, |
| 192 | }); |
| 193 | sql( |
| 194 | "INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES ('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1)", |
| 195 | ); |
| 196 | |
| 197 | // RustFS, with the bucket and lifecycle rule deploy/self-host/docker-compose.yml makes. |
| 198 | const s3Vars = []; |
| 199 | if (S3) { |
| 200 | run("docker", ["rm", "-f", STORAGE], { allowFail: true }); |
| 201 | run("docker", [ |
| 202 | "run", |
| 203 | "-d", |
| 204 | "--rm", |
| 205 | "--name", |
| 206 | STORAGE, |
| 207 | "-p", |
| 208 | `${S3_PORT}:9000`, |
| 209 | "-e", |
| 210 | `RUSTFS_ACCESS_KEY=${S3_USER}`, |
| 211 | "-e", |
| 212 | `RUSTFS_SECRET_KEY=${S3_PASSWORD}`, |
| 213 | "-e", |
| 214 | "RUSTFS_CONSOLE_ENABLE=false", |
| 215 | RUSTFS_IMAGE, |
| 216 | ]); |
| 217 | await waitFor(`http://localhost:${S3_PORT}/health/ready`, "RustFS"); |
| 218 | s3api(["create-bucket", "--bucket", PACKS_BUCKET]); |
| 219 | s3api([ |
| 220 | "put-bucket-lifecycle-configuration", |
| 221 | "--bucket", |
| 222 | PACKS_BUCKET, |
| 223 | "--lifecycle-configuration", |
| 224 | JSON.stringify({ |
| 225 | Rules: [ |
| 226 | { ID: "expire-packs", Status: "Enabled", Filter: { Prefix: "packs/" }, Expiration: { Days: 7 } }, |
| 227 | { ID: "abort-unfinished-uploads", Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } }, |
| 228 | ], |
| 229 | }), |
| 230 | ]); |
| 231 | for (const [name, value] of Object.entries({ |
| 232 | PACK_STORE: "s3", |
| 233 | PACK_S3_BUCKET: PACKS_BUCKET, |
| 234 | S3_ENDPOINT: `http://localhost:${S3_PORT}`, |
| 235 | S3_REGION: "us-east-1", |
| 236 | S3_ACCESS_KEY_ID: S3_USER, |
| 237 | S3_SECRET_ACCESS_KEY: S3_PASSWORD, |
| 238 | })) { |
| 239 | s3Vars.push("--var", `${name}:${value}`); |
| 240 | } |
| 241 | } |
| 242 | start( |
| 243 | "node", |
| 244 | [ |
| 245 | WRANGLER, |
| 246 | "dev", |
| 247 | "-c", |
| 248 | "dev/repos.jsonc", |
| 249 | "-c", |
| 250 | GITSTORE_CONFIG, |
| 251 | "-c", |
| 252 | "dev/stubs.jsonc", |
| 253 | "--local", |
| 254 | "--persist-to", |
| 255 | persist, |
| 256 | "--port", |
| 257 | REPOS_PORT, |
| 258 | ...s3Vars, |
| 259 | ], |
| 260 | { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] }, |
| 261 | ); |
| 262 | await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev"); |
| 263 | |
| 264 | twice("full, v2", [], 5); |
| 265 | twice("shallow, v2", ["--depth=1"], 1); |
| 266 | twice("full, v0", ["-c", "protocol.version=0"], 5); |
| 267 | twice("shallow, v0", ["-c", "protocol.version=0", "--depth=1"], 1); |
| 268 | |
| 269 | // A change to the refs: the next clone goes to the store. |
| 270 | sql("UPDATE repos SET refs_version = refs_version + 1 WHERE id = 'rep_rocket'"); |
| 271 | // The service keeps a row it read a moment ago for the same clone's next request. |
| 272 | await new Promise((resolve) => setTimeout(resolve, 6000)); |
| 273 | const after = clone("after-refs", ["--depth=1"]); |
| 274 | check("after the refs version moves, a clone misses", after.pack.includes("miss"), after.pack.join(",")); |
| 275 | |
| 276 | if (S3) { |
| 277 | // Fills finish after git has its answer: give the last one a moment. |
| 278 | await new Promise((resolve) => setTimeout(resolve, 3000)); |
| 279 | const listed = s3json(["list-objects-v2", "--bucket", PACKS_BUCKET, "--prefix", "packs/"]).Contents ?? []; |
| 280 | const sizes = listed.map((entry) => entry.Size); |
| 281 | // Nine clones: four kinds twice, each kept once, and one more after the refs moved. |
| 282 | check("the packs are in RustFS, one per distinct clone", listed.length === 5, `${listed.length}: ${sizes.join(", ")}`); |
| 283 | check( |
| 284 | "the full clone's pack went up in parts", |
| 285 | sizes.some((size) => size > 5 * 1024 * 1024), |
| 286 | `largest ${Math.max(...sizes)}`, |
| 287 | ); |
| 288 | // A multipart object's ETag ends in -<number of parts>. |
| 289 | check( |
| 290 | "the large pack is one multipart object", |
| 291 | listed.some((entry) => /-\d+"?$/.test(entry.ETag ?? "")), |
| 292 | listed.map((entry) => entry.ETag).join(", "), |
| 293 | ); |
| 294 | const short = listed.filter((entry) => s3json(["head-object", "--bucket", PACKS_BUCKET, "--key", entry.Key]).ContentLength !== entry.Size); |
| 295 | check("every pack reads back whole", short.length === 0, short.map((entry) => entry.Key).join(", ")); |
| 296 | const incomplete = s3json(["list-multipart-uploads", "--bucket", PACKS_BUCKET]).Uploads ?? []; |
| 297 | check("no upload is left unfinished", incomplete.length === 0, incomplete.map((upload) => upload.Key).join(", ")); |
| 298 | const rules = s3json(["get-bucket-lifecycle-configuration", "--bucket", PACKS_BUCKET]).Rules ?? []; |
| 299 | check( |
| 300 | "the bucket expires packs after 7 days", |
| 301 | rules.some((rule) => rule.Expiration?.Days === 7 && rule.Filter?.Prefix === "packs/"), |
| 302 | JSON.stringify(rules), |
| 303 | ); |
| 304 | check( |
| 305 | "the bucket aborts uploads unfinished after a day", |
| 306 | rules.some((rule) => rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1), |
| 307 | ); |
| 308 | } |
| 309 | } catch (error) { |
| 310 | console.error(error); |
| 311 | checks.push({ what: "ran", ok: false }); |
| 312 | } finally { |
| 313 | for (const child of children) { |
| 314 | if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" }); |
| 315 | else child.kill(); |
| 316 | } |
| 317 | if (S3) run("docker", ["rm", "-f", STORAGE], { allowFail: true }); |
| 318 | try { |
| 319 | rmSync(work, { recursive: true, force: true }); |
| 320 | } catch {} |
| 321 | } |
| 322 | |
| 323 | const failed = checks.filter((c) => !c.ok); |
| 324 | console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`); |
| 325 | process.exit(failed.length ? 1 : 0); |