| 1 | #!/usr/bin/env node |
| 2 | // Pushes through the repos service with real git and checks that git sends |
| 3 | // a pack without outside bases: the receive-pack advertisement says |
| 4 | // `no-thin` (src/git_http.rs `with_no_thin`), git runs pack-objects without |
| 5 | // `--thin`, and the service says the pack came whole (`thin;desc=no` in |
| 6 | // Server-Timing) and read no bases from the store. Then the same into an |
| 7 | // empty repository, whose advertisement is a `capabilities^{}` line. |
| 8 | // |
| 9 | // Runs everything on this machine, as clone-check.mjs does: the self-hosted |
| 10 | // git store, and `wrangler dev` with dev/repos.jsonc, dev/artifacts.jsonc |
| 11 | // and dev/stubs.jsonc (whose identity stub knows `dev`, the pusher). Build |
| 12 | // first: |
| 13 | // |
| 14 | // cd services/repos && node ../../scripts/build-rust-worker.mjs |
| 15 | // node dev/push-check.mjs |
| 16 | // |
| 17 | // GITSTORE_PORT and REPOS_PORT move it off 8799 and 8791. Needs node, git |
| 18 | // (with git-http-backend) and the repository's npm packages. |
| 19 | |
| 20 | import { spawn, spawnSync } from "node:child_process"; |
| 21 | import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; |
| 22 | import { tmpdir } from "node:os"; |
| 23 | import { dirname, join, resolve } from "node:path"; |
| 24 | import { fileURLToPath } from "node:url"; |
| 25 | import { createRequire } from "node:module"; |
| 26 | |
| 27 | const here = dirname(fileURLToPath(import.meta.url)); |
| 28 | const service = resolve(here, ".."); |
| 29 | const root = resolve(service, "../.."); |
| 30 | const work = mkdtempSync(join(tmpdir(), "g1t-push-check-")); |
| 31 | const persist = join(work, "state"); |
| 32 | const SECRET = "dev-gitstore-secret-0123"; |
| 33 | const STORE_PORT = process.env.GITSTORE_PORT ?? "8799"; |
| 34 | const REPOS_PORT = process.env.REPOS_PORT ?? "8791"; |
| 35 | const STORE = `http://localhost:${STORE_PORT}`; |
| 36 | const REPOS = `http://localhost:${REPOS_PORT}`; |
| 37 | // The pusher the identity stub knows (dev/stubs.js). |
| 38 | const PUSHER = "dev:dev-push-secret"; |
| 39 | const ARTIFACTS_CONFIG = join(work, "artifacts.json"); |
| 40 | writeFileSync( |
| 41 | ARTIFACTS_CONFIG, |
| 42 | JSON.stringify({ |
| 43 | name: "g1t-artifacts", |
| 44 | main: join(root, "deploy/self-host/workers/artifacts/index.js"), |
| 45 | compatibility_date: "2026-09-26", |
| 46 | vars: { GITSTORE_URL: STORE, GITSTORE_SECRET: SECRET }, |
| 47 | }), |
| 48 | ); |
| 49 | const children = []; |
| 50 | const WRANGLER = join(dirname(createRequire(join(service, "package.json")).resolve("wrangler/package.json")), "bin/wrangler.js"); |
| 51 | |
| 52 | function run(command, args, options = {}) { |
| 53 | const done = spawnSync(command, args, { encoding: "utf8", ...options }); |
| 54 | if (done.status !== 0 && !options.allowFail) { |
| 55 | throw new Error(`${command} ${args.join(" ")} failed:\n${done.stdout}\n${done.stderr}`); |
| 56 | } |
| 57 | return done; |
| 58 | } |
| 59 | |
| 60 | const git = (args, cwd = work, env = {}) => run("git", args, { cwd, env: { ...process.env, ...env } }); |
| 61 | const identity = ["-c", "user.name=dev", "-c", "user.email=dev@example.com"]; |
| 62 | |
| 63 | function start(command, args, options) { |
| 64 | const child = spawn(command, args, { ...options }); |
| 65 | children.push(child); |
| 66 | return child; |
| 67 | } |
| 68 | |
| 69 | async function waitFor(url, what) { |
| 70 | for (let i = 0; i < 120; i++) { |
| 71 | try { |
| 72 | const response = await fetch(url); |
| 73 | if (response.status < 500) return; |
| 74 | } catch {} |
| 75 | await new Promise((resolve) => setTimeout(resolve, 500)); |
| 76 | } |
| 77 | throw new Error(`${what} did not start`); |
| 78 | } |
| 79 | |
| 80 | const sql = (command) => |
| 81 | run("node", [WRANGLER, "d1", "execute", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc", "--command", command], { |
| 82 | cwd: service, |
| 83 | env: { ...process.env, CI: "1" }, |
| 84 | }); |
| 85 | |
| 86 | const checks = []; |
| 87 | function check(what, ok, detail = "") { |
| 88 | checks.push({ what, ok }); |
| 89 | console.log(`${ok ? "ok " : "FAIL"} ${what}${detail ? ` (${detail})` : ""}`); |
| 90 | } |
| 91 | |
| 92 | /** Pushes `dir`'s main through the service, with git's traces. */ |
| 93 | function push(label, dir, name) { |
| 94 | const trace = join(work, `${label}.trace`); |
| 95 | const done = git(["push", `http://${PUSHER}@localhost:${REPOS_PORT}/acme/${name}.git`, "main"], dir, { |
| 96 | GIT_TRACE: trace, |
| 97 | GIT_TRACE_PACKET: trace, |
| 98 | GIT_TRACE_CURL: trace, |
| 99 | GIT_TRACE_CURL_NO_DATA: "1", |
| 100 | }); |
| 101 | const lines = readFileSync(trace, "utf8").split("\n"); |
| 102 | // The first ref line, with the capabilities after its NUL (`\0` in the trace). |
| 103 | const advertised = lines.find((line) => /packet:.*< [0-9a-f]{40} \S+\\0/.test(line)) ?? ""; |
| 104 | if (!advertised) console.log(lines.filter((line) => /report-status/.test(line)).slice(0, 3).join("\n")); |
| 105 | const packing = lines.find((line) => /run_command: git pack-objects/.test(line)) ?? ""; |
| 106 | const timing = lines.filter((line) => /server-timing:/i.test(line) && /recv;dur=/.test(line)).join(" "); |
| 107 | check(`${label}: the advertisement says no-thin`, /\bno-thin\b/.test(advertised), `...${advertised.slice(-48)}`); |
| 108 | check(`${label}: git packs without --thin`, packing !== "" && !/--thin\b/.test(packing), packing.replace(/.*run_command: /, "")); |
| 109 | check(`${label}: the service says the pack came whole`, /thin;desc=no/.test(timing), (/thin;desc=\w+/.exec(timing) ?? ["no thin note"])[0]); |
| 110 | check(`${label}: no base was read`, /read;dur=\d+/.test(timing) && !/thin;desc=yes/.test(timing), (/read;dur=\d+/.exec(timing) ?? [""])[0]); |
| 111 | return done; |
| 112 | } |
| 113 | |
| 114 | try { |
| 115 | start("node", [join(root, "deploy/self-host/gitstore/server.mjs")], { |
| 116 | env: { ...process.env, GITSTORE_ROOT: join(work, "git"), GITSTORE_SECRET: SECRET, GITSTORE_PORT: STORE_PORT, GITSTORE_URL: STORE }, |
| 117 | stdio: "inherit", |
| 118 | }); |
| 119 | await waitFor(`${STORE}/healthz`, "the git store"); |
| 120 | const api = (path, body) => |
| 121 | fetch(`${STORE}/api/repos${path}`, { |
| 122 | method: "POST", |
| 123 | headers: { "x-gitstore-secret": SECRET, "content-type": "application/json" }, |
| 124 | body: JSON.stringify(body), |
| 125 | }).then((response) => response.json()); |
| 126 | // acme/rocket: a large file, many versions of it, so a push changing one |
| 127 | // line of it would be a delta on the version the store has. |
| 128 | await api("", { name: "acme--rocket", defaultBranch: "main" }); |
| 129 | await api("", { name: "acme--empty", defaultBranch: "main" }); |
| 130 | const token = (await api("/acme--rocket/tokens", { scope: "write" })).plaintext; |
| 131 | const seed = join(work, "seed"); |
| 132 | git(["init", "-q", "-b", "main", seed]); |
| 133 | const lines = Array.from({ length: 4000 }, (_, at) => `line ${at} of a file that changes a little at a time\n`); |
| 134 | for (let i = 1; i <= 3; i++) { |
| 135 | lines[i * 100] = `changed in commit ${i}\n`; |
| 136 | writeFileSync(join(seed, "big.txt"), lines.join("")); |
| 137 | git(["add", "."], seed); |
| 138 | git([...identity, "commit", "-q", "-m", `commit ${i}`], seed); |
| 139 | } |
| 140 | git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/acme--rocket.git`, "main"], seed); |
| 141 | // The control: the store's own advertisement, without g1t in front, |
| 142 | // gets a thin pack for the same kind of change. |
| 143 | { |
| 144 | lines[3000] = "changed straight in the store\n"; |
| 145 | writeFileSync(join(seed, "big.txt"), lines.join("")); |
| 146 | git([...identity, "commit", "-q", "-am", "straight"], seed); |
| 147 | const trace = join(work, "control.trace"); |
| 148 | git(["-c", `http.extraHeader=Authorization: Bearer ${token}`, "push", "-q", `${STORE}/git/acme--rocket.git`, "main"], seed, { GIT_TRACE: trace }); |
| 149 | const packing = readFileSync(trace, "utf8").split("\n").find((line) => /run_command: git pack-objects/.test(line)) ?? ""; |
| 150 | check("control: straight to the store, git packs with --thin", /--thin\b/.test(packing), packing.replace(/.*run_command: /, "")); |
| 151 | } |
| 152 | |
| 153 | run("node", [WRANGLER, "d1", "migrations", "apply", "g1t-repos", "--local", "--persist-to", persist, "-c", "dev/repos.jsonc"], { |
| 154 | cwd: service, |
| 155 | env: { ...process.env, CI: "1" }, |
| 156 | }); |
| 157 | sql( |
| 158 | "INSERT INTO repos (id, namespace, name, is_private, owner_id, default_branch, refs_version) VALUES " + |
| 159 | "('rep_rocket', 'acme', 'rocket', 0, 'usr_dev', 'main', 1), ('rep_empty', 'acme', 'empty', 0, 'usr_dev', 'main', 1)", |
| 160 | ); |
| 161 | start( |
| 162 | "node", |
| 163 | [WRANGLER, "dev", "-c", "dev/repos.jsonc", "-c", ARTIFACTS_CONFIG, "-c", "dev/stubs.jsonc", "--local", "--persist-to", persist, "--port", REPOS_PORT], |
| 164 | { cwd: service, env: { ...process.env, CI: "1" }, stdio: ["ignore", "inherit", "inherit"] }, |
| 165 | ); |
| 166 | await waitFor(`${REPOS}/acme/rocket.git/info/refs?service=git-upload-pack`, "wrangler dev"); |
| 167 | |
| 168 | // One line of the big file changed: thin, the pack would be a small |
| 169 | // delta on the store's copy; whole, it carries the file. |
| 170 | const clone = join(work, "clone"); |
| 171 | git(["clone", "-q", `${REPOS}/acme/rocket.git`, clone]); |
| 172 | lines[2000] = "changed by the push\n"; |
| 173 | writeFileSync(join(clone, "big.txt"), lines.join("")); |
| 174 | git([...identity, "commit", "-q", "-am", "one line"], clone); |
| 175 | push("existing repository", clone, "rocket"); |
| 176 | const pushed = git(["ls-remote", `${STORE}/git/acme--rocket.git`, "refs/heads/main"], work, { |
| 177 | GIT_CONFIG_COUNT: "1", |
| 178 | GIT_CONFIG_KEY_0: "http.extraHeader", |
| 179 | GIT_CONFIG_VALUE_0: `Authorization: Bearer ${token}`, |
| 180 | }).stdout.split(/\s/)[0]; |
| 181 | const local = git(["rev-parse", "HEAD"], clone).stdout.trim(); |
| 182 | check("existing repository: the store has the pushed commit", pushed === local, `${pushed} / ${local}`); |
| 183 | |
| 184 | // An empty repository advertises `capabilities^{}`. |
| 185 | push("empty repository", clone, "empty"); |
| 186 | } catch (error) { |
| 187 | console.error(error); |
| 188 | checks.push({ what: "ran", ok: false }); |
| 189 | } finally { |
| 190 | for (const child of children) { |
| 191 | if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"], { stdio: "ignore" }); |
| 192 | else child.kill(); |
| 193 | } |
| 194 | try { |
| 195 | rmSync(work, { recursive: true, force: true }); |
| 196 | } catch {} |
| 197 | } |
| 198 | |
| 199 | const failed = checks.filter((c) => !c.ok); |
| 200 | console.log(failed.length ? `\n${failed.length} of ${checks.length} checks failed.` : `\nAll ${checks.length} checks passed.`); |
| 201 | process.exit(failed.length ? 1 : 0); |