Sign in

pulls/pr_01m47d15m3e54sn21z27rpy5n9

public

pr_01m47d15m3e54sn21z27rpy5n9/deploy/self-host/gitstore/server.mjs

504 lines19,711 bytesCodeBlame
1// g1t's git store for self-hosting: plain bare repositories on disk.
2//
3// Hosted g1t keeps repositories in Cloudflare Artifacts. This server does
4// the same job with nothing but git: one bare repository per store key
5// under GITSTORE_ROOT, git's own smart HTTP (git http-backend) for clones,
6// fetches and pushes, and a small JSON API for the reads the repos service
7// makes (commits, trees, blobs, files) and for creating and forking.
8//
9// It is reached only by the Artifacts-compatible shim (workers/artifacts),
10// which the repos service is bound to in place of the Artifacts binding, and
11// by the repos service itself for git's smart HTTP. Nothing else should be
12// able to reach it: the API takes a shared secret, and git requests a
13// short-lived token the shim minted with the same secret.
14//
15// No dependencies beyond Node and git.
16
17import { spawn } from "node:child_process";
18import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
19import { existsSync, mkdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs";
20import { createServer } from "node:http";
21import { rm } from "node:fs/promises";
22import { dirname, join } from "node:path";
23
24const ROOT = process.env.GITSTORE_ROOT ?? "/data/git";
25const PORT = Number(process.env.GITSTORE_PORT ?? 8080);
26const SECRET = loadSecret();
27// How the repos service reaches this server; it becomes each repository's
28// `remote`, exactly as Artifacts hands one out.
29const PUBLIC_URL = (process.env.GITSTORE_URL ?? `http://localhost:${PORT}`).replace(/\/$/, "");
30
31/**
32 * The secret shared with the Artifacts shim: GITSTORE_SECRET, or else the
33 * one in GITSTORE_SECRET_FILE, made on first start. The compose file shares
34 * that file with the g1t container, so nobody has to choose one.
35 */
36function loadSecret() {
37 if (process.env.GITSTORE_SECRET) return process.env.GITSTORE_SECRET;
38 const file = process.env.GITSTORE_SECRET_FILE;
39 if (!file) return "";
40 if (!existsSync(file)) {
41 mkdirSync(dirname(file), { recursive: true });
42 writeFileSync(file, randomBytes(32).toString("hex"), { mode: 0o600 });
43 }
44 return readFileSync(file, "utf8").trim();
45}
46
47if (SECRET.length < 16) {
48 console.error("Set GITSTORE_SECRET (16 characters or more) or GITSTORE_SECRET_FILE.");
49 process.exit(1);
50}
51mkdirSync(ROOT, { recursive: true });
52
53const KEY = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,199}$/;
54const HASH = /^[0-9a-f]{40}$/;
55
56class StoreError extends Error {
57 constructor(code, message, status = 400) {
58 super(message);
59 this.code = code;
60 this.status = status;
61 }
62}
63
64function repoDir(key) {
65 if (!KEY.test(key) || key.includes("..")) {
66 throw new StoreError("INVALID_REPO_NAME", `invalid repository name: ${key}`);
67 }
68 return join(ROOT, `${key}.git`);
69}
70
71function exists(key) {
72 return existsSync(join(repoDir(key), "HEAD"));
73}
74
75function requireRepo(key) {
76 if (!exists(key)) throw new StoreError("NOT_FOUND", `no repository ${key}`, 404);
77 return repoDir(key);
78}
79
80/** Runs git and resolves with its stdout as a Buffer. */
81function git(args, { cwd, input, allowFail = false } = {}) {
82 return new Promise((resolve, reject) => {
83 const child = spawn("git", args, { cwd, stdio: ["pipe", "pipe", "pipe"] });
84 const out = [];
85 const err = [];
86 child.stdout.on("data", (chunk) => out.push(chunk));
87 child.stderr.on("data", (chunk) => err.push(chunk));
88 child.on("error", reject);
89 child.on("close", (code) => {
90 if (code !== 0 && !allowFail) {
91 reject(new StoreError("INTERNAL_ERROR", `git ${args[0]} failed: ${Buffer.concat(err)}`, 500));
92 } else {
93 resolve({ code, stdout: Buffer.concat(out) });
94 }
95 });
96 child.stdin.end(input ?? undefined);
97 });
98}
99
100// ── Metadata kept beside each repository ────────────────────────────────
101
102function metaPath(key) {
103 return join(repoDir(key), "g1t.json");
104}
105
106function readMeta(key) {
107 try {
108 return JSON.parse(readFileSync(metaPath(key), "utf8"));
109 } catch {
110 return {};
111 }
112}
113
114function writeMeta(key, meta) {
115 writeFileSync(metaPath(key), JSON.stringify(meta, null, 2));
116}
117
118async function info(key) {
119 const dir = requireRepo(key);
120 const meta = readMeta(key);
121 const head = (await git(["symbolic-ref", "--short", "HEAD"], { cwd: dir, allowFail: true })).stdout
122 .toString()
123 .trim();
124 let lastPushAt = null;
125 try {
126 lastPushAt = statSync(join(dir, "g1t-pushed")).mtime.toISOString();
127 } catch {}
128 return {
129 id: meta.id ?? key,
130 name: key,
131 description: meta.description ?? null,
132 defaultBranch: head || "main",
133 createdAt: meta.createdAt ?? new Date(0).toISOString(),
134 updatedAt: lastPushAt ?? meta.createdAt ?? new Date(0).toISOString(),
135 lastPushAt,
136 source: meta.source ?? null,
137 readOnly: Boolean(meta.readOnly),
138 remote: `${PUBLIC_URL}/git/${key}.git`,
139 };
140}
141
142async function create(key, { description, defaultBranch, readOnly, source } = {}) {
143 const dir = repoDir(key);
144 if (exists(key)) throw new StoreError("ALREADY_EXISTS", `${key} already exists`, 409);
145 mkdirSync(dir, { recursive: true });
146 await git(["init", "--bare", "--quiet", `--initial-branch=${defaultBranch || "main"}`, dir]);
147 await configure(dir);
148 writeMeta(key, {
149 id: randomUUID(),
150 description: description ?? null,
151 createdAt: new Date().toISOString(),
152 readOnly: Boolean(readOnly),
153 source: source ?? null,
154 });
155 return info(key);
156}
157
158async function configure(dir) {
159 // Pushes arrive through git http-backend; the token has already been
160 // checked, so receive-pack is allowed for every write-scoped request.
161 await git(["config", "http.receivepack", "true"], { cwd: dir });
162 await git(["config", "receive.denyNonFastForwards", "false"], { cwd: dir });
163 await git(["config", "uploadpack.allowAnySHA1InWant", "true"], { cwd: dir });
164}
165
166async function fork(key, target, { description, readOnly, defaultBranchOnly = true } = {}) {
167 const source = requireRepo(key);
168 const dir = repoDir(target);
169 if (exists(target)) throw new StoreError("ALREADY_EXISTS", `${target} already exists`, 409);
170 const args = ["clone", "--bare", "--quiet", "--no-tags"];
171 if (defaultBranchOnly) args.push("--single-branch");
172 // A local clone hard-links the objects: cheap, and independent of the
173 // source from then on.
174 args.push(source, dir);
175 await git(args);
176 await git(["remote", "remove", "origin"], { cwd: dir, allowFail: true });
177 await configure(dir);
178 writeMeta(target, {
179 id: randomUUID(),
180 description: description ?? readMeta(key).description ?? null,
181 createdAt: new Date().toISOString(),
182 readOnly: Boolean(readOnly),
183 source: `artifacts:${key}`,
184 });
185 return info(target);
186}
187
188// ── Reading objects ─────────────────────────────────────────────────────
189
190async function objectType(dir, spec) {
191 const { code, stdout } = await git(["cat-file", "-t", "--", spec], { cwd: dir, allowFail: true });
192 return code === 0 ? stdout.toString().trim() : null;
193}
194
195function person(line) {
196 // `Name <email> 1700000000 +0000`
197 const match = /^(.*) <([^>]*)> (\d+) [+-]\d{4}$/.exec(line);
198 return match ? { name: match[1], email: match[2], at: Number(match[3]) } : { name: line, email: "", at: 0 };
199}
200
201function parseCommit(hash, raw) {
202 const text = raw.toString("utf8");
203 const split = text.indexOf("\n\n");
204 const headers = (split === -1 ? text : text.slice(0, split)).split("\n");
205 let message = split === -1 ? "" : text.slice(split + 2);
206 if (message.endsWith("\n")) message = message.slice(0, -1);
207 const commit = { hash, treeHash: "", message, parents: [], author: null, committer: null };
208 for (const header of headers) {
209 const space = header.indexOf(" ");
210 const name = header.slice(0, space);
211 const value = header.slice(space + 1);
212 if (name === "tree") commit.treeHash = value;
213 else if (name === "parent") commit.parents.push(value);
214 else if (name === "author") commit.author = person(value);
215 else if (name === "committer") commit.committer = person(value);
216 }
217 const author = commit.author ?? { name: "", email: "", at: 0 };
218 const committer = commit.committer ?? author;
219 return {
220 hash,
221 treeHash: commit.treeHash,
222 message: commit.message,
223 author: { name: author.name, email: author.email },
224 committer: { name: committer.name, email: committer.email },
225 parents: commit.parents,
226 authoredAt: author.at,
227 committedAt: committer.at,
228 };
229}
230
231async function readCommit(key, hash) {
232 const dir = requireRepo(key);
233 if (!HASH.test(hash)) return null;
234 if ((await objectType(dir, hash)) !== "commit") return null;
235 return parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout);
236}
237
238async function log(key, { ref = "HEAD", limit = 50, offset = 0 } = {}) {
239 const dir = requireRepo(key);
240 if (typeof ref !== "string" || ref.startsWith("-")) return [];
241 const count = Math.max(1, Math.min(Number(limit) || 50, 1000));
242 const skip = Math.max(0, Number(offset) || 0);
243 const listed = await git(
244 ["rev-list", "--first-parent", `--max-count=${count}`, `--skip=${skip}`, ref, "--"],
245 { cwd: dir, allowFail: true },
246 );
247 if (listed.code !== 0) return [];
248 const hashes = listed.stdout.toString().split("\n").filter(Boolean);
249 const commits = [];
250 for (const hash of hashes) {
251 commits.push(parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout));
252 }
253 return commits;
254}
255
256const TYPES = { "040000": "tree", "100644": "blob", "100755": "exec", "120000": "symlink", "160000": "gitlink" };
257
258async function readTree(key, hash) {
259 const dir = requireRepo(key);
260 if (!HASH.test(hash)) return null;
261 if ((await objectType(dir, hash)) !== "tree") return null;
262 const { stdout } = await git(["ls-tree", "-z", hash], { cwd: dir });
263 return stdout
264 .toString("utf8")
265 .split("\0")
266 .filter(Boolean)
267 .map((line) => {
268 const tab = line.indexOf("\t");
269 const [mode, , object] = line.slice(0, tab).split(" ");
270 return {
271 name: line.slice(tab + 1),
272 mode: mode === "040000" ? "40000" : mode,
273 hash: object,
274 type: TYPES[mode] ?? "blob",
275 };
276 });
277}
278
279async function readBlob(key, hash) {
280 const dir = requireRepo(key);
281 if (!HASH.test(hash)) return null;
282 if ((await objectType(dir, hash)) !== "blob") return null;
283 return (await git(["cat-file", "blob", hash], { cwd: dir })).stdout;
284}
285
286async function readFile(key, ref, path) {
287 const dir = requireRepo(key);
288 if (!ref || !path || ref.startsWith("-") || ref.includes(":")) return null;
289 const spec = `${ref}:${path.replace(/^\/+/, "")}`;
290 if ((await objectType(dir, spec)) !== "blob") return null;
291 return (await git(["cat-file", "blob", spec], { cwd: dir })).stdout;
292}
293
294// ── Tokens for git's smart HTTP ─────────────────────────────────────────
295
296function sign(payload) {
297 return createHmac("sha256", SECRET).update(payload).digest("base64url");
298}
299
300function mintToken(key, scope = "write", ttl = 86400) {
301 const seconds = Math.max(60, Math.min(Number(ttl) || 86400, 31536000));
302 const expires = Math.floor(Date.now() / 1000) + seconds;
303 const id = randomUUID();
304 const payload = Buffer.from(JSON.stringify({ k: key, s: scope, e: expires, i: id })).toString("base64url");
305 return {
306 id,
307 plaintext: `${payload}.${sign(payload)}`,
308 scope,
309 expiresAt: new Date(expires * 1000).toISOString(),
310 };
311}
312
313function checkToken(token, key) {
314 const [payload, signature] = String(token ?? "").split(".");
315 if (!payload || !signature) return null;
316 const expected = Buffer.from(sign(payload));
317 const given = Buffer.from(signature);
318 if (expected.length !== given.length || !timingSafeEqual(expected, given)) return null;
319 const claims = JSON.parse(Buffer.from(payload, "base64url").toString());
320 if (claims.k !== key || claims.e < Date.now() / 1000) return null;
321 return claims;
322}
323
324function bearer(request) {
325 const header = request.headers.authorization ?? "";
326 if (/^bearer /i.test(header)) return header.slice(7).trim();
327 if (/^basic /i.test(header)) {
328 // A git client given the token as a password: `x:<token>`.
329 const decoded = Buffer.from(header.slice(6).trim(), "base64").toString();
330 return decoded.slice(decoded.indexOf(":") + 1);
331 }
332 return null;
333}
334
335// ── Smart HTTP through git http-backend ─────────────────────────────────
336
337function smartHttp(request, response, key, rest, query) {
338 if (!exists(key)) return send(response, 404, "not found");
339 const claims = checkToken(bearer(request), key);
340 if (!claims) {
341 response.writeHead(401, { "www-authenticate": 'Basic realm="g1t-gitstore"' });
342 return response.end("unauthorized");
343 }
344 const service = rest === "info/refs" ? new URLSearchParams(query).get("service") : rest;
345 if (service === "git-receive-pack" && claims.s !== "write") return send(response, 403, "read-only token");
346 if (service !== "git-upload-pack" && service !== "git-receive-pack") return send(response, 404, "not found");
347
348 const env = {
349 PATH: process.env.PATH,
350 GIT_PROJECT_ROOT: ROOT,
351 GIT_HTTP_EXPORT_ALL: "1",
352 REQUEST_METHOD: request.method,
353 PATH_INFO: `/${key}.git/${rest}`,
354 QUERY_STRING: query,
355 CONTENT_TYPE: request.headers["content-type"] ?? "",
356 REMOTE_USER: "g1t",
357 REMOTE_ADDR: request.socket.remoteAddress ?? "",
358 };
359 if (request.headers["git-protocol"]) env.GIT_PROTOCOL = request.headers["git-protocol"];
360 if (request.headers["content-encoding"]) env.HTTP_CONTENT_ENCODING = request.headers["content-encoding"];
361 if (request.headers["content-length"]) env.CONTENT_LENGTH = request.headers["content-length"];
362
363 const child = spawn("git", ["http-backend"], { env, stdio: ["pipe", "pipe", "pipe"] });
364 request.pipe(child.stdin);
365 child.stderr.on("data", (chunk) => process.stderr.write(chunk));
366
367 // CGI: headers, a blank line, then the body.
368 let buffered = Buffer.alloc(0);
369 let headersDone = false;
370 child.stdout.on("data", (chunk) => {
371 if (headersDone) return response.write(chunk);
372 buffered = Buffer.concat([buffered, chunk]);
373 let end = buffered.indexOf("\r\n\r\n");
374 let gap = 4;
375 if (end === -1) {
376 end = buffered.indexOf("\n\n");
377 gap = 2;
378 }
379 if (end === -1) return;
380 headersDone = true;
381 let status = 200;
382 const headers = {};
383 for (const line of buffered.slice(0, end).toString().split(/\r?\n/)) {
384 const colon = line.indexOf(":");
385 if (colon === -1) continue;
386 const name = line.slice(0, colon).trim().toLowerCase();
387 const value = line.slice(colon + 1).trim();
388 if (name === "status") status = Number.parseInt(value, 10);
389 else headers[name] = value;
390 }
391 response.writeHead(status, headers);
392 response.write(buffered.slice(end + gap));
393 });
394 child.on("close", (code) => {
395 if (!headersDone) {
396 send(response, 500, "git http-backend failed");
397 return;
398 }
399 if (service === "git-receive-pack" && request.method === "POST" && code === 0) {
400 const marker = join(repoDir(key), "g1t-pushed");
401 try {
402 utimesSync(marker, new Date(), new Date());
403 } catch {
404 writeFileSync(marker, "");
405 }
406 }
407 response.end();
408 });
409}
410
411// ── HTTP ────────────────────────────────────────────────────────────────
412
413function send(response, status, body, headers = {}) {
414 const isBuffer = Buffer.isBuffer(body);
415 const payload = isBuffer ? body : typeof body === "string" ? body : JSON.stringify(body);
416 response.writeHead(status, {
417 "content-type": isBuffer ? "application/octet-stream" : typeof body === "string" ? "text/plain" : "application/json",
418 ...headers,
419 });
420 response.end(payload);
421}
422
423async function readJson(request) {
424 const chunks = [];
425 for await (const chunk of request) chunks.push(chunk);
426 const text = Buffer.concat(chunks).toString();
427 return text ? JSON.parse(text) : {};
428}
429
430function authorized(request) {
431 const given = Buffer.from(request.headers["x-gitstore-secret"] ?? "");
432 const expected = Buffer.from(SECRET);
433 return given.length === expected.length && timingSafeEqual(given, expected);
434}
435
436async function api(request, response, parts, params) {
437 if (!authorized(request)) return send(response, 401, { code: "UNAUTHORIZED", message: "bad secret" });
438 const method = request.method;
439 // POST /api/repos create
440 if (parts.length === 0 && method === "POST") {
441 const body = await readJson(request);
442 return send(response, 200, await create(body.name, body));
443 }
444 const [key, action, arg] = parts;
445 if (method === "GET" && !action) return send(response, 200, await info(key));
446 // DELETE /api/repos/<key> delete (a purged repository)
447 if (method === "DELETE" && !action) {
448 if (!exists(key)) return send(response, 404, { code: "NOT_FOUND", message: "no such repository" });
449 await rm(repoDir(key), { recursive: true, force: true });
450 return send(response, 200, { deleted: true });
451 }
452 if (method === "POST" && action === "tokens") {
453 requireRepo(key);
454 const body = await readJson(request);
455 return send(response, 200, mintToken(key, body.scope, body.ttl));
456 }
457 if (method === "POST" && action === "fork") {
458 const body = await readJson(request);
459 return send(response, 200, await fork(key, body.name, body));
460 }
461 if (method === "GET" && action === "commits") {
462 return send(response, 200, await readCommit(key, arg));
463 }
464 if (method === "GET" && action === "log") {
465 return send(response, 200, await log(key, Object.fromEntries(params)));
466 }
467 if (method === "GET" && action === "trees") {
468 return send(response, 200, await readTree(key, arg));
469 }
470 if (method === "GET" && (action === "blobs" || action === "file")) {
471 const bytes =
472 action === "blobs" ? await readBlob(key, arg) : await readFile(key, params.get("ref"), params.get("path"));
473 return bytes ? send(response, 200, bytes) : send(response, 404, { code: "NOT_FOUND", message: "no such object" });
474 }
475 return send(response, 404, { code: "NOT_FOUND", message: "no such route" });
476}
477
478const server = createServer(async (request, response) => {
479 const url = new URL(request.url, "http://gitstore");
480 try {
481 if (url.pathname === "/healthz") return send(response, 200, "ok");
482 const git = /^\/git\/([^/]+)\.git\/(info\/refs|git-upload-pack|git-receive-pack)$/.exec(url.pathname);
483 if (git) return smartHttp(request, response, decodeURIComponent(git[1]), git[2], url.search.slice(1));
484 if (url.pathname === "/api/repos" || url.pathname.startsWith("/api/repos/")) {
485 const parts = url.pathname.slice("/api/repos".length).split("/").filter(Boolean).map(decodeURIComponent);
486 return await api(request, response, parts, url.searchParams);
487 }
488 send(response, 404, "not found");
489 } catch (error) {
490 const status = error instanceof StoreError ? error.status : 500;
491 const code = error instanceof StoreError ? error.code : "INTERNAL_ERROR";
492 if (status >= 500) console.error(error);
493 if (!response.headersSent) send(response, status, { code, message: error.message });
494 else response.end();
495 }
496});
497
498server.listen(PORT, () => {
499 console.log(`g1t gitstore: ${ROOT} on :${PORT} (remote ${PUBLIC_URL})`);
500});
501
502for (const signal of ["SIGINT", "SIGTERM"]) {
503 process.on(signal, () => server.close(() => process.exit(0)));
504}