Sign in

flagon-io/g1t

public

g1t/deploy/self-host/gitstore/server.mjs

531 lines21,109 bytesCodeBlame
1// g1t's git store for self-hosting: plain bare repositories on disk.
2//
3// Hosted g1t keeps repositories in Cloudflare Artifacts. This server does
4// the same job with nothing but git: one bare repository per store key
5// under GITSTORE_ROOT, git's own smart HTTP (git http-backend) for clones,
6// fetches and pushes, and a small JSON API for the reads the repos service
7// makes (commits, trees, blobs, files) and for creating and forking.
8//
9// It is reached only by the Artifacts-compatible shim (workers/artifacts),
10// which the repos service is bound to in place of the Artifacts binding, and
11// by the repos service itself for git's smart HTTP. Nothing else should be
12// able to reach it: the API takes a shared secret, and git requests a
13// short-lived token the shim minted with the same secret.
14//
15// A key is a repository's name (`acme--rocket`), or a namespace and a name
16// (`g1t/acme--rocket`): hosted g1t's fallback store (docs/ARTIFACTS.md, R12)
17// keeps each Artifacts namespace's repositories in a directory of their
18// own, so a remote reads `<GITSTORE_URL>/git/<namespace>/<name>.git`, the
19// shape Artifacts gives remotes.
20//
21// GITSTORE_READ_ONLY=1 refuses everything that writes: pushes, creating,
22// forking, deleting, and minting write tokens. As a fallback the store
23// serves reads until told otherwise; the repos service refuses writes too.
24//
25// No dependencies beyond Node and git.
26
27import { spawn } from "node:child_process";
28import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
29import { existsSync, mkdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs";
30import { createServer } from "node:http";
31import { rm } from "node:fs/promises";
32import { dirname, join } from "node:path";
33
34const ROOT = process.env.GITSTORE_ROOT ?? "/data/git";
35const PORT = Number(process.env.GITSTORE_PORT ?? 8080);
36const SECRET = loadSecret();
37// How the repos service reaches this server; it becomes each repository's
38// `remote`, exactly as Artifacts hands one out.
39const PUBLIC_URL = (process.env.GITSTORE_URL ?? `http://localhost:${PORT}`).replace(/\/$/, "");
40const READ_ONLY = ["1", "true", "yes"].includes(String(process.env.GITSTORE_READ_ONLY ?? "").toLowerCase());
41
42/**
43 * The secret shared with the Artifacts shim: GITSTORE_SECRET, or else the
44 * one in GITSTORE_SECRET_FILE, made on first start. The compose file shares
45 * that file with the g1t container, so nobody has to choose one.
46 */
47function loadSecret() {
48 if (process.env.GITSTORE_SECRET) return process.env.GITSTORE_SECRET;
49 const file = process.env.GITSTORE_SECRET_FILE;
50 if (!file) return "";
51 if (!existsSync(file)) {
52 mkdirSync(dirname(file), { recursive: true });
53 writeFileSync(file, randomBytes(32).toString("hex"), { mode: 0o600 });
54 }
55 return readFileSync(file, "utf8").trim();
56}
57
58if (SECRET.length < 16) {
59 console.error("Set GITSTORE_SECRET (16 characters or more) or GITSTORE_SECRET_FILE.");
60 process.exit(1);
61}
62mkdirSync(ROOT, { recursive: true });
63
64const NAME = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,199}$/;
65const HASH = /^[0-9a-f]{40}$/;
66
67class StoreError extends Error {
68 constructor(code, message, status = 400) {
69 super(message);
70 this.code = code;
71 this.status = status;
72 }
73}
74
75/** Whether `key` is a name, or a namespace and a name. */
76function validKey(key) {
77 if (typeof key !== "string" || key.includes("..")) return false;
78 const parts = key.split("/");
79 return parts.length <= 2 && parts.every((part) => NAME.test(part));
80}
81
82function repoDir(key) {
83 if (!validKey(key)) {
84 throw new StoreError("INVALID_REPO_NAME", `invalid repository name: ${key}`);
85 }
86 return join(ROOT, `${key}.git`);
87}
88
89function refuseWrites(what) {
90 if (READ_ONLY) throw new StoreError("READ_ONLY", `the git store is read-only: ${what} is refused`, 403);
91}
92
93function exists(key) {
94 return existsSync(join(repoDir(key), "HEAD"));
95}
96
97function requireRepo(key) {
98 if (!exists(key)) throw new StoreError("NOT_FOUND", `no repository ${key}`, 404);
99 return repoDir(key);
100}
101
102/** Runs git and resolves with its stdout as a Buffer. */
103function git(args, { cwd, input, allowFail = false } = {}) {
104 return new Promise((resolve, reject) => {
105 const child = spawn("git", args, { cwd, stdio: ["pipe", "pipe", "pipe"] });
106 const out = [];
107 const err = [];
108 child.stdout.on("data", (chunk) => out.push(chunk));
109 child.stderr.on("data", (chunk) => err.push(chunk));
110 child.on("error", reject);
111 child.on("close", (code) => {
112 if (code !== 0 && !allowFail) {
113 reject(new StoreError("INTERNAL_ERROR", `git ${args[0]} failed: ${Buffer.concat(err)}`, 500));
114 } else {
115 resolve({ code, stdout: Buffer.concat(out) });
116 }
117 });
118 child.stdin.end(input ?? undefined);
119 });
120}
121
122// ── Metadata kept beside each repository ────────────────────────────────
123
124function metaPath(key) {
125 return join(repoDir(key), "g1t.json");
126}
127
128function readMeta(key) {
129 try {
130 return JSON.parse(readFileSync(metaPath(key), "utf8"));
131 } catch {
132 return {};
133 }
134}
135
136function writeMeta(key, meta) {
137 writeFileSync(metaPath(key), JSON.stringify(meta, null, 2));
138}
139
140async function info(key) {
141 const dir = requireRepo(key);
142 const meta = readMeta(key);
143 const head = (await git(["symbolic-ref", "--short", "HEAD"], { cwd: dir, allowFail: true })).stdout
144 .toString()
145 .trim();
146 let lastPushAt = null;
147 try {
148 lastPushAt = statSync(join(dir, "g1t-pushed")).mtime.toISOString();
149 } catch {}
150 return {
151 id: meta.id ?? key,
152 name: key,
153 description: meta.description ?? null,
154 defaultBranch: head || "main",
155 createdAt: meta.createdAt ?? new Date(0).toISOString(),
156 updatedAt: lastPushAt ?? meta.createdAt ?? new Date(0).toISOString(),
157 lastPushAt,
158 source: meta.source ?? null,
159 readOnly: Boolean(meta.readOnly),
160 remote: `${PUBLIC_URL}/git/${key}.git`,
161 };
162}
163
164async function create(key, { description, defaultBranch, readOnly, source } = {}) {
165 refuseWrites("creating a repository");
166 const dir = repoDir(key);
167 if (exists(key)) throw new StoreError("ALREADY_EXISTS", `${key} already exists`, 409);
168 mkdirSync(dir, { recursive: true });
169 await git(["init", "--bare", "--quiet", `--initial-branch=${defaultBranch || "main"}`, dir]);
170 await configure(dir);
171 writeMeta(key, {
172 id: randomUUID(),
173 description: description ?? null,
174 createdAt: new Date().toISOString(),
175 readOnly: Boolean(readOnly),
176 source: source ?? null,
177 });
178 return info(key);
179}
180
181async function configure(dir) {
182 // Pushes arrive through git http-backend; the token has already been
183 // checked, so receive-pack is allowed for every write-scoped request.
184 await git(["config", "http.receivepack", "true"], { cwd: dir });
185 await git(["config", "receive.denyNonFastForwards", "false"], { cwd: dir });
186 await git(["config", "uploadpack.allowAnySHA1InWant", "true"], { cwd: dir });
187}
188
189async function fork(key, target, { description, readOnly, defaultBranchOnly = true } = {}) {
190 refuseWrites("forking");
191 const source = requireRepo(key);
192 const dir = repoDir(target);
193 if (exists(target)) throw new StoreError("ALREADY_EXISTS", `${target} already exists`, 409);
194 const args = ["clone", "--bare", "--quiet", "--no-tags"];
195 if (defaultBranchOnly) args.push("--single-branch");
196 // A local clone hard-links the objects: cheap, and independent of the
197 // source from then on.
198 args.push(source, dir);
199 await git(args);
200 await git(["remote", "remove", "origin"], { cwd: dir, allowFail: true });
201 await configure(dir);
202 writeMeta(target, {
203 id: randomUUID(),
204 description: description ?? readMeta(key).description ?? null,
205 createdAt: new Date().toISOString(),
206 readOnly: Boolean(readOnly),
207 source: `artifacts:${key}`,
208 });
209 return info(target);
210}
211
212// ── Reading objects ─────────────────────────────────────────────────────
213
214async function objectType(dir, spec) {
215 const { code, stdout } = await git(["cat-file", "-t", "--", spec], { cwd: dir, allowFail: true });
216 return code === 0 ? stdout.toString().trim() : null;
217}
218
219function person(line) {
220 // `Name <email> 1700000000 +0000`
221 const match = /^(.*) <([^>]*)> (\d+) [+-]\d{4}$/.exec(line);
222 return match ? { name: match[1], email: match[2], at: Number(match[3]) } : { name: line, email: "", at: 0 };
223}
224
225function parseCommit(hash, raw) {
226 const text = raw.toString("utf8");
227 const split = text.indexOf("\n\n");
228 const headers = (split === -1 ? text : text.slice(0, split)).split("\n");
229 let message = split === -1 ? "" : text.slice(split + 2);
230 if (message.endsWith("\n")) message = message.slice(0, -1);
231 const commit = { hash, treeHash: "", message, parents: [], author: null, committer: null };
232 for (const header of headers) {
233 const space = header.indexOf(" ");
234 const name = header.slice(0, space);
235 const value = header.slice(space + 1);
236 if (name === "tree") commit.treeHash = value;
237 else if (name === "parent") commit.parents.push(value);
238 else if (name === "author") commit.author = person(value);
239 else if (name === "committer") commit.committer = person(value);
240 }
241 const author = commit.author ?? { name: "", email: "", at: 0 };
242 const committer = commit.committer ?? author;
243 return {
244 hash,
245 treeHash: commit.treeHash,
246 message: commit.message,
247 author: { name: author.name, email: author.email },
248 committer: { name: committer.name, email: committer.email },
249 parents: commit.parents,
250 authoredAt: author.at,
251 committedAt: committer.at,
252 };
253}
254
255async function readCommit(key, hash) {
256 const dir = requireRepo(key);
257 if (!HASH.test(hash)) return null;
258 if ((await objectType(dir, hash)) !== "commit") return null;
259 return parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout);
260}
261
262async function log(key, { ref = "HEAD", limit = 50, offset = 0 } = {}) {
263 const dir = requireRepo(key);
264 if (typeof ref !== "string" || ref.startsWith("-")) return [];
265 const count = Math.max(1, Math.min(Number(limit) || 50, 1000));
266 const skip = Math.max(0, Number(offset) || 0);
267 const listed = await git(
268 ["rev-list", "--first-parent", `--max-count=${count}`, `--skip=${skip}`, ref, "--"],
269 { cwd: dir, allowFail: true },
270 );
271 if (listed.code !== 0) return [];
272 const hashes = listed.stdout.toString().split("\n").filter(Boolean);
273 const commits = [];
274 for (const hash of hashes) {
275 commits.push(parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout));
276 }
277 return commits;
278}
279
280const TYPES = { "040000": "tree", "100644": "blob", "100755": "exec", "120000": "symlink", "160000": "gitlink" };
281
282async function readTree(key, hash) {
283 const dir = requireRepo(key);
284 if (!HASH.test(hash)) return null;
285 if ((await objectType(dir, hash)) !== "tree") return null;
286 const { stdout } = await git(["ls-tree", "-z", hash], { cwd: dir });
287 return stdout
288 .toString("utf8")
289 .split("\0")
290 .filter(Boolean)
291 .map((line) => {
292 const tab = line.indexOf("\t");
293 const [mode, , object] = line.slice(0, tab).split(" ");
294 return {
295 name: line.slice(tab + 1),
296 mode: mode === "040000" ? "40000" : mode,
297 hash: object,
298 type: TYPES[mode] ?? "blob",
299 };
300 });
301}
302
303async function readBlob(key, hash) {
304 const dir = requireRepo(key);
305 if (!HASH.test(hash)) return null;
306 if ((await objectType(dir, hash)) !== "blob") return null;
307 return (await git(["cat-file", "blob", hash], { cwd: dir })).stdout;
308}
309
310async function readFile(key, ref, path) {
311 const dir = requireRepo(key);
312 if (!ref || !path || ref.startsWith("-") || ref.includes(":")) return null;
313 const spec = `${ref}:${path.replace(/^\/+/, "")}`;
314 if ((await objectType(dir, spec)) !== "blob") return null;
315 return (await git(["cat-file", "blob", spec], { cwd: dir })).stdout;
316}
317
318// ── Tokens for git's smart HTTP ─────────────────────────────────────────
319
320function sign(payload) {
321 return createHmac("sha256", SECRET).update(payload).digest("base64url");
322}
323
324function mintToken(key, scope = "write", ttl = 86400) {
325 if (scope === "write") refuseWrites("a write token");
326 const seconds = Math.max(60, Math.min(Number(ttl) || 86400, 31536000));
327 const expires = Math.floor(Date.now() / 1000) + seconds;
328 const id = randomUUID();
329 const payload = Buffer.from(JSON.stringify({ k: key, s: scope, e: expires, i: id })).toString("base64url");
330 return {
331 id,
332 plaintext: `${payload}.${sign(payload)}`,
333 scope,
334 expiresAt: new Date(expires * 1000).toISOString(),
335 };
336}
337
338function checkToken(token, key) {
339 const [payload, signature] = String(token ?? "").split(".");
340 if (!payload || !signature) return null;
341 const expected = Buffer.from(sign(payload));
342 const given = Buffer.from(signature);
343 if (expected.length !== given.length || !timingSafeEqual(expected, given)) return null;
344 const claims = JSON.parse(Buffer.from(payload, "base64url").toString());
345 if (claims.k !== key || claims.e < Date.now() / 1000) return null;
346 return claims;
347}
348
349function bearer(request) {
350 const header = request.headers.authorization ?? "";
351 if (/^bearer /i.test(header)) return header.slice(7).trim();
352 if (/^basic /i.test(header)) {
353 // A git client given the token as a password: `x:<token>`.
354 const decoded = Buffer.from(header.slice(6).trim(), "base64").toString();
355 return decoded.slice(decoded.indexOf(":") + 1);
356 }
357 return null;
358}
359
360// ── Smart HTTP through git http-backend ─────────────────────────────────
361
362function smartHttp(request, response, key, rest, query) {
363 if (!exists(key)) return send(response, 404, "not found");
364 const claims = checkToken(bearer(request), key);
365 if (!claims) {
366 response.writeHead(401, { "www-authenticate": 'Basic realm="g1t-gitstore"' });
367 return response.end("unauthorized");
368 }
369 const service = rest === "info/refs" ? new URLSearchParams(query).get("service") : rest;
370 if (service === "git-receive-pack" && claims.s !== "write") return send(response, 403, "read-only token");
371 if (service === "git-receive-pack" && READ_ONLY) return send(response, 403, "the git store is read-only");
372 if (service !== "git-upload-pack" && service !== "git-receive-pack") return send(response, 404, "not found");
373
374 const env = {
375 PATH: process.env.PATH,
376 GIT_PROJECT_ROOT: ROOT,
377 GIT_HTTP_EXPORT_ALL: "1",
378 REQUEST_METHOD: request.method,
379 PATH_INFO: `/${key}.git/${rest}`,
380 QUERY_STRING: query,
381 CONTENT_TYPE: request.headers["content-type"] ?? "",
382 REMOTE_USER: "g1t",
383 REMOTE_ADDR: request.socket.remoteAddress ?? "",
384 };
385 if (request.headers["git-protocol"]) env.GIT_PROTOCOL = request.headers["git-protocol"];
386 if (request.headers["content-encoding"]) env.HTTP_CONTENT_ENCODING = request.headers["content-encoding"];
387 if (request.headers["content-length"]) env.CONTENT_LENGTH = request.headers["content-length"];
388
389 const child = spawn("git", ["http-backend"], { env, stdio: ["pipe", "pipe", "pipe"] });
390 request.pipe(child.stdin);
391 child.stderr.on("data", (chunk) => process.stderr.write(chunk));
392
393 // CGI: headers, a blank line, then the body.
394 let buffered = Buffer.alloc(0);
395 let headersDone = false;
396 child.stdout.on("data", (chunk) => {
397 if (headersDone) return response.write(chunk);
398 buffered = Buffer.concat([buffered, chunk]);
399 let end = buffered.indexOf("\r\n\r\n");
400 let gap = 4;
401 if (end === -1) {
402 end = buffered.indexOf("\n\n");
403 gap = 2;
404 }
405 if (end === -1) return;
406 headersDone = true;
407 let status = 200;
408 const headers = {};
409 for (const line of buffered.slice(0, end).toString().split(/\r?\n/)) {
410 const colon = line.indexOf(":");
411 if (colon === -1) continue;
412 const name = line.slice(0, colon).trim().toLowerCase();
413 const value = line.slice(colon + 1).trim();
414 if (name === "status") status = Number.parseInt(value, 10);
415 else headers[name] = value;
416 }
417 response.writeHead(status, headers);
418 response.write(buffered.slice(end + gap));
419 });
420 child.on("close", (code) => {
421 if (!headersDone) {
422 send(response, 500, "git http-backend failed");
423 return;
424 }
425 if (service === "git-receive-pack" && request.method === "POST" && code === 0) {
426 const marker = join(repoDir(key), "g1t-pushed");
427 try {
428 utimesSync(marker, new Date(), new Date());
429 } catch {
430 writeFileSync(marker, "");
431 }
432 }
433 response.end();
434 });
435}
436
437// ── HTTP ────────────────────────────────────────────────────────────────
438
439function send(response, status, body, headers = {}) {
440 const isBuffer = Buffer.isBuffer(body);
441 const payload = isBuffer ? body : typeof body === "string" ? body : JSON.stringify(body);
442 response.writeHead(status, {
443 "content-type": isBuffer ? "application/octet-stream" : typeof body === "string" ? "text/plain" : "application/json",
444 ...headers,
445 });
446 response.end(payload);
447}
448
449async function readJson(request) {
450 const chunks = [];
451 for await (const chunk of request) chunks.push(chunk);
452 const text = Buffer.concat(chunks).toString();
453 return text ? JSON.parse(text) : {};
454}
455
456function authorized(request) {
457 const given = Buffer.from(request.headers["x-gitstore-secret"] ?? "");
458 const expected = Buffer.from(SECRET);
459 return given.length === expected.length && timingSafeEqual(given, expected);
460}
461
462async function api(request, response, parts, params) {
463 if (!authorized(request)) return send(response, 401, { code: "UNAUTHORIZED", message: "bad secret" });
464 const method = request.method;
465 // POST /api/repos create
466 if (parts.length === 0 && method === "POST") {
467 const body = await readJson(request);
468 return send(response, 200, await create(body.name, body));
469 }
470 const [key, action, arg] = parts;
471 if (method === "GET" && !action) return send(response, 200, await info(key));
472 // DELETE /api/repos/<key> delete (a purged repository)
473 if (method === "DELETE" && !action) {
474 refuseWrites("deleting a repository");
475 if (!exists(key)) return send(response, 404, { code: "NOT_FOUND", message: "no such repository" });
476 await rm(repoDir(key), { recursive: true, force: true });
477 return send(response, 200, { deleted: true });
478 }
479 if (method === "POST" && action === "tokens") {
480 requireRepo(key);
481 const body = await readJson(request);
482 return send(response, 200, mintToken(key, body.scope, body.ttl));
483 }
484 if (method === "POST" && action === "fork") {
485 const body = await readJson(request);
486 return send(response, 200, await fork(key, body.name, body));
487 }
488 if (method === "GET" && action === "commits") {
489 return send(response, 200, await readCommit(key, arg));
490 }
491 if (method === "GET" && action === "log") {
492 return send(response, 200, await log(key, Object.fromEntries(params)));
493 }
494 if (method === "GET" && action === "trees") {
495 return send(response, 200, await readTree(key, arg));
496 }
497 if (method === "GET" && (action === "blobs" || action === "file")) {
498 const bytes =
499 action === "blobs" ? await readBlob(key, arg) : await readFile(key, params.get("ref"), params.get("path"));
500 return bytes ? send(response, 200, bytes) : send(response, 404, { code: "NOT_FOUND", message: "no such object" });
501 }
502 return send(response, 404, { code: "NOT_FOUND", message: "no such route" });
503}
504
505const server = createServer(async (request, response) => {
506 const url = new URL(request.url, "http://gitstore");
507 try {
508 if (url.pathname === "/healthz") return send(response, 200, READ_ONLY ? "ok read-only" : "ok");
509 const git = /^\/git\/((?:[^/]+\/)?[^/]+)\.git\/(info\/refs|git-upload-pack|git-receive-pack)$/.exec(url.pathname);
510 if (git) return smartHttp(request, response, decodeURIComponent(git[1]), git[2], url.search.slice(1));
511 if (url.pathname === "/api/repos" || url.pathname.startsWith("/api/repos/")) {
512 const parts = url.pathname.slice("/api/repos".length).split("/").filter(Boolean).map(decodeURIComponent);
513 return await api(request, response, parts, url.searchParams);
514 }
515 send(response, 404, "not found");
516 } catch (error) {
517 const status = error instanceof StoreError ? error.status : 500;
518 const code = error instanceof StoreError ? error.code : "INTERNAL_ERROR";
519 if (status >= 500) console.error(error);
520 if (!response.headersSent) send(response, status, { code, message: error.message });
521 else response.end();
522 }
523});
524
525server.listen(PORT, () => {
526 console.log(`g1t gitstore: ${ROOT} on :${PORT} (remote ${PUBLIC_URL})${READ_ONLY ? ", read-only" : ""}`);
527});
528
529for (const signal of ["SIGINT", "SIGTERM"]) {
530 process.on(signal, () => server.close(() => process.exit(0)));
531}