Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/deploy/self-host/gitstore/server.mjs

497 lines19,339 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Running g1t yourself: the design, a docker compose proof, and a guide to what works today1// g1t's git store for self-hosting: plain bare repositories on disk.
2//
3// Hosted g1t keeps repositories in Cloudflare Artifacts. This server does
4// the same job with nothing but git: one bare repository per store key
5// under GITSTORE_ROOT, git's own smart HTTP (git http-backend) for clones,
6// fetches and pushes, and a small JSON API for the reads the repos service
7// makes (commits, trees, blobs, files) and for creating and forking.
8//
9// It is reached only by the Artifacts-compatible shim (workers/artifacts),
10// which the repos service is bound to in place of the Artifacts binding, and
11// by the repos service itself for git's smart HTTP. Nothing else should be
12// able to reach it: the API takes a shared secret, and git requests a
13// short-lived token the shim minted with the same secret.
14//
15// No dependencies beyond Node and git.
16
17import { spawn } from "node:child_process";
18import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
19import { existsSync, mkdirSync, readFileSync, statSync, utimesSync, writeFileSync } from "node:fs";
20import { createServer } from "node:http";
21import { dirname, join } from "node:path";
22
23const ROOT = process.env.GITSTORE_ROOT ?? "/data/git";
24const PORT = Number(process.env.GITSTORE_PORT ?? 8080);
25const SECRET = loadSecret();
26// How the repos service reaches this server; it becomes each repository's
27// `remote`, exactly as Artifacts hands one out.
28const PUBLIC_URL = (process.env.GITSTORE_URL ?? `http://localhost:${PORT}`).replace(/\/$/, "");
29
30/**
31 * The secret shared with the Artifacts shim: GITSTORE_SECRET, or else the
32 * one in GITSTORE_SECRET_FILE, made on first start. The compose file shares
33 * that file with the g1t container, so nobody has to choose one.
34 */
35function loadSecret() {
36 if (process.env.GITSTORE_SECRET) return process.env.GITSTORE_SECRET;
37 const file = process.env.GITSTORE_SECRET_FILE;
38 if (!file) return "";
39 if (!existsSync(file)) {
40 mkdirSync(dirname(file), { recursive: true });
41 writeFileSync(file, randomBytes(32).toString("hex"), { mode: 0o600 });
42 }
43 return readFileSync(file, "utf8").trim();
44}
45
46if (SECRET.length < 16) {
47 console.error("Set GITSTORE_SECRET (16 characters or more) or GITSTORE_SECRET_FILE.");
48 process.exit(1);
49}
50mkdirSync(ROOT, { recursive: true });
51
52const KEY = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,199}$/;
53const HASH = /^[0-9a-f]{40}$/;
54
55class StoreError extends Error {
56 constructor(code, message, status = 400) {
57 super(message);
58 this.code = code;
59 this.status = status;
60 }
61}
62
63function repoDir(key) {
64 if (!KEY.test(key) || key.includes("..")) {
65 throw new StoreError("INVALID_REPO_NAME", `invalid repository name: ${key}`);
66 }
67 return join(ROOT, `${key}.git`);
68}
69
70function exists(key) {
71 return existsSync(join(repoDir(key), "HEAD"));
72}
73
74function requireRepo(key) {
75 if (!exists(key)) throw new StoreError("NOT_FOUND", `no repository ${key}`, 404);
76 return repoDir(key);
77}
78
79/** Runs git and resolves with its stdout as a Buffer. */
80function git(args, { cwd, input, allowFail = false } = {}) {
81 return new Promise((resolve, reject) => {
82 const child = spawn("git", args, { cwd, stdio: ["pipe", "pipe", "pipe"] });
83 const out = [];
84 const err = [];
85 child.stdout.on("data", (chunk) => out.push(chunk));
86 child.stderr.on("data", (chunk) => err.push(chunk));
87 child.on("error", reject);
88 child.on("close", (code) => {
89 if (code !== 0 && !allowFail) {
90 reject(new StoreError("INTERNAL_ERROR", `git ${args[0]} failed: ${Buffer.concat(err)}`, 500));
91 } else {
92 resolve({ code, stdout: Buffer.concat(out) });
93 }
94 });
95 child.stdin.end(input ?? undefined);
96 });
97}
98
99// ── Metadata kept beside each repository ────────────────────────────────
100
101function metaPath(key) {
102 return join(repoDir(key), "g1t.json");
103}
104
105function readMeta(key) {
106 try {
107 return JSON.parse(readFileSync(metaPath(key), "utf8"));
108 } catch {
109 return {};
110 }
111}
112
113function writeMeta(key, meta) {
114 writeFileSync(metaPath(key), JSON.stringify(meta, null, 2));
115}
116
117async function info(key) {
118 const dir = requireRepo(key);
119 const meta = readMeta(key);
120 const head = (await git(["symbolic-ref", "--short", "HEAD"], { cwd: dir, allowFail: true })).stdout
121 .toString()
122 .trim();
123 let lastPushAt = null;
124 try {
125 lastPushAt = statSync(join(dir, "g1t-pushed")).mtime.toISOString();
126 } catch {}
127 return {
128 id: meta.id ?? key,
129 name: key,
130 description: meta.description ?? null,
131 defaultBranch: head || "main",
132 createdAt: meta.createdAt ?? new Date(0).toISOString(),
133 updatedAt: lastPushAt ?? meta.createdAt ?? new Date(0).toISOString(),
134 lastPushAt,
135 source: meta.source ?? null,
136 readOnly: Boolean(meta.readOnly),
137 remote: `${PUBLIC_URL}/git/${key}.git`,
138 };
139}
140
141async function create(key, { description, defaultBranch, readOnly, source } = {}) {
142 const dir = repoDir(key);
143 if (exists(key)) throw new StoreError("ALREADY_EXISTS", `${key} already exists`, 409);
144 mkdirSync(dir, { recursive: true });
145 await git(["init", "--bare", "--quiet", `--initial-branch=${defaultBranch || "main"}`, dir]);
146 await configure(dir);
147 writeMeta(key, {
148 id: randomUUID(),
149 description: description ?? null,
150 createdAt: new Date().toISOString(),
151 readOnly: Boolean(readOnly),
152 source: source ?? null,
153 });
154 return info(key);
155}
156
157async function configure(dir) {
158 // Pushes arrive through git http-backend; the token has already been
159 // checked, so receive-pack is allowed for every write-scoped request.
160 await git(["config", "http.receivepack", "true"], { cwd: dir });
161 await git(["config", "receive.denyNonFastForwards", "false"], { cwd: dir });
162 await git(["config", "uploadpack.allowAnySHA1InWant", "true"], { cwd: dir });
163}
164
165async function fork(key, target, { description, readOnly, defaultBranchOnly = true } = {}) {
166 const source = requireRepo(key);
167 const dir = repoDir(target);
168 if (exists(target)) throw new StoreError("ALREADY_EXISTS", `${target} already exists`, 409);
169 const args = ["clone", "--bare", "--quiet", "--no-tags"];
170 if (defaultBranchOnly) args.push("--single-branch");
171 // A local clone hard-links the objects: cheap, and independent of the
172 // source from then on.
173 args.push(source, dir);
174 await git(args);
175 await git(["remote", "remove", "origin"], { cwd: dir, allowFail: true });
176 await configure(dir);
177 writeMeta(target, {
178 id: randomUUID(),
179 description: description ?? readMeta(key).description ?? null,
180 createdAt: new Date().toISOString(),
181 readOnly: Boolean(readOnly),
182 source: `artifacts:${key}`,
183 });
184 return info(target);
185}
186
187// ── Reading objects ─────────────────────────────────────────────────────
188
189async function objectType(dir, spec) {
190 const { code, stdout } = await git(["cat-file", "-t", "--", spec], { cwd: dir, allowFail: true });
191 return code === 0 ? stdout.toString().trim() : null;
192}
193
194function person(line) {
195 // `Name <email> 1700000000 +0000`
196 const match = /^(.*) <([^>]*)> (\d+) [+-]\d{4}$/.exec(line);
197 return match ? { name: match[1], email: match[2], at: Number(match[3]) } : { name: line, email: "", at: 0 };
198}
199
200function parseCommit(hash, raw) {
201 const text = raw.toString("utf8");
202 const split = text.indexOf("\n\n");
203 const headers = (split === -1 ? text : text.slice(0, split)).split("\n");
204 let message = split === -1 ? "" : text.slice(split + 2);
205 if (message.endsWith("\n")) message = message.slice(0, -1);
206 const commit = { hash, treeHash: "", message, parents: [], author: null, committer: null };
207 for (const header of headers) {
208 const space = header.indexOf(" ");
209 const name = header.slice(0, space);
210 const value = header.slice(space + 1);
211 if (name === "tree") commit.treeHash = value;
212 else if (name === "parent") commit.parents.push(value);
213 else if (name === "author") commit.author = person(value);
214 else if (name === "committer") commit.committer = person(value);
215 }
216 const author = commit.author ?? { name: "", email: "", at: 0 };
217 const committer = commit.committer ?? author;
218 return {
219 hash,
220 treeHash: commit.treeHash,
221 message: commit.message,
222 author: { name: author.name, email: author.email },
223 committer: { name: committer.name, email: committer.email },
224 parents: commit.parents,
225 authoredAt: author.at,
226 committedAt: committer.at,
227 };
228}
229
230async function readCommit(key, hash) {
231 const dir = requireRepo(key);
232 if (!HASH.test(hash)) return null;
233 if ((await objectType(dir, hash)) !== "commit") return null;
234 return parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout);
235}
236
237async function log(key, { ref = "HEAD", limit = 50, offset = 0 } = {}) {
238 const dir = requireRepo(key);
239 if (typeof ref !== "string" || ref.startsWith("-")) return [];
240 const count = Math.max(1, Math.min(Number(limit) || 50, 1000));
241 const skip = Math.max(0, Number(offset) || 0);
242 const listed = await git(
243 ["rev-list", "--first-parent", `--max-count=${count}`, `--skip=${skip}`, ref, "--"],
244 { cwd: dir, allowFail: true },
245 );
246 if (listed.code !== 0) return [];
247 const hashes = listed.stdout.toString().split("\n").filter(Boolean);
248 const commits = [];
249 for (const hash of hashes) {
250 commits.push(parseCommit(hash, (await git(["cat-file", "commit", hash], { cwd: dir })).stdout));
251 }
252 return commits;
253}
254
255const TYPES = { "040000": "tree", "100644": "blob", "100755": "exec", "120000": "symlink", "160000": "gitlink" };
256
257async function readTree(key, hash) {
258 const dir = requireRepo(key);
259 if (!HASH.test(hash)) return null;
260 if ((await objectType(dir, hash)) !== "tree") return null;
261 const { stdout } = await git(["ls-tree", "-z", hash], { cwd: dir });
262 return stdout
263 .toString("utf8")
264 .split("\0")
265 .filter(Boolean)
266 .map((line) => {
267 const tab = line.indexOf("\t");
268 const [mode, , object] = line.slice(0, tab).split(" ");
269 return {
270 name: line.slice(tab + 1),
271 mode: mode === "040000" ? "40000" : mode,
272 hash: object,
273 type: TYPES[mode] ?? "blob",
274 };
275 });
276}
277
278async function readBlob(key, hash) {
279 const dir = requireRepo(key);
280 if (!HASH.test(hash)) return null;
281 if ((await objectType(dir, hash)) !== "blob") return null;
282 return (await git(["cat-file", "blob", hash], { cwd: dir })).stdout;
283}
284
285async function readFile(key, ref, path) {
286 const dir = requireRepo(key);
287 if (!ref || !path || ref.startsWith("-") || ref.includes(":")) return null;
288 const spec = `${ref}:${path.replace(/^\/+/, "")}`;
289 if ((await objectType(dir, spec)) !== "blob") return null;
290 return (await git(["cat-file", "blob", spec], { cwd: dir })).stdout;
291}
292
293// ── Tokens for git's smart HTTP ─────────────────────────────────────────
294
295function sign(payload) {
296 return createHmac("sha256", SECRET).update(payload).digest("base64url");
297}
298
299function mintToken(key, scope = "write", ttl = 86400) {
300 const seconds = Math.max(60, Math.min(Number(ttl) || 86400, 31536000));
301 const expires = Math.floor(Date.now() / 1000) + seconds;
302 const id = randomUUID();
303 const payload = Buffer.from(JSON.stringify({ k: key, s: scope, e: expires, i: id })).toString("base64url");
304 return {
305 id,
306 plaintext: `${payload}.${sign(payload)}`,
307 scope,
308 expiresAt: new Date(expires * 1000).toISOString(),
309 };
310}
311
312function checkToken(token, key) {
313 const [payload, signature] = String(token ?? "").split(".");
314 if (!payload || !signature) return null;
315 const expected = Buffer.from(sign(payload));
316 const given = Buffer.from(signature);
317 if (expected.length !== given.length || !timingSafeEqual(expected, given)) return null;
318 const claims = JSON.parse(Buffer.from(payload, "base64url").toString());
319 if (claims.k !== key || claims.e < Date.now() / 1000) return null;
320 return claims;
321}
322
323function bearer(request) {
324 const header = request.headers.authorization ?? "";
325 if (/^bearer /i.test(header)) return header.slice(7).trim();
326 if (/^basic /i.test(header)) {
327 // A git client given the token as a password: `x:<token>`.
328 const decoded = Buffer.from(header.slice(6).trim(), "base64").toString();
329 return decoded.slice(decoded.indexOf(":") + 1);
330 }
331 return null;
332}
333
334// ── Smart HTTP through git http-backend ─────────────────────────────────
335
336function smartHttp(request, response, key, rest, query) {
337 if (!exists(key)) return send(response, 404, "not found");
338 const claims = checkToken(bearer(request), key);
339 if (!claims) {
340 response.writeHead(401, { "www-authenticate": 'Basic realm="g1t-gitstore"' });
341 return response.end("unauthorized");
342 }
343 const service = rest === "info/refs" ? new URLSearchParams(query).get("service") : rest;
344 if (service === "git-receive-pack" && claims.s !== "write") return send(response, 403, "read-only token");
345 if (service !== "git-upload-pack" && service !== "git-receive-pack") return send(response, 404, "not found");
346
347 const env = {
348 PATH: process.env.PATH,
349 GIT_PROJECT_ROOT: ROOT,
350 GIT_HTTP_EXPORT_ALL: "1",
351 REQUEST_METHOD: request.method,
352 PATH_INFO: `/${key}.git/${rest}`,
353 QUERY_STRING: query,
354 CONTENT_TYPE: request.headers["content-type"] ?? "",
355 REMOTE_USER: "g1t",
356 REMOTE_ADDR: request.socket.remoteAddress ?? "",
357 };
358 if (request.headers["git-protocol"]) env.GIT_PROTOCOL = request.headers["git-protocol"];
359 if (request.headers["content-encoding"]) env.HTTP_CONTENT_ENCODING = request.headers["content-encoding"];
360 if (request.headers["content-length"]) env.CONTENT_LENGTH = request.headers["content-length"];
361
362 const child = spawn("git", ["http-backend"], { env, stdio: ["pipe", "pipe", "pipe"] });
363 request.pipe(child.stdin);
364 child.stderr.on("data", (chunk) => process.stderr.write(chunk));
365
366 // CGI: headers, a blank line, then the body.
367 let buffered = Buffer.alloc(0);
368 let headersDone = false;
369 child.stdout.on("data", (chunk) => {
370 if (headersDone) return response.write(chunk);
371 buffered = Buffer.concat([buffered, chunk]);
372 let end = buffered.indexOf("\r\n\r\n");
373 let gap = 4;
374 if (end === -1) {
375 end = buffered.indexOf("\n\n");
376 gap = 2;
377 }
378 if (end === -1) return;
379 headersDone = true;
380 let status = 200;
381 const headers = {};
382 for (const line of buffered.slice(0, end).toString().split(/\r?\n/)) {
383 const colon = line.indexOf(":");
384 if (colon === -1) continue;
385 const name = line.slice(0, colon).trim().toLowerCase();
386 const value = line.slice(colon + 1).trim();
387 if (name === "status") status = Number.parseInt(value, 10);
388 else headers[name] = value;
389 }
390 response.writeHead(status, headers);
391 response.write(buffered.slice(end + gap));
392 });
393 child.on("close", (code) => {
394 if (!headersDone) {
395 send(response, 500, "git http-backend failed");
396 return;
397 }
398 if (service === "git-receive-pack" && request.method === "POST" && code === 0) {
399 const marker = join(repoDir(key), "g1t-pushed");
400 try {
401 utimesSync(marker, new Date(), new Date());
402 } catch {
403 writeFileSync(marker, "");
404 }
405 }
406 response.end();
407 });
408}
409
410// ── HTTP ────────────────────────────────────────────────────────────────
411
412function send(response, status, body, headers = {}) {
413 const isBuffer = Buffer.isBuffer(body);
414 const payload = isBuffer ? body : typeof body === "string" ? body : JSON.stringify(body);
415 response.writeHead(status, {
416 "content-type": isBuffer ? "application/octet-stream" : typeof body === "string" ? "text/plain" : "application/json",
417 ...headers,
418 });
419 response.end(payload);
420}
421
422async function readJson(request) {
423 const chunks = [];
424 for await (const chunk of request) chunks.push(chunk);
425 const text = Buffer.concat(chunks).toString();
426 return text ? JSON.parse(text) : {};
427}
428
429function authorized(request) {
430 const given = Buffer.from(request.headers["x-gitstore-secret"] ?? "");
431 const expected = Buffer.from(SECRET);
432 return given.length === expected.length && timingSafeEqual(given, expected);
433}
434
435async function api(request, response, parts, params) {
436 if (!authorized(request)) return send(response, 401, { code: "UNAUTHORIZED", message: "bad secret" });
437 const method = request.method;
438 // POST /api/repos create
439 if (parts.length === 0 && method === "POST") {
440 const body = await readJson(request);
441 return send(response, 200, await create(body.name, body));
442 }
443 const [key, action, arg] = parts;
444 if (method === "GET" && !action) return send(response, 200, await info(key));
445 if (method === "POST" && action === "tokens") {
446 requireRepo(key);
447 const body = await readJson(request);
448 return send(response, 200, mintToken(key, body.scope, body.ttl));
449 }
450 if (method === "POST" && action === "fork") {
451 const body = await readJson(request);
452 return send(response, 200, await fork(key, body.name, body));
453 }
454 if (method === "GET" && action === "commits") {
455 return send(response, 200, await readCommit(key, arg));
456 }
457 if (method === "GET" && action === "log") {
458 return send(response, 200, await log(key, Object.fromEntries(params)));
459 }
460 if (method === "GET" && action === "trees") {
461 return send(response, 200, await readTree(key, arg));
462 }
463 if (method === "GET" && (action === "blobs" || action === "file")) {
464 const bytes =
465 action === "blobs" ? await readBlob(key, arg) : await readFile(key, params.get("ref"), params.get("path"));
466 return bytes ? send(response, 200, bytes) : send(response, 404, { code: "NOT_FOUND", message: "no such object" });
467 }
468 return send(response, 404, { code: "NOT_FOUND", message: "no such route" });
469}
470
471const server = createServer(async (request, response) => {
472 const url = new URL(request.url, "http://gitstore");
473 try {
474 if (url.pathname === "/healthz") return send(response, 200, "ok");
475 const git = /^\/git\/([^/]+)\.git\/(info\/refs|git-upload-pack|git-receive-pack)$/.exec(url.pathname);
476 if (git) return smartHttp(request, response, decodeURIComponent(git[1]), git[2], url.search.slice(1));
477 if (url.pathname === "/api/repos" || url.pathname.startsWith("/api/repos/")) {
478 const parts = url.pathname.slice("/api/repos".length).split("/").filter(Boolean).map(decodeURIComponent);
479 return await api(request, response, parts, url.searchParams);
480 }
481 send(response, 404, "not found");
482 } catch (error) {
483 const status = error instanceof StoreError ? error.status : 500;
484 const code = error instanceof StoreError ? error.code : "INTERNAL_ERROR";
485 if (status >= 500) console.error(error);
486 if (!response.headersSent) send(response, status, { code, message: error.message });
487 else response.end();
488 }
489});
490
491server.listen(PORT, () => {
492 console.log(`g1t gitstore: ${ROOT} on :${PORT} (remote ${PUBLIC_URL})`);
493});
494
495for (const signal of ["SIGINT", "SIGTERM"]) {
496 process.on(signal, () => server.close(() => process.exit(0)));
497}