Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
281 linesCodeBlameRaw
1// What the Ask bench shares: where the repository is, the credentials it
2// reads (never printed, logged or written: every string that leaves this
3// process goes through `redact`), g1t's REST API, its MCP server, the site
4// as the signed-in user, and one Ask answer read as it streams.
5
6import { execFileSync } from "node:child_process";
7import { existsSync, readFileSync } from "node:fs";
8import { homedir } from "node:os";
9import { dirname, isAbsolute, join, resolve } from "node:path";
10import { fileURLToPath } from "node:url";
11
12export const HERE = dirname(fileURLToPath(import.meta.url));
13
14/** The checkout the bench runs from: a worktree's own folder. */
15export const ROOT = resolve(HERE, "..", "..");
16
17/**
18 * The main checkout, where `.credentials/` and `.secrets/` live: a worktree
19 * has neither, so they are read from the checkout its git directory belongs to.
20 */
21export function mainCheckout() {
22 try {
23 const common = execFileSync("git", ["rev-parse", "--git-common-dir"], { cwd: ROOT, encoding: "utf8" }).trim();
24 const dir = isAbsolute(common) ? common : resolve(ROOT, common);
25 return dirname(dir);
26 } catch {
27 return ROOT;
28 }
29}
30
31export const SITE = (process.env.G1T_SITE ?? "https://g1t.sh").replace(/\/$/, "");
32export const API = (process.env.G1T_API ?? "https://api.g1t.sh").replace(/\/$/, "");
33export const MCP = (process.env.G1T_MCP ?? "https://mcp.g1t.sh").replace(/\/$/, "");
34const UA = "g1t-ask-bench/1";
35
36// --- credentials ----------------------------------------------------------
37
38const SECRETS = new Set();
39
40function remember(value) {
41 if (value && value.length >= 8) SECRETS.add(value);
42 return value;
43}
44
45function readSecretFile(path) {
46 if (!existsSync(path)) return null;
47 // Either the bare value or a `token: value` line.
48 const text = readFileSync(path, "utf8").trim();
49 const value = text.replace(/^\s*token:\s*/i, "").split(/\r?\n/)[0].trim();
50 return remember(value) || null;
51}
52
53let apiToken;
54/** The API token: G1T_TOKEN, or the export in ~/.bashrc. */
55export function apiTokenValue() {
56 if (apiToken !== undefined) return apiToken;
57 let value = process.env.G1T_TOKEN;
58 if (!value) {
59 const rc = join(homedir(), ".bashrc");
60 if (existsSync(rc)) value = readFileSync(rc, "utf8").match(/G1T_TOKEN=["']?([^"'\s]+)/)?.[1];
61 }
62 apiToken = remember(value) || null;
63 return apiToken;
64}
65
66let webToken;
67/** The site's token, for g1t.sh routes as the user: .credentials/g1t_token.txt (or G1T_WEB_TOKEN_FILE). */
68export function webTokenValue() {
69 if (webToken !== undefined) return webToken;
70 const file = process.env.G1T_WEB_TOKEN_FILE ?? join(mainCheckout(), ".credentials", "g1t_token.txt");
71 webToken = readSecretFile(file) ?? apiTokenValue();
72 return webToken;
73}
74
75let anthropicKey;
76/** The judge's key: .secrets/anthropic-key.txt (or ANTHROPIC_KEY_FILE). */
77export function anthropicKeyValue() {
78 if (anthropicKey !== undefined) return anthropicKey;
79 const file = process.env.ANTHROPIC_KEY_FILE ?? join(mainCheckout(), ".secrets", "anthropic-key.txt");
80 anthropicKey = readSecretFile(file);
81 return anthropicKey;
82}
83
84/** Loads every credential, so `redact` knows them all before anything is written. */
85export function loadCredentials() {
86 return { api: Boolean(apiTokenValue()), web: Boolean(webTokenValue()), anthropic: Boolean(anthropicKeyValue()) };
87}
88
89/** Every credential this process read, for the no-secrets check. */
90export function knownSecrets() {
91 return [...SECRETS];
92}
93
94/** Takes every credential this process read, and anything shaped like one, out of text. */
95export function redact(text) {
96 let out = String(text ?? "");
97 for (const secret of SECRETS) out = out.split(secret).join("[REDACTED]");
98 return out
99 .replace(/\bsk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-[REDACTED]")
100 .replace(/\bg1t_[A-Za-z0-9_-]{16,}/g, "g1t_[REDACTED]");
101}
102
103// --- g1t ------------------------------------------------------------------
104
105async function request(url, token, init = {}) {
106 const headers = { "user-agent": UA, accept: "application/json", ...(init.headers ?? {}) };
107 if (token) headers.authorization = `Bearer ${token}`;
108 const response = await fetch(url, { ...init, headers, redirect: init.redirect ?? "manual" });
109 return response;
110}
111
112/** GET (or more) on api.g1t.sh as the user: `{ status, data }`, data null when not JSON. */
113export async function api(path, init = {}) {
114 const response = await request(`${API}${path}`, apiTokenValue(), init);
115 const text = await response.text();
116 let data = null;
117 try {
118 data = JSON.parse(text);
119 } catch {}
120 return { status: response.status, data, text };
121}
122
123/** A g1t.sh route as the signed-in user (the site's token). */
124export async function site(path, init = {}) {
125 const url = path.startsWith("http") ? path : `${SITE}${path}`;
126 return request(url, webTokenValue(), init);
127}
128
129let mcpId = 0;
130/** One MCP tool call on mcp.g1t.sh as the user; the tool's text parsed as JSON when it is. */
131export async function mcp(tool, args) {
132 const response = await request(MCP, apiTokenValue(), {
133 method: "POST",
134 headers: { "content-type": "application/json", accept: "application/json, text/event-stream" },
135 body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpId, method: "tools/call", params: { name: tool, arguments: args } }),
136 });
137 const text = await response.text();
138 // A streamable HTTP server may answer as one SSE event.
139 const json = text.trimStart().startsWith("{") ? text : (text.match(/^data: (.*)$/m)?.[1] ?? "null");
140 const message = JSON.parse(json);
141 if (message?.error) throw new Error(`mcp ${tool}: ${message.error.message}`);
142 const body = message?.result?.content?.map((c) => c.text ?? "").join("") ?? "";
143 if (message?.result?.isError) throw new Error(`mcp ${tool}: ${body.slice(0, 300)}`);
144 try {
145 return JSON.parse(body);
146 } catch {
147 return body;
148 }
149}
150
151/** Every page of a list endpoint that takes per_page and page, up to `max` items. */
152export async function apiAll(path, max = 300) {
153 const items = [];
154 for (let page = 1; items.length < max; page++) {
155 const sep = path.includes("?") ? "&" : "?";
156 const { status, data } = await api(`${path}${sep}per_page=100&page=${page}`);
157 if (status !== 200) break;
158 const list = Array.isArray(data) ? data : (data?.items ?? data?.workflow_runs ?? data?.deployments ?? []);
159 items.push(...list);
160 if (list.length < 100) break;
161 }
162 return items.slice(0, max);
163}
164
165// --- Ask ------------------------------------------------------------------
166
167/** Event types the bench knows. Anything else is recorded as unknown, never dropped. */
168const KNOWN = new Set(["meta", "text", "done", "error"]);
169
170/** Looks like a tool step: `tool`, `tool_call`, `tool_result`, `step`, … */
171export const isToolEvent = (event) => /tool|step|action|call/i.test(String(event?.type ?? ""));
172/** Looks like a source or citation. */
173export const isSourceEvent = (event) => /source|citation|reference|link/i.test(String(event?.type ?? ""));
174
175/** URLs an event names, wherever it keeps them. */
176function urlsIn(value, found = new Set()) {
177 if (typeof value === "string") {
178 for (const match of value.matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) found.add(match[0]);
179 if (/^\/[a-z0-9]/i.test(value) && !value.includes(" ")) found.add(`${SITE}${value}`);
180 } else if (Array.isArray(value)) value.forEach((v) => urlsIn(v, found));
181 else if (value && typeof value === "object") Object.values(value).forEach((v) => urlsIn(v, found));
182 return found;
183}
184
185/**
186 * Asks once, as the user, and reads the whole answer: every event with the
187 * time it came, the time to the first text, the total, the tool steps and
188 * sources it reported, and the text.
189 */
190export async function ask(workspace, { messages, page }, { timeoutMs = 180_000 } = {}) {
191 const started = performance.now();
192 const at = () => Math.round(performance.now() - started);
193 const result = {
194 status: null,
195 events: [],
196 unknown_types: [],
197 text: "",
198 first_event_ms: null,
199 first_text_ms: null,
200 total_ms: null,
201 meta: null,
202 done: null,
203 error: null,
204 tool_events: [],
205 source_events: [],
206 source_urls: [],
207 };
208 const controller = new AbortController();
209 const timer = setTimeout(() => controller.abort(), timeoutMs);
210 try {
211 const response = await site(`/${workspace}/-/ask`, {
212 method: "POST",
213 headers: { "content-type": "application/json", accept: "application/x-ndjson" },
214 body: JSON.stringify({ messages, page: page ?? {} }),
215 signal: controller.signal,
216 });
217 result.status = response.status;
218 if (!response.body) throw new Error(`no body (${response.status})`);
219 const decoder = new TextDecoder();
220 let pending = "";
221 const take = (line) => {
222 if (!line.trim()) return;
223 let event;
224 try {
225 event = JSON.parse(line);
226 } catch {
227 event = { type: "_unparsed", line: line.slice(0, 500) };
228 }
229 const ms = at();
230 result.first_event_ms ??= ms;
231 result.events.push({ ms, event });
232 const type = String(event?.type ?? "");
233 if (!KNOWN.has(type) && !result.unknown_types.includes(type)) result.unknown_types.push(type);
234 if (type === "meta") result.meta = event;
235 if (type === "text" && typeof event.text === "string") {
236 result.first_text_ms ??= ms;
237 result.text += event.text;
238 }
239 if (type === "done") result.done = event;
240 if (type === "error") result.error = event.message ?? "error";
241 if (!KNOWN.has(type) && isToolEvent(event)) result.tool_events.push({ ms, event });
242 if (!KNOWN.has(type) && isSourceEvent(event)) result.source_events.push({ ms, event });
243 };
244 for await (const chunk of response.body) {
245 pending += decoder.decode(chunk, { stream: true });
246 const lines = pending.split("\n");
247 pending = lines.pop() ?? "";
248 lines.forEach(take);
249 }
250 take(pending);
251 } catch (error) {
252 result.error = controller.signal.aborted ? `timed out after ${timeoutMs} ms` : String(error?.message ?? error);
253 } finally {
254 clearTimeout(timer);
255 result.total_ms = at();
256 }
257 const urls = new Set();
258 for (const { event } of result.source_events) urlsIn(event, urls);
259 result.source_urls = [...urls];
260 return result;
261}
262
263export const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
264
265/** `--flag value` and `--flag` into an object; the first bare word is the command. */
266export function parseArgs(argv) {
267 const flags = {};
268 const words = [];
269 for (let i = 0; i < argv.length; i++) {
270 const arg = argv[i];
271 if (!arg.startsWith("--")) {
272 words.push(arg);
273 continue;
274 }
275 const [name, inline] = arg.slice(2).split("=", 2);
276 if (inline !== undefined) flags[name] = inline;
277 else if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) flags[name] = true;
278 else flags[name] = argv[++i];
279 }
280 return { command: words[0], words, flags };
281}