Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
310 linesCodeBlameRaw
1// What the Ask bench shares: where the repository is, the credentials it
2// reads (never printed, logged or written: every string that leaves this
3// process goes through `redact`), g1t's REST API, its MCP server, the site
4// as the signed-in user, and one Ask answer read as it streams.
5
6import { execFileSync } from "node:child_process";
7import { existsSync, readFileSync } from "node:fs";
8import { homedir } from "node:os";
9import { dirname, isAbsolute, join, resolve } from "node:path";
10import { fileURLToPath } from "node:url";
11
12export const HERE = dirname(fileURLToPath(import.meta.url));
13
14/** The checkout the bench runs from: a worktree's own folder. */
15export const ROOT = resolve(HERE, "..", "..");
16
17/**
18 * The main checkout, where `.credentials/` and `.secrets/` live: a worktree
19 * has neither, so they are read from the checkout its git directory belongs to.
20 */
21export function mainCheckout() {
22 try {
23 const common = execFileSync("git", ["rev-parse", "--git-common-dir"], { cwd: ROOT, encoding: "utf8" }).trim();
24 const dir = isAbsolute(common) ? common : resolve(ROOT, common);
25 return dirname(dir);
26 } catch {
27 return ROOT;
28 }
29}
30
31export const SITE = (process.env.G1T_SITE ?? "https://g1t.sh").replace(/\/$/, "");
32export const API = (process.env.G1T_API ?? "https://api.g1t.sh").replace(/\/$/, "");
33export const MCP = (process.env.G1T_MCP ?? "https://mcp.g1t.sh").replace(/\/$/, "");
34const UA = "g1t-ask-bench/1";
35
36// --- credentials ----------------------------------------------------------
37
38const SECRETS = new Set();
39
40function remember(value) {
41 if (value && value.length >= 8) SECRETS.add(value);
42 return value;
43}
44
45function readSecretFile(path) {
46 if (!existsSync(path)) return null;
47 // Either the bare value or a `token: value` line.
48 const text = readFileSync(path, "utf8").trim();
49 const value = text
50 .replace(/^\s*token:\s*/i, "")
51 .split(/\r?\n/)[0]
52 .trim();
53 return remember(value) || null;
54}
55
56let apiToken;
57/** The API token: G1T_TOKEN, or the export in ~/.bashrc. */
58export function apiTokenValue() {
59 if (apiToken !== undefined) return apiToken;
60 let value = process.env.G1T_TOKEN;
61 if (!value) {
62 const rc = join(homedir(), ".bashrc");
63 if (existsSync(rc)) value = readFileSync(rc, "utf8").match(/G1T_TOKEN=["']?([^"'\s]+)/)?.[1];
64 }
65 apiToken = remember(value) || null;
66 return apiToken;
67}
68
69let webToken;
70/** The site's token, for g1t.sh routes as the user: .credentials/g1t_token.txt (or G1T_WEB_TOKEN_FILE). */
71export function webTokenValue() {
72 if (webToken !== undefined) return webToken;
73 const file = process.env.G1T_WEB_TOKEN_FILE ?? join(mainCheckout(), ".credentials", "g1t_token.txt");
74 webToken = readSecretFile(file) ?? apiTokenValue();
75 return webToken;
76}
77
78let anthropicKey;
79/** The judge's key: .secrets/anthropic-key.txt (or ANTHROPIC_KEY_FILE). */
80export function anthropicKeyValue() {
81 if (anthropicKey !== undefined) return anthropicKey;
82 const file = process.env.ANTHROPIC_KEY_FILE ?? join(mainCheckout(), ".secrets", "anthropic-key.txt");
83 anthropicKey = readSecretFile(file);
84 return anthropicKey;
85}
86
87/** Loads every credential, so `redact` knows them all before anything is written. */
88export function loadCredentials() {
89 return { api: Boolean(apiTokenValue()), web: Boolean(webTokenValue()), anthropic: Boolean(anthropicKeyValue()) };
90}
91
92/** Every credential this process read, for the no-secrets check. */
93export function knownSecrets() {
94 return [...SECRETS];
95}
96
97/** Takes every credential this process read, and anything shaped like one, out of text. */
98export function redact(text) {
99 let out = String(text ?? "");
100 for (const secret of SECRETS) out = out.split(secret).join("[REDACTED]");
101 return out.replace(/\bsk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-[REDACTED]").replace(/\bg1t_[A-Za-z0-9_-]{16,}/g, "g1t_[REDACTED]");
102}
103
104// --- g1t ------------------------------------------------------------------
105
106async function request(url, token, init = {}) {
107 const headers = { "user-agent": UA, accept: "application/json", ...(init.headers ?? {}) };
108 if (token) headers.authorization = `Bearer ${token}`;
109 const response = await fetch(url, { ...init, headers, redirect: init.redirect ?? "manual" });
110 return response;
111}
112
113/** GET (or more) on api.g1t.sh as the user: `{ status, data }`, data null when not JSON. */
114export async function api(path, init = {}) {
115 const response = await request(`${API}${path}`, apiTokenValue(), init);
116 const text = await response.text();
117 let data = null;
118 try {
119 data = JSON.parse(text);
120 } catch {}
121 return { status: response.status, data, text };
122}
123
124/** A g1t.sh route as the signed-in user (the site's token). */
125export async function site(path, init = {}) {
126 const url = path.startsWith("http") ? path : `${SITE}${path}`;
127 return request(url, webTokenValue(), init);
128}
129
130let mcpId = 0;
131/** One MCP tool call on mcp.g1t.sh as the user; the tool's text parsed as JSON when it is. */
132export async function mcp(tool, args) {
133 const response = await request(MCP, apiTokenValue(), {
134 method: "POST",
135 headers: { "content-type": "application/json", accept: "application/json, text/event-stream" },
136 body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpId, method: "tools/call", params: { name: tool, arguments: args } }),
137 });
138 const text = await response.text();
139 // A streamable HTTP server may answer as one SSE event.
140 const json = text.trimStart().startsWith("{") ? text : (text.match(/^data: (.*)$/m)?.[1] ?? "null");
141 const message = JSON.parse(json);
142 if (message?.error) throw new Error(`mcp ${tool}: ${message.error.message}`);
143 const body = message?.result?.content?.map((c) => c.text ?? "").join("") ?? "";
144 if (message?.result?.isError) throw new Error(`mcp ${tool}: ${body.slice(0, 300)}`);
145 try {
146 return JSON.parse(body);
147 } catch {
148 return body;
149 }
150}
151
152/** Every page of a list endpoint that takes per_page and page, up to `max` items. */
153export async function apiAll(path, max = 300) {
154 const items = [];
155 for (let page = 1; items.length < max; page++) {
156 const sep = path.includes("?") ? "&" : "?";
157 const { status, data } = await api(`${path}${sep}per_page=100&page=${page}`);
158 if (status !== 200) break;
159 const list = Array.isArray(data) ? data : (data?.items ?? data?.workflow_runs ?? data?.deployments ?? []);
160 items.push(...list);
161 if (list.length < 100) break;
162 }
163 return items.slice(0, max);
164}
165
166// --- Ask ------------------------------------------------------------------
167
168/** Event types the bench knows. Anything else is recorded as unknown, never dropped. */
169const KNOWN = new Set(["meta", "text", "done", "error", "tool", "sources", "proposal"]);
170
171/** Looks like a tool step: `tool`, `tool_call`, `tool_result`, `step`, … */
172export const isToolEvent = (event) => /tool|step|action|call/i.test(String(event?.type ?? ""));
173/** Looks like a source or citation. */
174export const isSourceEvent = (event) => /source|citation|reference|link/i.test(String(event?.type ?? ""));
175
176/** URLs an event names, wherever it keeps them. */
177function urlsIn(value, found = new Set()) {
178 if (typeof value === "string") {
179 for (const match of value.matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) found.add(match[0]);
180 if (/^\/[a-z0-9]/i.test(value) && !value.includes(" ")) found.add(`${SITE}${value}`);
181 } else if (Array.isArray(value)) value.forEach((v) => urlsIn(v, found));
182 else if (value && typeof value === "object") Object.values(value).forEach((v) => urlsIn(v, found));
183 return found;
184}
185
186/** What every question the bench posts starts with. */
187export const TEST_PREFIX = "[TEST] ";
188
189/** A chat API call on the site as the user: the `{ ok, value | error }` it answers. */
190async function chatApi(workspace, body) {
191 const response = await site(`/${workspace}/-/chat/api`, {
192 method: "POST",
193 headers: { "content-type": "application/json", origin: SITE },
194 body: JSON.stringify(body),
195 });
196 const text = await response.text();
197 try {
198 return JSON.parse(text);
199 } catch {
200 return { ok: false, error: { message: `chat api ${response.status}: ${text.slice(0, 200)}` } };
201 }
202}
203
204async function chatMessages(workspace, channel) {
205 const response = await site(`/${workspace}/-/chat/api?channel=${encodeURIComponent(channel)}&limit=60`);
206 const data = await response.json().catch(() => null);
207 return data?.value?.messages ?? [];
208}
209
210/**
211 * Asks once, as the user, the way Ask and Chat both do: in the user's
212 * direct message with @g1t, after a New conversation mark, posting each of
213 * the conversation's questions in turn and following g1t's answer to each
214 * as it streams (polled). The last answer is the one graded: its text, the
215 * time to its first words and to its end, and the steps and sources it kept,
216 * as events in the order the stream gave them.
217 */
218export async function ask(workspace, { messages, page }, { timeoutMs = 180_000 } = {}) {
219 const started = performance.now();
220 const at = () => Math.round(performance.now() - started);
221 const result = {
222 status: null,
223 events: [],
224 unknown_types: [],
225 text: "",
226 first_event_ms: null,
227 first_text_ms: null,
228 total_ms: null,
229 meta: null,
230 done: null,
231 error: null,
232 tool_events: [],
233 source_events: [],
234 source_urls: [],
235 };
236 try {
237 const dm = await chatApi(workspace, { intent: "g1t_dm" });
238 if (!dm.ok) throw new Error(dm.error?.message ?? "could not open the direct message with g1t");
239 const channel = dm.value.channel.id;
240 await chatApi(workspace, { intent: "g1t_reset", channel_id: channel });
241 const questions = messages.filter((m) => m.role === "user");
242 let answer = null;
243 for (const [index, question] of questions.entries()) {
244 const last = index === questions.length - 1;
245 const asked = performance.now();
246 // Marked, so the person whose direct message it is can tell the bench's questions from their own.
247 const posted = await chatApi(workspace, {
248 intent: "post",
249 channel_id: channel,
250 body: `${TEST_PREFIX}${question.content}`,
251 page: last ? (page ?? null) : null,
252 });
253 result.status = posted.ok ? 200 : 400;
254 if (!posted.ok) throw new Error(posted.error?.message ?? "could not post");
255 const mine = posted.value.id;
256 answer = null;
257 while (performance.now() - asked < timeoutMs) {
258 await sleep(700);
259 const list = await chatMessages(workspace, channel);
260 answer = list.find((m) => m.id > mine && m.author?.kind === "system" && m.answer) ?? null;
261 if (last && answer?.body && result.first_text_ms === null) result.first_text_ms = at();
262 if (last && answer && result.first_event_ms === null) result.first_event_ms = at();
263 if (answer && ["done", "error", "stopped"].includes(answer.answer.state)) break;
264 }
265 if (!answer || !["done", "error", "stopped"].includes(answer.answer?.state)) throw new Error(`timed out after ${timeoutMs} ms`);
266 }
267 const kept = answer.answer;
268 result.meta = { type: "meta", provider_label: kept.provider_label, model: kept.model };
269 result.events.push({ ms: 0, event: result.meta });
270 for (const step of kept.steps ?? []) {
271 const type = String(step?.type ?? "");
272 result.events.push({ ms: null, event: step });
273 if (!KNOWN.has(type) && !result.unknown_types.includes(type)) result.unknown_types.push(type);
274 if (isToolEvent(step)) result.tool_events.push({ ms: null, event: step });
275 if (isSourceEvent(step)) result.source_events.push({ ms: null, event: step });
276 }
277 result.text = answer.body ?? "";
278 result.events.push({ ms: at(), event: { type: "text", text: result.text } });
279 if (kept.state === "done") result.done = { type: "done", stop_reason: kept.stop_reason ?? null };
280 else result.error = kept.error ?? kept.state;
281 } catch (error) {
282 result.error = String(error?.message ?? error);
283 } finally {
284 result.total_ms = at();
285 }
286 const urls = new Set();
287 for (const { event } of result.source_events) urlsIn(event, urls);
288 result.source_urls = [...urls];
289 return result;
290}
291
292export const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
293
294/** `--flag value` and `--flag` into an object; the first bare word is the command. */
295export function parseArgs(argv) {
296 const flags = {};
297 const words = [];
298 for (let i = 0; i < argv.length; i++) {
299 const arg = argv[i];
300 if (!arg.startsWith("--")) {
301 words.push(arg);
302 continue;
303 }
304 const [name, inline] = arg.slice(2).split("=", 2);
305 if (inline !== undefined) flags[name] = inline;
306 else if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) flags[name] = true;
307 else flags[name] = argv[++i];
308 }
309 return { command: words[0], words, flags };
310}