Skip to content
Sign up

flagon-io/g1t

public
0

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
300 linesCodeBlameRaw
1// What the Ask bench shares: where the repository is, the credentials it
2// reads (never printed, logged or written: every string that leaves this
3// process goes through `redact`), g1t's REST API, its MCP server, the site
4// as the signed-in user, and one Ask answer read as it streams.
5
6import { execFileSync } from "node:child_process";
7import { existsSync, readFileSync } from "node:fs";
8import { homedir } from "node:os";
9import { dirname, isAbsolute, join, resolve } from "node:path";
10import { fileURLToPath } from "node:url";
11
12export const HERE = dirname(fileURLToPath(import.meta.url));
13
14/** The checkout the bench runs from: a worktree's own folder. */
15export const ROOT = resolve(HERE, "..", "..");
16
17/**
18 * The main checkout, where `.credentials/` and `.secrets/` live: a worktree
19 * has neither, so they are read from the checkout its git directory belongs to.
20 */
21export function mainCheckout() {
22 try {
23 const common = execFileSync("git", ["rev-parse", "--git-common-dir"], { cwd: ROOT, encoding: "utf8" }).trim();
24 const dir = isAbsolute(common) ? common : resolve(ROOT, common);
25 return dirname(dir);
26 } catch {
27 return ROOT;
28 }
29}
30
31export const SITE = (process.env.G1T_SITE ?? "https://g1t.sh").replace(/\/$/, "");
32export const API = (process.env.G1T_API ?? "https://api.g1t.sh").replace(/\/$/, "");
33export const MCP = (process.env.G1T_MCP ?? "https://mcp.g1t.sh").replace(/\/$/, "");
34const UA = "g1t-ask-bench/1";
35
36// --- credentials ----------------------------------------------------------
37
38const SECRETS = new Set();
39
40function remember(value) {
41 if (value && value.length >= 8) SECRETS.add(value);
42 return value;
43}
44
45function readSecretFile(path) {
46 if (!existsSync(path)) return null;
47 // Either the bare value or a `token: value` line.
48 const text = readFileSync(path, "utf8").trim();
49 const value = text.replace(/^\s*token:\s*/i, "").split(/\r?\n/)[0].trim();
50 return remember(value) || null;
51}
52
53let apiToken;
54/** The API token: G1T_TOKEN, or the export in ~/.bashrc. */
55export function apiTokenValue() {
56 if (apiToken !== undefined) return apiToken;
57 let value = process.env.G1T_TOKEN;
58 if (!value) {
59 const rc = join(homedir(), ".bashrc");
60 if (existsSync(rc)) value = readFileSync(rc, "utf8").match(/G1T_TOKEN=["']?([^"'\s]+)/)?.[1];
61 }
62 apiToken = remember(value) || null;
63 return apiToken;
64}
65
66let webToken;
67/** The site's token, for g1t.sh routes as the user: .credentials/g1t_token.txt (or G1T_WEB_TOKEN_FILE). */
68export function webTokenValue() {
69 if (webToken !== undefined) return webToken;
70 const file = process.env.G1T_WEB_TOKEN_FILE ?? join(mainCheckout(), ".credentials", "g1t_token.txt");
71 webToken = readSecretFile(file) ?? apiTokenValue();
72 return webToken;
73}
74
75let anthropicKey;
76/** The judge's key: .secrets/anthropic-key.txt (or ANTHROPIC_KEY_FILE). */
77export function anthropicKeyValue() {
78 if (anthropicKey !== undefined) return anthropicKey;
79 const file = process.env.ANTHROPIC_KEY_FILE ?? join(mainCheckout(), ".secrets", "anthropic-key.txt");
80 anthropicKey = readSecretFile(file);
81 return anthropicKey;
82}
83
84/** Loads every credential, so `redact` knows them all before anything is written. */
85export function loadCredentials() {
86 return { api: Boolean(apiTokenValue()), web: Boolean(webTokenValue()), anthropic: Boolean(anthropicKeyValue()) };
87}
88
89/** Every credential this process read, for the no-secrets check. */
90export function knownSecrets() {
91 return [...SECRETS];
92}
93
94/** Takes every credential this process read, and anything shaped like one, out of text. */
95export function redact(text) {
96 let out = String(text ?? "");
97 for (const secret of SECRETS) out = out.split(secret).join("[REDACTED]");
98 return out
99 .replace(/\bsk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-[REDACTED]")
100 .replace(/\bg1t_[A-Za-z0-9_-]{16,}/g, "g1t_[REDACTED]");
101}
102
103// --- g1t ------------------------------------------------------------------
104
105async function request(url, token, init = {}) {
106 const headers = { "user-agent": UA, accept: "application/json", ...(init.headers ?? {}) };
107 if (token) headers.authorization = `Bearer ${token}`;
108 const response = await fetch(url, { ...init, headers, redirect: init.redirect ?? "manual" });
109 return response;
110}
111
112/** GET (or more) on api.g1t.sh as the user: `{ status, data }`, data null when not JSON. */
113export async function api(path, init = {}) {
114 const response = await request(`${API}${path}`, apiTokenValue(), init);
115 const text = await response.text();
116 let data = null;
117 try {
118 data = JSON.parse(text);
119 } catch {}
120 return { status: response.status, data, text };
121}
122
123/** A g1t.sh route as the signed-in user (the site's token). */
124export async function site(path, init = {}) {
125 const url = path.startsWith("http") ? path : `${SITE}${path}`;
126 return request(url, webTokenValue(), init);
127}
128
129let mcpId = 0;
130/** One MCP tool call on mcp.g1t.sh as the user; the tool's text parsed as JSON when it is. */
131export async function mcp(tool, args) {
132 const response = await request(MCP, apiTokenValue(), {
133 method: "POST",
134 headers: { "content-type": "application/json", accept: "application/json, text/event-stream" },
135 body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpId, method: "tools/call", params: { name: tool, arguments: args } }),
136 });
137 const text = await response.text();
138 // A streamable HTTP server may answer as one SSE event.
139 const json = text.trimStart().startsWith("{") ? text : (text.match(/^data: (.*)$/m)?.[1] ?? "null");
140 const message = JSON.parse(json);
141 if (message?.error) throw new Error(`mcp ${tool}: ${message.error.message}`);
142 const body = message?.result?.content?.map((c) => c.text ?? "").join("") ?? "";
143 if (message?.result?.isError) throw new Error(`mcp ${tool}: ${body.slice(0, 300)}`);
144 try {
145 return JSON.parse(body);
146 } catch {
147 return body;
148 }
149}
150
151/** Every page of a list endpoint that takes per_page and page, up to `max` items. */
152export async function apiAll(path, max = 300) {
153 const items = [];
154 for (let page = 1; items.length < max; page++) {
155 const sep = path.includes("?") ? "&" : "?";
156 const { status, data } = await api(`${path}${sep}per_page=100&page=${page}`);
157 if (status !== 200) break;
158 const list = Array.isArray(data) ? data : (data?.items ?? data?.workflow_runs ?? data?.deployments ?? []);
159 items.push(...list);
160 if (list.length < 100) break;
161 }
162 return items.slice(0, max);
163}
164
165// --- Ask ------------------------------------------------------------------
166
167/** Event types the bench knows. Anything else is recorded as unknown, never dropped. */
168const KNOWN = new Set(["meta", "text", "done", "error", "tool", "sources", "proposal"]);
169
170/** Looks like a tool step: `tool`, `tool_call`, `tool_result`, `step`, … */
171export const isToolEvent = (event) => /tool|step|action|call/i.test(String(event?.type ?? ""));
172/** Looks like a source or citation. */
173export const isSourceEvent = (event) => /source|citation|reference|link/i.test(String(event?.type ?? ""));
174
175/** URLs an event names, wherever it keeps them. */
176function urlsIn(value, found = new Set()) {
177 if (typeof value === "string") {
178 for (const match of value.matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) found.add(match[0]);
179 if (/^\/[a-z0-9]/i.test(value) && !value.includes(" ")) found.add(`${SITE}${value}`);
180 } else if (Array.isArray(value)) value.forEach((v) => urlsIn(v, found));
181 else if (value && typeof value === "object") Object.values(value).forEach((v) => urlsIn(v, found));
182 return found;
183}
184
185/** A chat API call on the site as the user: the `{ ok, value | error }` it answers. */
186async function chatApi(workspace, body) {
187 const response = await site(`/${workspace}/-/chat/api`, {
188 method: "POST",
189 headers: { "content-type": "application/json", origin: SITE },
190 body: JSON.stringify(body),
191 });
192 const text = await response.text();
193 try {
194 return JSON.parse(text);
195 } catch {
196 return { ok: false, error: { message: `chat api ${response.status}: ${text.slice(0, 200)}` } };
197 }
198}
199
200async function chatMessages(workspace, channel) {
201 const response = await site(`/${workspace}/-/chat/api?channel=${encodeURIComponent(channel)}&limit=60`);
202 const data = await response.json().catch(() => null);
203 return data?.value?.messages ?? [];
204}
205
206/**
207 * Asks once, as the user, the way Ask and Chat both do: in the user's
208 * direct message with @g1t, after a New conversation mark, posting each of
209 * the conversation's questions in turn and following g1t's answer to each
210 * as it streams (polled). The last answer is the one graded: its text, the
211 * time to its first words and to its end, and the steps and sources it kept,
212 * as events in the order the stream gave them.
213 */
214export async function ask(workspace, { messages, page }, { timeoutMs = 180_000 } = {}) {
215 const started = performance.now();
216 const at = () => Math.round(performance.now() - started);
217 const result = {
218 status: null,
219 events: [],
220 unknown_types: [],
221 text: "",
222 first_event_ms: null,
223 first_text_ms: null,
224 total_ms: null,
225 meta: null,
226 done: null,
227 error: null,
228 tool_events: [],
229 source_events: [],
230 source_urls: [],
231 };
232 try {
233 const dm = await chatApi(workspace, { intent: "g1t_dm" });
234 if (!dm.ok) throw new Error(dm.error?.message ?? "could not open the direct message with g1t");
235 const channel = dm.value.channel.id;
236 await chatApi(workspace, { intent: "g1t_reset", channel_id: channel });
237 const questions = messages.filter((m) => m.role === "user");
238 let answer = null;
239 for (const [index, question] of questions.entries()) {
240 const last = index === questions.length - 1;
241 const asked = performance.now();
242 const posted = await chatApi(workspace, { intent: "post", channel_id: channel, body: question.content, page: last ? (page ?? null) : null });
243 result.status = posted.ok ? 200 : 400;
244 if (!posted.ok) throw new Error(posted.error?.message ?? "could not post");
245 const mine = posted.value.id;
246 answer = null;
247 while (performance.now() - asked < timeoutMs) {
248 await sleep(700);
249 const list = await chatMessages(workspace, channel);
250 answer = list.find((m) => m.id > mine && m.author?.kind === "system" && m.answer) ?? null;
251 if (last && answer?.body && result.first_text_ms === null) result.first_text_ms = at();
252 if (last && answer && result.first_event_ms === null) result.first_event_ms = at();
253 if (answer && ["done", "error", "stopped"].includes(answer.answer.state)) break;
254 }
255 if (!answer || !["done", "error", "stopped"].includes(answer.answer?.state)) throw new Error(`timed out after ${timeoutMs} ms`);
256 }
257 const kept = answer.answer;
258 result.meta = { type: "meta", provider_label: kept.provider_label, model: kept.model };
259 result.events.push({ ms: 0, event: result.meta });
260 for (const step of kept.steps ?? []) {
261 const type = String(step?.type ?? "");
262 result.events.push({ ms: null, event: step });
263 if (!KNOWN.has(type) && !result.unknown_types.includes(type)) result.unknown_types.push(type);
264 if (isToolEvent(step)) result.tool_events.push({ ms: null, event: step });
265 if (isSourceEvent(step)) result.source_events.push({ ms: null, event: step });
266 }
267 result.text = answer.body ?? "";
268 result.events.push({ ms: at(), event: { type: "text", text: result.text } });
269 if (kept.state === "done") result.done = { type: "done", stop_reason: kept.stop_reason ?? null };
270 else result.error = kept.error ?? kept.state;
271 } catch (error) {
272 result.error = String(error?.message ?? error);
273 } finally {
274 result.total_ms = at();
275 }
276 const urls = new Set();
277 for (const { event } of result.source_events) urlsIn(event, urls);
278 result.source_urls = [...urls];
279 return result;
280}
281
282export const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
283
284/** `--flag value` and `--flag` into an object; the first bare word is the command. */
285export function parseArgs(argv) {
286 const flags = {};
287 const words = [];
288 for (let i = 0; i < argv.length; i++) {
289 const arg = argv[i];
290 if (!arg.startsWith("--")) {
291 words.push(arg);
292 continue;
293 }
294 const [name, inline] = arg.slice(2).split("=", 2);
295 if (inline !== undefined) flags[name] = inline;
296 else if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) flags[name] = true;
297 else flags[name] = argv[++i];
298 }
299 return { command: words[0], words, flags };
300}