| 1 | // What the Ask bench shares: where the repository is, the credentials it |
| 2 | // reads (never printed, logged or written: every string that leaves this |
| 3 | // process goes through `redact`), g1t's REST API, its MCP server, the site |
| 4 | // as the signed-in user, and one Ask answer read as it streams. |
| 5 | |
| 6 | import { execFileSync } from "node:child_process"; |
| 7 | import { existsSync, readFileSync } from "node:fs"; |
| 8 | import { homedir } from "node:os"; |
| 9 | import { dirname, isAbsolute, join, resolve } from "node:path"; |
| 10 | import { fileURLToPath } from "node:url"; |
| 11 | |
| 12 | export const HERE = dirname(fileURLToPath(import.meta.url)); |
| 13 | |
| 14 | /** The checkout the bench runs from: a worktree's own folder. */ |
| 15 | export const ROOT = resolve(HERE, "..", ".."); |
| 16 | |
| 17 | /** |
| 18 | * The main checkout, where `.credentials/` and `.secrets/` live: a worktree |
| 19 | * has neither, so they are read from the checkout its git directory belongs to. |
| 20 | */ |
| 21 | export function mainCheckout() { |
| 22 | try { |
| 23 | const common = execFileSync("git", ["rev-parse", "--git-common-dir"], { cwd: ROOT, encoding: "utf8" }).trim(); |
| 24 | const dir = isAbsolute(common) ? common : resolve(ROOT, common); |
| 25 | return dirname(dir); |
| 26 | } catch { |
| 27 | return ROOT; |
| 28 | } |
| 29 | } |
| 30 | |
| 31 | export const SITE = (process.env.G1T_SITE ?? "https://g1t.sh").replace(/\/$/, ""); |
| 32 | export const API = (process.env.G1T_API ?? "https://api.g1t.sh").replace(/\/$/, ""); |
| 33 | export const MCP = (process.env.G1T_MCP ?? "https://mcp.g1t.sh").replace(/\/$/, ""); |
| 34 | const UA = "g1t-ask-bench/1"; |
| 35 | |
| 36 | // --- credentials ---------------------------------------------------------- |
| 37 | |
| 38 | const SECRETS = new Set(); |
| 39 | |
| 40 | function remember(value) { |
| 41 | if (value && value.length >= 8) SECRETS.add(value); |
| 42 | return value; |
| 43 | } |
| 44 | |
| 45 | function readSecretFile(path) { |
| 46 | if (!existsSync(path)) return null; |
| 47 | // Either the bare value or a `token: value` line. |
| 48 | const text = readFileSync(path, "utf8").trim(); |
| 49 | const value = text.replace(/^\s*token:\s*/i, "").split(/\r?\n/)[0].trim(); |
| 50 | return remember(value) || null; |
| 51 | } |
| 52 | |
| 53 | let apiToken; |
| 54 | /** The API token: G1T_TOKEN, or the export in ~/.bashrc. */ |
| 55 | export function apiTokenValue() { |
| 56 | if (apiToken !== undefined) return apiToken; |
| 57 | let value = process.env.G1T_TOKEN; |
| 58 | if (!value) { |
| 59 | const rc = join(homedir(), ".bashrc"); |
| 60 | if (existsSync(rc)) value = readFileSync(rc, "utf8").match(/G1T_TOKEN=["']?([^"'\s]+)/)?.[1]; |
| 61 | } |
| 62 | apiToken = remember(value) || null; |
| 63 | return apiToken; |
| 64 | } |
| 65 | |
| 66 | let webToken; |
| 67 | /** The site's token, for g1t.sh routes as the user: .credentials/g1t_token.txt (or G1T_WEB_TOKEN_FILE). */ |
| 68 | export function webTokenValue() { |
| 69 | if (webToken !== undefined) return webToken; |
| 70 | const file = process.env.G1T_WEB_TOKEN_FILE ?? join(mainCheckout(), ".credentials", "g1t_token.txt"); |
| 71 | webToken = readSecretFile(file) ?? apiTokenValue(); |
| 72 | return webToken; |
| 73 | } |
| 74 | |
| 75 | let anthropicKey; |
| 76 | /** The judge's key: .secrets/anthropic-key.txt (or ANTHROPIC_KEY_FILE). */ |
| 77 | export function anthropicKeyValue() { |
| 78 | if (anthropicKey !== undefined) return anthropicKey; |
| 79 | const file = process.env.ANTHROPIC_KEY_FILE ?? join(mainCheckout(), ".secrets", "anthropic-key.txt"); |
| 80 | anthropicKey = readSecretFile(file); |
| 81 | return anthropicKey; |
| 82 | } |
| 83 | |
| 84 | /** Loads every credential, so `redact` knows them all before anything is written. */ |
| 85 | export function loadCredentials() { |
| 86 | return { api: Boolean(apiTokenValue()), web: Boolean(webTokenValue()), anthropic: Boolean(anthropicKeyValue()) }; |
| 87 | } |
| 88 | |
| 89 | /** Every credential this process read, for the no-secrets check. */ |
| 90 | export function knownSecrets() { |
| 91 | return [...SECRETS]; |
| 92 | } |
| 93 | |
| 94 | /** Takes every credential this process read, and anything shaped like one, out of text. */ |
| 95 | export function redact(text) { |
| 96 | let out = String(text ?? ""); |
| 97 | for (const secret of SECRETS) out = out.split(secret).join("[REDACTED]"); |
| 98 | return out |
| 99 | .replace(/\bsk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-[REDACTED]") |
| 100 | .replace(/\bg1t_[A-Za-z0-9_-]{16,}/g, "g1t_[REDACTED]"); |
| 101 | } |
| 102 | |
| 103 | // --- g1t ------------------------------------------------------------------ |
| 104 | |
| 105 | async function request(url, token, init = {}) { |
| 106 | const headers = { "user-agent": UA, accept: "application/json", ...(init.headers ?? {}) }; |
| 107 | if (token) headers.authorization = `Bearer ${token}`; |
| 108 | const response = await fetch(url, { ...init, headers, redirect: init.redirect ?? "manual" }); |
| 109 | return response; |
| 110 | } |
| 111 | |
| 112 | /** GET (or more) on api.g1t.sh as the user: `{ status, data }`, data null when not JSON. */ |
| 113 | export async function api(path, init = {}) { |
| 114 | const response = await request(`${API}${path}`, apiTokenValue(), init); |
| 115 | const text = await response.text(); |
| 116 | let data = null; |
| 117 | try { |
| 118 | data = JSON.parse(text); |
| 119 | } catch {} |
| 120 | return { status: response.status, data, text }; |
| 121 | } |
| 122 | |
| 123 | /** A g1t.sh route as the signed-in user (the site's token). */ |
| 124 | export async function site(path, init = {}) { |
| 125 | const url = path.startsWith("http") ? path : `${SITE}${path}`; |
| 126 | return request(url, webTokenValue(), init); |
| 127 | } |
| 128 | |
| 129 | let mcpId = 0; |
| 130 | /** One MCP tool call on mcp.g1t.sh as the user; the tool's text parsed as JSON when it is. */ |
| 131 | export async function mcp(tool, args) { |
| 132 | const response = await request(MCP, apiTokenValue(), { |
| 133 | method: "POST", |
| 134 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream" }, |
| 135 | body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpId, method: "tools/call", params: { name: tool, arguments: args } }), |
| 136 | }); |
| 137 | const text = await response.text(); |
| 138 | // A streamable HTTP server may answer as one SSE event. |
| 139 | const json = text.trimStart().startsWith("{") ? text : (text.match(/^data: (.*)$/m)?.[1] ?? "null"); |
| 140 | const message = JSON.parse(json); |
| 141 | if (message?.error) throw new Error(`mcp ${tool}: ${message.error.message}`); |
| 142 | const body = message?.result?.content?.map((c) => c.text ?? "").join("") ?? ""; |
| 143 | if (message?.result?.isError) throw new Error(`mcp ${tool}: ${body.slice(0, 300)}`); |
| 144 | try { |
| 145 | return JSON.parse(body); |
| 146 | } catch { |
| 147 | return body; |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | /** Every page of a list endpoint that takes per_page and page, up to `max` items. */ |
| 152 | export async function apiAll(path, max = 300) { |
| 153 | const items = []; |
| 154 | for (let page = 1; items.length < max; page++) { |
| 155 | const sep = path.includes("?") ? "&" : "?"; |
| 156 | const { status, data } = await api(`${path}${sep}per_page=100&page=${page}`); |
| 157 | if (status !== 200) break; |
| 158 | const list = Array.isArray(data) ? data : (data?.items ?? data?.workflow_runs ?? data?.deployments ?? []); |
| 159 | items.push(...list); |
| 160 | if (list.length < 100) break; |
| 161 | } |
| 162 | return items.slice(0, max); |
| 163 | } |
| 164 | |
| 165 | // --- Ask ------------------------------------------------------------------ |
| 166 | |
| 167 | /** Event types the bench knows. Anything else is recorded as unknown, never dropped. */ |
| 168 | const KNOWN = new Set(["meta", "text", "done", "error", "tool", "sources", "proposal"]); |
| 169 | |
| 170 | /** Looks like a tool step: `tool`, `tool_call`, `tool_result`, `step`, … */ |
| 171 | export const isToolEvent = (event) => /tool|step|action|call/i.test(String(event?.type ?? "")); |
| 172 | /** Looks like a source or citation. */ |
| 173 | export const isSourceEvent = (event) => /source|citation|reference|link/i.test(String(event?.type ?? "")); |
| 174 | |
| 175 | /** URLs an event names, wherever it keeps them. */ |
| 176 | function urlsIn(value, found = new Set()) { |
| 177 | if (typeof value === "string") { |
| 178 | for (const match of value.matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) found.add(match[0]); |
| 179 | if (/^\/[a-z0-9]/i.test(value) && !value.includes(" ")) found.add(`${SITE}${value}`); |
| 180 | } else if (Array.isArray(value)) value.forEach((v) => urlsIn(v, found)); |
| 181 | else if (value && typeof value === "object") Object.values(value).forEach((v) => urlsIn(v, found)); |
| 182 | return found; |
| 183 | } |
| 184 | |
| 185 | /** What every question the bench posts starts with. */ |
| 186 | export const TEST_PREFIX = "[TEST] "; |
| 187 | |
| 188 | /** A chat API call on the site as the user: the `{ ok, value | error }` it answers. */ |
| 189 | async function chatApi(workspace, body) { |
| 190 | const response = await site(`/${workspace}/-/chat/api`, { |
| 191 | method: "POST", |
| 192 | headers: { "content-type": "application/json", origin: SITE }, |
| 193 | body: JSON.stringify(body), |
| 194 | }); |
| 195 | const text = await response.text(); |
| 196 | try { |
| 197 | return JSON.parse(text); |
| 198 | } catch { |
| 199 | return { ok: false, error: { message: `chat api ${response.status}: ${text.slice(0, 200)}` } }; |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | async function chatMessages(workspace, channel) { |
| 204 | const response = await site(`/${workspace}/-/chat/api?channel=${encodeURIComponent(channel)}&limit=60`); |
| 205 | const data = await response.json().catch(() => null); |
| 206 | return data?.value?.messages ?? []; |
| 207 | } |
| 208 | |
| 209 | /** |
| 210 | * Asks once, as the user, the way Ask and Chat both do: in the user's |
| 211 | * direct message with @g1t, after a New conversation mark, posting each of |
| 212 | * the conversation's questions in turn and following g1t's answer to each |
| 213 | * as it streams (polled). The last answer is the one graded: its text, the |
| 214 | * time to its first words and to its end, and the steps and sources it kept, |
| 215 | * as events in the order the stream gave them. |
| 216 | */ |
| 217 | export async function ask(workspace, { messages, page }, { timeoutMs = 180_000 } = {}) { |
| 218 | const started = performance.now(); |
| 219 | const at = () => Math.round(performance.now() - started); |
| 220 | const result = { |
| 221 | status: null, |
| 222 | events: [], |
| 223 | unknown_types: [], |
| 224 | text: "", |
| 225 | first_event_ms: null, |
| 226 | first_text_ms: null, |
| 227 | total_ms: null, |
| 228 | meta: null, |
| 229 | done: null, |
| 230 | error: null, |
| 231 | tool_events: [], |
| 232 | source_events: [], |
| 233 | source_urls: [], |
| 234 | }; |
| 235 | try { |
| 236 | const dm = await chatApi(workspace, { intent: "g1t_dm" }); |
| 237 | if (!dm.ok) throw new Error(dm.error?.message ?? "could not open the direct message with g1t"); |
| 238 | const channel = dm.value.channel.id; |
| 239 | await chatApi(workspace, { intent: "g1t_reset", channel_id: channel }); |
| 240 | const questions = messages.filter((m) => m.role === "user"); |
| 241 | let answer = null; |
| 242 | for (const [index, question] of questions.entries()) { |
| 243 | const last = index === questions.length - 1; |
| 244 | const asked = performance.now(); |
| 245 | // Marked, so the person whose direct message it is can tell the bench's questions from their own. |
| 246 | const posted = await chatApi(workspace, { intent: "post", channel_id: channel, body: `${TEST_PREFIX}${question.content}`, page: last ? (page ?? null) : null }); |
| 247 | result.status = posted.ok ? 200 : 400; |
| 248 | if (!posted.ok) throw new Error(posted.error?.message ?? "could not post"); |
| 249 | const mine = posted.value.id; |
| 250 | answer = null; |
| 251 | while (performance.now() - asked < timeoutMs) { |
| 252 | await sleep(700); |
| 253 | const list = await chatMessages(workspace, channel); |
| 254 | answer = list.find((m) => m.id > mine && m.author?.kind === "system" && m.answer) ?? null; |
| 255 | if (last && answer?.body && result.first_text_ms === null) result.first_text_ms = at(); |
| 256 | if (last && answer && result.first_event_ms === null) result.first_event_ms = at(); |
| 257 | if (answer && ["done", "error", "stopped"].includes(answer.answer.state)) break; |
| 258 | } |
| 259 | if (!answer || !["done", "error", "stopped"].includes(answer.answer?.state)) throw new Error(`timed out after ${timeoutMs} ms`); |
| 260 | } |
| 261 | const kept = answer.answer; |
| 262 | result.meta = { type: "meta", provider_label: kept.provider_label, model: kept.model }; |
| 263 | result.events.push({ ms: 0, event: result.meta }); |
| 264 | for (const step of kept.steps ?? []) { |
| 265 | const type = String(step?.type ?? ""); |
| 266 | result.events.push({ ms: null, event: step }); |
| 267 | if (!KNOWN.has(type) && !result.unknown_types.includes(type)) result.unknown_types.push(type); |
| 268 | if (isToolEvent(step)) result.tool_events.push({ ms: null, event: step }); |
| 269 | if (isSourceEvent(step)) result.source_events.push({ ms: null, event: step }); |
| 270 | } |
| 271 | result.text = answer.body ?? ""; |
| 272 | result.events.push({ ms: at(), event: { type: "text", text: result.text } }); |
| 273 | if (kept.state === "done") result.done = { type: "done", stop_reason: kept.stop_reason ?? null }; |
| 274 | else result.error = kept.error ?? kept.state; |
| 275 | } catch (error) { |
| 276 | result.error = String(error?.message ?? error); |
| 277 | } finally { |
| 278 | result.total_ms = at(); |
| 279 | } |
| 280 | const urls = new Set(); |
| 281 | for (const { event } of result.source_events) urlsIn(event, urls); |
| 282 | result.source_urls = [...urls]; |
| 283 | return result; |
| 284 | } |
| 285 | |
| 286 | export const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); |
| 287 | |
| 288 | /** `--flag value` and `--flag` into an object; the first bare word is the command. */ |
| 289 | export function parseArgs(argv) { |
| 290 | const flags = {}; |
| 291 | const words = []; |
| 292 | for (let i = 0; i < argv.length; i++) { |
| 293 | const arg = argv[i]; |
| 294 | if (!arg.startsWith("--")) { |
| 295 | words.push(arg); |
| 296 | continue; |
| 297 | } |
| 298 | const [name, inline] = arg.slice(2).split("=", 2); |
| 299 | if (inline !== undefined) flags[name] = inline; |
| 300 | else if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) flags[name] = true; |
| 301 | else flags[name] = argv[++i]; |
| 302 | } |
| 303 | return { command: words[0], words, flags }; |
| 304 | } |