Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 1 | // What the Ask bench shares: where the repository is, the credentials it |
| 2 | // reads (never printed, logged or written: every string that leaves this | |
| 3 | // process goes through `redact`), g1t's REST API, its MCP server, the site | |
| 4 | // as the signed-in user, and one Ask answer read as it streams. | |
| 5 | ||
| 6 | import { execFileSync } from "node:child_process"; | |
| 7 | import { existsSync, readFileSync } from "node:fs"; | |
| 8 | import { homedir } from "node:os"; | |
| 9 | import { dirname, isAbsolute, join, resolve } from "node:path"; | |
| 10 | import { fileURLToPath } from "node:url"; | |
| 11 | ||
| 12 | export const HERE = dirname(fileURLToPath(import.meta.url)); | |
| 13 | ||
| 14 | /** The checkout the bench runs from: a worktree's own folder. */ | |
| 15 | export const ROOT = resolve(HERE, "..", ".."); | |
| 16 | ||
| 17 | /** | |
| 18 | * The main checkout, where `.credentials/` and `.secrets/` live: a worktree | |
| 19 | * has neither, so they are read from the checkout its git directory belongs to. | |
| 20 | */ | |
| 21 | export function mainCheckout() { | |
| 22 | try { | |
| 23 | const common = execFileSync("git", ["rev-parse", "--git-common-dir"], { cwd: ROOT, encoding: "utf8" }).trim(); | |
| 24 | const dir = isAbsolute(common) ? common : resolve(ROOT, common); | |
| 25 | return dirname(dir); | |
| 26 | } catch { | |
| 27 | return ROOT; | |
| 28 | } | |
| 29 | } | |
| 30 | ||
| 31 | export const SITE = (process.env.G1T_SITE ?? "https://g1t.sh").replace(/\/$/, ""); | |
| 32 | export const API = (process.env.G1T_API ?? "https://api.g1t.sh").replace(/\/$/, ""); | |
| 33 | export const MCP = (process.env.G1T_MCP ?? "https://mcp.g1t.sh").replace(/\/$/, ""); | |
| 34 | const UA = "g1t-ask-bench/1"; | |
| 35 | ||
| 36 | // --- credentials ---------------------------------------------------------- | |
| 37 | ||
| 38 | const SECRETS = new Set(); | |
| 39 | ||
| 40 | function remember(value) { | |
| 41 | if (value && value.length >= 8) SECRETS.add(value); | |
| 42 | return value; | |
| 43 | } | |
| 44 | ||
| 45 | function readSecretFile(path) { | |
| 46 | if (!existsSync(path)) return null; | |
| 47 | // Either the bare value or a `token: value` line. | |
| 48 | const text = readFileSync(path, "utf8").trim(); | |
| 49 | const value = text.replace(/^\s*token:\s*/i, "").split(/\r?\n/)[0].trim(); | |
| 50 | return remember(value) || null; | |
| 51 | } | |
| 52 | ||
| 53 | let apiToken; | |
| 54 | /** The API token: G1T_TOKEN, or the export in ~/.bashrc. */ | |
| 55 | export function apiTokenValue() { | |
| 56 | if (apiToken !== undefined) return apiToken; | |
| 57 | let value = process.env.G1T_TOKEN; | |
| 58 | if (!value) { | |
| 59 | const rc = join(homedir(), ".bashrc"); | |
| 60 | if (existsSync(rc)) value = readFileSync(rc, "utf8").match(/G1T_TOKEN=["']?([^"'\s]+)/)?.[1]; | |
| 61 | } | |
| 62 | apiToken = remember(value) || null; | |
| 63 | return apiToken; | |
| 64 | } | |
| 65 | ||
| 66 | let webToken; | |
| 67 | /** The site's token, for g1t.sh routes as the user: .credentials/g1t_token.txt (or G1T_WEB_TOKEN_FILE). */ | |
| 68 | export function webTokenValue() { | |
| 69 | if (webToken !== undefined) return webToken; | |
| 70 | const file = process.env.G1T_WEB_TOKEN_FILE ?? join(mainCheckout(), ".credentials", "g1t_token.txt"); | |
| 71 | webToken = readSecretFile(file) ?? apiTokenValue(); | |
| 72 | return webToken; | |
| 73 | } | |
| 74 | ||
| 75 | let anthropicKey; | |
| 76 | /** The judge's key: .secrets/anthropic-key.txt (or ANTHROPIC_KEY_FILE). */ | |
| 77 | export function anthropicKeyValue() { | |
| 78 | if (anthropicKey !== undefined) return anthropicKey; | |
| 79 | const file = process.env.ANTHROPIC_KEY_FILE ?? join(mainCheckout(), ".secrets", "anthropic-key.txt"); | |
| 80 | anthropicKey = readSecretFile(file); | |
| 81 | return anthropicKey; | |
| 82 | } | |
| 83 | ||
| 84 | /** Loads every credential, so `redact` knows them all before anything is written. */ | |
| 85 | export function loadCredentials() { | |
| 86 | return { api: Boolean(apiTokenValue()), web: Boolean(webTokenValue()), anthropic: Boolean(anthropicKeyValue()) }; | |
| 87 | } | |
| 88 | ||
| 89 | /** Every credential this process read, for the no-secrets check. */ | |
| 90 | export function knownSecrets() { | |
| 91 | return [...SECRETS]; | |
| 92 | } | |
| 93 | ||
| 94 | /** Takes every credential this process read, and anything shaped like one, out of text. */ | |
| 95 | export function redact(text) { | |
| 96 | let out = String(text ?? ""); | |
| 97 | for (const secret of SECRETS) out = out.split(secret).join("[REDACTED]"); | |
| 98 | return out | |
| 99 | .replace(/\bsk-ant-[A-Za-z0-9_-]{8,}/g, "sk-ant-[REDACTED]") | |
| 100 | .replace(/\bg1t_[A-Za-z0-9_-]{16,}/g, "g1t_[REDACTED]"); | |
| 101 | } | |
| 102 | ||
| 103 | // --- g1t ------------------------------------------------------------------ | |
| 104 | ||
| 105 | async function request(url, token, init = {}) { | |
| 106 | const headers = { "user-agent": UA, accept: "application/json", ...(init.headers ?? {}) }; | |
| 107 | if (token) headers.authorization = `Bearer ${token}`; | |
| 108 | const response = await fetch(url, { ...init, headers, redirect: init.redirect ?? "manual" }); | |
| 109 | return response; | |
| 110 | } | |
| 111 | ||
| 112 | /** GET (or more) on api.g1t.sh as the user: `{ status, data }`, data null when not JSON. */ | |
| 113 | export async function api(path, init = {}) { | |
| 114 | const response = await request(`${API}${path}`, apiTokenValue(), init); | |
| 115 | const text = await response.text(); | |
| 116 | let data = null; | |
| 117 | try { | |
| 118 | data = JSON.parse(text); | |
| 119 | } catch {} | |
| 120 | return { status: response.status, data, text }; | |
| 121 | } | |
| 122 | ||
| 123 | /** A g1t.sh route as the signed-in user (the site's token). */ | |
| 124 | export async function site(path, init = {}) { | |
| 125 | const url = path.startsWith("http") ? path : `${SITE}${path}`; | |
| 126 | return request(url, webTokenValue(), init); | |
| 127 | } | |
| 128 | ||
| 129 | let mcpId = 0; | |
| 130 | /** One MCP tool call on mcp.g1t.sh as the user; the tool's text parsed as JSON when it is. */ | |
| 131 | export async function mcp(tool, args) { | |
| 132 | const response = await request(MCP, apiTokenValue(), { | |
| 133 | method: "POST", | |
| 134 | headers: { "content-type": "application/json", accept: "application/json, text/event-stream" }, | |
| 135 | body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpId, method: "tools/call", params: { name: tool, arguments: args } }), | |
| 136 | }); | |
| 137 | const text = await response.text(); | |
| 138 | // A streamable HTTP server may answer as one SSE event. | |
| 139 | const json = text.trimStart().startsWith("{") ? text : (text.match(/^data: (.*)$/m)?.[1] ?? "null"); | |
| 140 | const message = JSON.parse(json); | |
| 141 | if (message?.error) throw new Error(`mcp ${tool}: ${message.error.message}`); | |
| 142 | const body = message?.result?.content?.map((c) => c.text ?? "").join("") ?? ""; | |
| 143 | if (message?.result?.isError) throw new Error(`mcp ${tool}: ${body.slice(0, 300)}`); | |
| 144 | try { | |
| 145 | return JSON.parse(body); | |
| 146 | } catch { | |
| 147 | return body; | |
| 148 | } | |
| 149 | } | |
| 150 | ||
| 151 | /** Every page of a list endpoint that takes per_page and page, up to `max` items. */ | |
| 152 | export async function apiAll(path, max = 300) { | |
| 153 | const items = []; | |
| 154 | for (let page = 1; items.length < max; page++) { | |
| 155 | const sep = path.includes("?") ? "&" : "?"; | |
| 156 | const { status, data } = await api(`${path}${sep}per_page=100&page=${page}`); | |
| 157 | if (status !== 200) break; | |
| 158 | const list = Array.isArray(data) ? data : (data?.items ?? data?.workflow_runs ?? data?.deployments ?? []); | |
| 159 | items.push(...list); | |
| 160 | if (list.length < 100) break; | |
| 161 | } | |
| 162 | return items.slice(0, max); | |
| 163 | } | |
| 164 | ||
| 165 | // --- Ask ------------------------------------------------------------------ | |
| 166 | ||
| 167 | /** Event types the bench knows. Anything else is recorded as unknown, never dropped. */ | |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 168 | const KNOWN = new Set(["meta", "text", "done", "error", "tool", "sources", "proposal"]); |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 169 | |
| 170 | /** Looks like a tool step: `tool`, `tool_call`, `tool_result`, `step`, … */ | |
| 171 | export const isToolEvent = (event) => /tool|step|action|call/i.test(String(event?.type ?? "")); | |
| 172 | /** Looks like a source or citation. */ | |
| 173 | export const isSourceEvent = (event) => /source|citation|reference|link/i.test(String(event?.type ?? "")); | |
| 174 | ||
| 175 | /** URLs an event names, wherever it keeps them. */ | |
| 176 | function urlsIn(value, found = new Set()) { | |
| 177 | if (typeof value === "string") { | |
| 178 | for (const match of value.matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) found.add(match[0]); | |
| 179 | if (/^\/[a-z0-9]/i.test(value) && !value.includes(" ")) found.add(`${SITE}${value}`); | |
| 180 | } else if (Array.isArray(value)) value.forEach((v) => urlsIn(v, found)); | |
| 181 | else if (value && typeof value === "object") Object.values(value).forEach((v) => urlsIn(v, found)); | |
| 182 | return found; | |
| 183 | } | |
| 184 | ||
| A question to @g1t asked on a page says which page under it, in Chat and in the Ask panel: "Asked on pull request #8 in flagon-io/g1t", linking back, so "summarize this pull request" or "is this done?" reads as the one it was about when the conversation is read later, as g1t was told. A question written in Chat itself, whose page is the direct message, says nothing. The quality bar's questions start with [TEST], so they are told apart from the person's own in the direct message they are asked in. Checked: the site's typecheck and its 890 tests, a production build whose chunks load first without error (scripted in a browser over the build), and the bench's next run. | 185 | /** What every question the bench posts starts with. */ |
| 186 | export const TEST_PREFIX = "[TEST] "; | |
| 187 | ||
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 188 | /** A chat API call on the site as the user: the `{ ok, value | error }` it answers. */ |
| 189 | async function chatApi(workspace, body) { | |
| 190 | const response = await site(`/${workspace}/-/chat/api`, { | |
| 191 | method: "POST", | |
| 192 | headers: { "content-type": "application/json", origin: SITE }, | |
| 193 | body: JSON.stringify(body), | |
| 194 | }); | |
| 195 | const text = await response.text(); | |
| 196 | try { | |
| 197 | return JSON.parse(text); | |
| 198 | } catch { | |
| 199 | return { ok: false, error: { message: `chat api ${response.status}: ${text.slice(0, 200)}` } }; | |
| 200 | } | |
| 201 | } | |
| 202 | ||
| 203 | async function chatMessages(workspace, channel) { | |
| 204 | const response = await site(`/${workspace}/-/chat/api?channel=${encodeURIComponent(channel)}&limit=60`); | |
| 205 | const data = await response.json().catch(() => null); | |
| 206 | return data?.value?.messages ?? []; | |
| 207 | } | |
| 208 | ||
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 209 | /** |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 210 | * Asks once, as the user, the way Ask and Chat both do: in the user's |
| 211 | * direct message with @g1t, after a New conversation mark, posting each of | |
| 212 | * the conversation's questions in turn and following g1t's answer to each | |
| 213 | * as it streams (polled). The last answer is the one graded: its text, the | |
| 214 | * time to its first words and to its end, and the steps and sources it kept, | |
| 215 | * as events in the order the stream gave them. | |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 216 | */ |
| 217 | export async function ask(workspace, { messages, page }, { timeoutMs = 180_000 } = {}) { | |
| 218 | const started = performance.now(); | |
| 219 | const at = () => Math.round(performance.now() - started); | |
| 220 | const result = { | |
| 221 | status: null, | |
| 222 | events: [], | |
| 223 | unknown_types: [], | |
| 224 | text: "", | |
| 225 | first_event_ms: null, | |
| 226 | first_text_ms: null, | |
| 227 | total_ms: null, | |
| 228 | meta: null, | |
| 229 | done: null, | |
| 230 | error: null, | |
| 231 | tool_events: [], | |
| 232 | source_events: [], | |
| 233 | source_urls: [], | |
| 234 | }; | |
| 235 | try { | |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 236 | const dm = await chatApi(workspace, { intent: "g1t_dm" }); |
| 237 | if (!dm.ok) throw new Error(dm.error?.message ?? "could not open the direct message with g1t"); | |
| 238 | const channel = dm.value.channel.id; | |
| 239 | await chatApi(workspace, { intent: "g1t_reset", channel_id: channel }); | |
| 240 | const questions = messages.filter((m) => m.role === "user"); | |
| 241 | let answer = null; | |
| 242 | for (const [index, question] of questions.entries()) { | |
| 243 | const last = index === questions.length - 1; | |
| 244 | const asked = performance.now(); | |
| A question to @g1t asked on a page says which page under it, in Chat and in the Ask panel: "Asked on pull request #8 in flagon-io/g1t", linking back, so "summarize this pull request" or "is this done?" reads as the one it was about when the conversation is read later, as g1t was told. A question written in Chat itself, whose page is the direct message, says nothing. The quality bar's questions start with [TEST], so they are told apart from the person's own in the direct message they are asked in. Checked: the site's typecheck and its 890 tests, a production build whose chunks load first without error (scripted in a browser over the build), and the bench's next run. | 245 | // Marked, so the person whose direct message it is can tell the bench's questions from their own. |
| 246 | const posted = await chatApi(workspace, { intent: "post", channel_id: channel, body: `${TEST_PREFIX}${question.content}`, page: last ? (page ?? null) : null }); | |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 247 | result.status = posted.ok ? 200 : 400; |
| 248 | if (!posted.ok) throw new Error(posted.error?.message ?? "could not post"); | |
| 249 | const mine = posted.value.id; | |
| 250 | answer = null; | |
| 251 | while (performance.now() - asked < timeoutMs) { | |
| 252 | await sleep(700); | |
| 253 | const list = await chatMessages(workspace, channel); | |
| 254 | answer = list.find((m) => m.id > mine && m.author?.kind === "system" && m.answer) ?? null; | |
| 255 | if (last && answer?.body && result.first_text_ms === null) result.first_text_ms = at(); | |
| 256 | if (last && answer && result.first_event_ms === null) result.first_event_ms = at(); | |
| 257 | if (answer && ["done", "error", "stopped"].includes(answer.answer.state)) break; | |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 258 | } |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 259 | if (!answer || !["done", "error", "stopped"].includes(answer.answer?.state)) throw new Error(`timed out after ${timeoutMs} ms`); |
| 260 | } | |
| 261 | const kept = answer.answer; | |
| 262 | result.meta = { type: "meta", provider_label: kept.provider_label, model: kept.model }; | |
| 263 | result.events.push({ ms: 0, event: result.meta }); | |
| 264 | for (const step of kept.steps ?? []) { | |
| 265 | const type = String(step?.type ?? ""); | |
| 266 | result.events.push({ ms: null, event: step }); | |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 267 | if (!KNOWN.has(type) && !result.unknown_types.includes(type)) result.unknown_types.push(type); |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 268 | if (isToolEvent(step)) result.tool_events.push({ ms: null, event: step }); |
| 269 | if (isSourceEvent(step)) result.source_events.push({ ms: null, event: step }); | |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 270 | } |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 271 | result.text = answer.body ?? ""; |
| 272 | result.events.push({ ms: at(), event: { type: "text", text: result.text } }); | |
| 273 | if (kept.state === "done") result.done = { type: "done", stop_reason: kept.stop_reason ?? null }; | |
| 274 | else result.error = kept.error ?? kept.state; | |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 275 | } catch (error) { |
| Ask answers "what shipped today" for the asker's own today, and says what changed rather than only how much. "Today" begins at the asker's local midnight: integrations reads the time zone on their profile, or failing that the one their browser sends with the question (the Ask panel and the Chat composer send it; chat keeps it on a question to @g1t, an IANA name only), works out with the runtime's Intl when their day began in UTC (local_day.rs, tested across zones west and east of UTC and on a half hour), and the system prompt says "they are in America/Denver, where it is 2026-10-12: today means since 2026-10-12T06:00:00Z"; without a zone it stays UTC. get_workspace_activity's answer gains `landed`: for the repositories whose default branch was pushed to in the span (busiest first, at most eight), the commits made in the span, one line each (repo, short sha, subject, author, time, link), at most forty, so a workspace that pushes to main rather than merging pull requests still shows what shipped; the prompt has Ask say what those changes were, pass `since` as the start of the asker's today, and narrow the kinds or the span when `complete` is false rather than guessing. A question asked in Chat keeps the direct message's path when its page has none of its own. The quality bar (bench/ask) asks through the direct message with @g1t, as the panel and Chat do now: it opens it, marks a new conversation, posts each question and follows g1t's answer to its end, and grades the steps, sources and proposals it kept. Its first run on production, with tools, had shipped-today pass every check (8/8) and the judge mark it 2.83, for the UTC day and the thin summary this fixes. Checked: cargo test for the workspace (the actions crates' repository-file tests on their own, as on this machine they cannot find their files from the workspace), the wasm build of integrations and the API, the TypeScript typecheck, every workspace's tests, the deploy tool's and the plugin's tests, and the OpenAPI document regenerated. | 276 | result.error = String(error?.message ?? error); |
| Ask g1t has a quality bar, in bench/ask: 27 questions over flagon-io (what shipped today, what is open and blocking, summarizing pull requests, why the last Deploy failed, whether production is healthy, who is working on what, why a file changed, search, questions asked from an issue or pull request page, a follow-up in a conversation, a pull request that does not exist, and four safety cases: a secret's value, the workspace's provider key, an issue body that tells assistants what to do, and a repository the asker cannot see) are asked on the live Ask endpoint as the user, and each answer is graded against ground truth fetched from g1t's API and MCP server at the same moment. Deterministic checks look for the facts that ground truth requires, for links that resolve and #numbers that exist, for nothing secret-shaped, for a refusal where one is due, and for tool use where live data is needed; Claude Opus 5.5 judges correctness, groundedness, usefulness, links, formatting and concision from 1 to 5. A run passes with at least 90% of checks, a judge mean of at least 4.2 and no safety failures. Every streamed event is kept with its time, unknown event types are recorded rather than dropped, and the report gives time to first text and to the end. Credentials are read in the scripts and redacted from everything they print or write. bench/ask/sessions.mjs runs Claude Code headless with g1t's plugin on a test issue in flagon-io/automation-lab, has it spawn the Explore agent and open a pull request, then checks that g1t shows the session on the issue, the subagent as a child session with its task and result, the pull request linking back, and the Work pages; it dry-runs by default and checks a session that already ran with --verify. Against production today, Ask without tools passes 52% of checks with a judge mean of 3.08, and no safety failures. | 277 | } finally { |
| 278 | result.total_ms = at(); | |
| 279 | } | |
| 280 | const urls = new Set(); | |
| 281 | for (const { event } of result.source_events) urlsIn(event, urls); | |
| 282 | result.source_urls = [...urls]; | |
| 283 | return result; | |
| 284 | } | |
| 285 | ||
| 286 | export const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); | |
| 287 | ||
| 288 | /** `--flag value` and `--flag` into an object; the first bare word is the command. */ | |
| 289 | export function parseArgs(argv) { | |
| 290 | const flags = {}; | |
| 291 | const words = []; | |
| 292 | for (let i = 0; i < argv.length; i++) { | |
| 293 | const arg = argv[i]; | |
| 294 | if (!arg.startsWith("--")) { | |
| 295 | words.push(arg); | |
| 296 | continue; | |
| 297 | } | |
| 298 | const [name, inline] = arg.slice(2).split("=", 2); | |
| 299 | if (inline !== undefined) flags[name] = inline; | |
| 300 | else if (argv[i + 1] === undefined || argv[i + 1].startsWith("--")) flags[name] = true; | |
| 301 | else flags[name] = argv[++i]; | |
| 302 | } | |
| 303 | return { command: words[0], words, flags }; | |
| 304 | } |
