| 1 | #!/usr/bin/env node |
| 2 | // Sessions from the user's own harness, end to end: Claude Code, headless, |
| 3 | // with g1t's plugin, works a real issue in the test repository and spawns a |
| 4 | // subagent; then g1t must show it: a session linked to the issue, the |
| 5 | // subagent as a child session with its task and result, the pull request |
| 6 | // linking back, and the Work pages showing all of it. |
| 7 | // |
| 8 | // node bench/ask/sessions.mjs dry run: preflight and the plan; changes nothing |
| 9 | // node bench/ask/sessions.mjs --live open (or reuse) the test issue, run Claude Code, verify |
| 10 | // node bench/ask/sessions.mjs --verify --session ses_… [--issue N] [--pull N] |
| 11 | // only the checks, on a session that already ran |
| 12 | // |
| 13 | // Options: --repo flagon-io/automation-lab (the test repository; it must be |
| 14 | // one meant for tests), --budget 3 (USD cap for Claude Code), --max-turns 40, |
| 15 | // --installed (use the installed g1t plugin instead of --plugin-dir |
| 16 | // plugins/claude-code), --keep (keep the scratch clone). |
| 17 | // |
| 18 | // Needs: the `claude` CLI; the `g1t` CLI on PATH (or G1T_BIN), which the |
| 19 | // plugin records through; G1T_TOKEN (or its export in ~/.bashrc); the |
| 20 | // judge's key file .secrets/anthropic-key.txt, given to Claude Code as |
| 21 | // ANTHROPIC_API_KEY in its own environment only. Nothing is printed or |
| 22 | // written with a credential in it: the MCP config names ${G1T_TOKEN} and |
| 23 | // git gets its header through GIT_CONFIG_* variables of the child. |
| 24 | // |
| 25 | // Results land in bench/ask/runs/sessions-<timestamp>/ (git-ignored). |
| 26 | |
| 27 | import { spawn, spawnSync } from "node:child_process"; |
| 28 | import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 29 | import { tmpdir } from "node:os"; |
| 30 | import { delimiter, join } from "node:path"; |
| 31 | |
| 32 | import { HERE, ROOT, SITE, anthropicKeyValue, api, apiAll, apiTokenValue, loadCredentials, mcp, parseArgs, redact, site, webTokenValue } from "./lib.mjs"; |
| 33 | |
| 34 | const { flags } = parseArgs(process.argv.slice(2)); |
| 35 | const REPO = String(flags.repo ?? "flagon-io/automation-lab"); |
| 36 | const LIVE = Boolean(flags.live); |
| 37 | const VERIFY = Boolean(flags.verify); |
| 38 | const say = (line = "") => console.log(redact(line)); |
| 39 | |
| 40 | /** Issue the scenario works on: small, real, and only in the test repository. */ |
| 41 | const TASK = { |
| 42 | title: "[bench] sessions scenario: add abs() to math.js", |
| 43 | body: "Add `abs(x)` to math.js: the absolute value of a number, exported like the other functions, with a test next to the others.\n\nThis issue is opened by g1t's sessions bench (bench/ask/sessions.mjs) to check that sessions, subagents and pull requests show on the Work page. Its pull request can be merged or closed.", |
| 44 | }; |
| 45 | |
| 46 | const prompt = (issue) => |
| 47 | [ |
| 48 | `You are working on issue #${issue} in ${REPO} (this checkout).`, |
| 49 | "First use the Agent tool with the Explore agent to find where the math functions and their tests live and how they are exported; wait for its answer.", |
| 50 | `Then claim the issue, make the change on a branch named ${issue}-abs, run the tests if there are any, commit, push the branch,`, |
| 51 | `and open a pull request on g1t (the g1t MCP server's pull_request tool) that closes #${issue}. Keep the change small. Do not merge it.`, |
| 52 | ].join(" "); |
| 53 | |
| 54 | // --- preflight ------------------------------------------------------------------ |
| 55 | |
| 56 | function which(name) { |
| 57 | const names = process.platform === "win32" ? [`${name}.exe`, `${name}.cmd`, name] : [name]; |
| 58 | for (const dir of (process.env.PATH ?? process.env.Path ?? "").split(delimiter)) { |
| 59 | for (const n of names) if (dir && existsSync(join(dir, n))) return join(dir, n); |
| 60 | } |
| 61 | return null; |
| 62 | } |
| 63 | |
| 64 | async function preflight() { |
| 65 | const creds = loadCredentials(); |
| 66 | const rows = []; |
| 67 | const add = (name, ok, detail, blocking = true) => rows.push({ name, ok: Boolean(ok), detail, blocking }); |
| 68 | |
| 69 | const claude = spawnSync("claude", ["--version"], { encoding: "utf8" }); |
| 70 | add("claude CLI", claude.status === 0, claude.status === 0 ? claude.stdout.trim() : "not on PATH"); |
| 71 | |
| 72 | const g1tBin = process.env.G1T_BIN ?? which("g1t"); |
| 73 | add("g1t CLI (the plugin records through it)", g1tBin, g1tBin ?? "not on PATH and no G1T_BIN: install it from g1t.sh/download or `cargo build -p g1t` and set G1T_BIN"); |
| 74 | |
| 75 | const pluginDir = join(ROOT, "plugins", "claude-code"); |
| 76 | if (flags.installed) { |
| 77 | const list = spawnSync("claude", ["plugin", "list"], { encoding: "utf8" }); |
| 78 | add("g1t plugin installed", /\bg1t\b/.test(list.stdout ?? ""), "claude plugin marketplace add https://g1t.sh/flagon-io/g1t.git && claude plugin install g1t@g1t"); |
| 79 | } else add("g1t plugin (--plugin-dir)", existsSync(join(pluginDir, ".claude-plugin", "plugin.json")), pluginDir); |
| 80 | |
| 81 | const hooks = existsSync(join(pluginDir, "hooks", "hooks.json")) ? JSON.parse(readFileSync(join(pluginDir, "hooks", "hooks.json"), "utf8")) : {}; |
| 82 | const subagentHooks = ["SubagentStart", "SubagentStop"].filter((h) => hooks.hooks?.[h]); |
| 83 | add("plugin records subagents (SubagentStart/SubagentStop hooks)", subagentHooks.length === 2, subagentHooks.length ? subagentHooks.join(", ") : "the plugin has no subagent hooks yet", false); |
| 84 | |
| 85 | add("G1T_TOKEN", creds.api, creds.api ? "found" : "export G1T_TOKEN or put it in ~/.bashrc"); |
| 86 | add("site token (.credentials/g1t_token.txt)", creds.web, creds.web ? "found" : "missing"); |
| 87 | add("Anthropic key (.secrets/anthropic-key.txt)", creds.anthropic, creds.anthropic ? "found" : "missing"); |
| 88 | |
| 89 | const repo = await api(`/repos/${REPO}`); |
| 90 | const description = repo.data?.description ?? repo.data?.repository?.description ?? ""; |
| 91 | add(`test repository ${REPO}`, repo.status === 200 && /test|scratch|lab/i.test(`${REPO} ${description}`), repo.status === 200 ? `"${description}"` : `API ${repo.status}`); |
| 92 | |
| 93 | // Does the server know child sessions yet? The session tool's schema says. |
| 94 | const schema = await mcpSchema("session"); |
| 95 | const parentField = schema ? ["parent_id", "parent_session_id", "parent", "agent_id"].find((f) => schema.includes(`"${f}"`)) : null; |
| 96 | add("server: sessions take a parent (child sessions)", parentField, parentField ? `session tool has ${parentField}` : "the session tool has no parent field yet", false); |
| 97 | |
| 98 | return { rows, ready: rows.filter((r) => r.blocking).every((r) => r.ok), pluginDir, g1tBin }; |
| 99 | } |
| 100 | |
| 101 | async function mcpSchema(tool) { |
| 102 | try { |
| 103 | const response = await fetch("https://mcp.g1t.sh", { |
| 104 | method: "POST", |
| 105 | headers: { authorization: `Bearer ${apiTokenValue()}`, "content-type": "application/json", accept: "application/json, text/event-stream", "user-agent": "g1t-ask-bench/1" }, |
| 106 | body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }), |
| 107 | }); |
| 108 | const json = await response.json(); |
| 109 | const found = json.result?.tools?.find((t) => t.name === tool); |
| 110 | return found ? JSON.stringify(found.inputSchema) : null; |
| 111 | } catch { |
| 112 | return null; |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // --- the run --------------------------------------------------------------------- |
| 117 | |
| 118 | async function testIssue() { |
| 119 | const all = await apiAll(`/repos/${REPO}/issues?state=open`, 300); |
| 120 | const open = all.find((i) => i.title === TASK.title); |
| 121 | if (open) return open.number; |
| 122 | if (!LIVE) return null; |
| 123 | const created = await api(`/repos/${REPO}/issues`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(TASK) }); |
| 124 | const number = created.data?.number ?? created.data?.issue?.number; |
| 125 | if (!number) throw new Error(`could not open the test issue (API ${created.status})`); |
| 126 | return number; |
| 127 | } |
| 128 | |
| 129 | function childEnv(extra = {}) { |
| 130 | const env = { ...process.env, ...extra }; |
| 131 | // The judge's key, for this child only; never the parent's own. |
| 132 | env.ANTHROPIC_API_KEY = anthropicKeyValue(); |
| 133 | delete env.ANTHROPIC_AUTH_TOKEN; |
| 134 | env.G1T_TOKEN = apiTokenValue(); |
| 135 | // git's header for g1t.sh, without a credential helper or a file. |
| 136 | const user = process.env.G1T_USER ?? "syntaqx"; |
| 137 | env.GIT_CONFIG_COUNT = "1"; |
| 138 | env.GIT_CONFIG_KEY_0 = `http.${SITE}/.extraheader`; |
| 139 | env.GIT_CONFIG_VALUE_0 = `Authorization: Basic ${Buffer.from(`${user}:${webTokenValue()}`).toString("base64")}`; |
| 140 | env.GIT_TERMINAL_PROMPT = "0"; |
| 141 | env.GCM_INTERACTIVE = "never"; |
| 142 | return env; |
| 143 | } |
| 144 | |
| 145 | function runClaude({ cwd, issue, pluginDir, out }) { |
| 146 | const mcpConfig = join(out, "mcp.json"); |
| 147 | // The token stays in the environment: Claude Code expands ${G1T_TOKEN}. |
| 148 | writeFileSync(mcpConfig, JSON.stringify({ mcpServers: { g1t: { type: "http", url: "https://mcp.g1t.sh", headers: { Authorization: "Bearer ${G1T_TOKEN}" } } } }, null, 2)); |
| 149 | const args = [ |
| 150 | "-p", |
| 151 | prompt(issue), |
| 152 | "--output-format", |
| 153 | "stream-json", |
| 154 | "--verbose", |
| 155 | "--mcp-config", |
| 156 | mcpConfig, |
| 157 | "--permission-mode", |
| 158 | "acceptEdits", |
| 159 | "--allowedTools", |
| 160 | "Bash(git:*) Bash(npm:*) Bash(node:*) Bash(g1t:*) Read Edit Write Glob Grep Agent Task mcp__g1t", |
| 161 | "--max-budget-usd", |
| 162 | String(flags.budget ?? 3), |
| 163 | "--max-turns", |
| 164 | String(flags["max-turns"] ?? 40), |
| 165 | ]; |
| 166 | if (!flags.installed) args.push("--plugin-dir", pluginDir); |
| 167 | return new Promise((done) => { |
| 168 | const child = spawn("claude", args, { cwd, env: childEnv(), windowsHide: true }); |
| 169 | const events = []; |
| 170 | let pending = ""; |
| 171 | child.stdout.on("data", (chunk) => { |
| 172 | pending += chunk; |
| 173 | const lines = pending.split("\n"); |
| 174 | pending = lines.pop() ?? ""; |
| 175 | for (const line of lines) { |
| 176 | try { |
| 177 | events.push(JSON.parse(line)); |
| 178 | } catch {} |
| 179 | } |
| 180 | }); |
| 181 | let stderr = ""; |
| 182 | child.stderr.on("data", (c) => (stderr += c)); |
| 183 | child.on("close", (code) => done({ code, events, stderr: redact(stderr).slice(-4000) })); |
| 184 | }); |
| 185 | } |
| 186 | |
| 187 | /** What the stream says happened: Claude Code's session id, subagents it ran, the PR it opened. */ |
| 188 | function readStream(events) { |
| 189 | const init = events.find((e) => e.type === "system" && e.subtype === "init"); |
| 190 | const subagents = []; |
| 191 | for (const e of events) { |
| 192 | for (const block of e.message?.content ?? []) { |
| 193 | if (block.type === "tool_use" && (block.name === "Agent" || block.name === "Task")) { |
| 194 | subagents.push({ id: block.id, type: block.input?.subagent_type, description: block.input?.description, prompt: String(block.input?.prompt ?? "").slice(0, 400) }); |
| 195 | } |
| 196 | } |
| 197 | } |
| 198 | const result = events.findLast((e) => e.type === "result"); |
| 199 | return { sessionId: init?.session_id ?? result?.session_id ?? null, subagents, result: result ? { subtype: result.subtype, cost_usd: result.total_cost_usd, turns: result.num_turns, text: String(result.result ?? "").slice(0, 2000) } : null }; |
| 200 | } |
| 201 | |
| 202 | // --- the checks ---------------------------------------------------------------------- |
| 203 | |
| 204 | async function verify({ sessionId, harnessSessionId, issue, pull, subagents = [] }) { |
| 205 | const checks = []; |
| 206 | const add = (name, pass, detail) => checks.push({ name, pass: Boolean(pass), detail }); |
| 207 | const sessions = await mcp("session", { action: "list", repo: REPO }).catch((e) => ({ error: e.message })); |
| 208 | const list = Array.isArray(sessions) ? sessions : []; |
| 209 | const session = list.find((s) => s.id === sessionId) ?? list.find((s) => harnessSessionId && s.harness_session_id === harnessSessionId) ?? null; |
| 210 | add("a session exists for the run", session, session ? `${session.id} (${session.harness}, ${session.state})` : `none of ${list.length} sessions on ${REPO} is ${sessionId ?? harnessSessionId}`); |
| 211 | if (!session) return checks; |
| 212 | issue ??= session.issue; |
| 213 | add("the session is linked to the issue", issue && session.issue === issue, `session issue ${session.issue ?? "none"}, expected ${issue ?? "?"}`); |
| 214 | |
| 215 | // Children: whatever the server calls the parent link. |
| 216 | const parentOf = (s) => s.parent_id ?? s.parent_session_id ?? s.parent?.id ?? s.parent ?? null; |
| 217 | const children = list.filter((s) => parentOf(s) === session.id); |
| 218 | add("subagents appear as child sessions", children.length > 0, children.length ? `${children.length}: ${children.map((c) => c.id).join(", ")}` : `no session names ${session.id} as its parent${list.some((s) => parentOf(s)) ? "" : " (no session has a parent field at all)"}`); |
| 219 | if (subagents.length) add("one child per subagent the run spawned", children.length >= subagents.length, `${children.length} children for ${subagents.length} subagents`); |
| 220 | const detail = await mcp("session", { action: "get", repo: REPO, id: session.id }).catch(() => null); |
| 221 | for (const child of children) { |
| 222 | const task = child.title ?? child.task ?? child.description; |
| 223 | const result = child.result ?? child.summary ?? child.end_reason; |
| 224 | add(`child ${child.id} has its task and result`, task && result, `task ${task ? `"${String(task).slice(0, 80)}"` : "missing"}; result ${result ? `"${String(result).slice(0, 80)}"` : "missing"}`); |
| 225 | } |
| 226 | const entries = detail?.entries ?? []; |
| 227 | add("the session recorded tool calls", entries.some((e) => e.kind === "tool_call"), `${entries.length} entries`); |
| 228 | |
| 229 | // The pull request, and its link back to the issue and session. |
| 230 | let pr = pull ?? session.pull ?? null; |
| 231 | if (!pr && issue) { |
| 232 | const pulls = await apiAll(`/repos/${REPO}/pulls?state=all`, 100); |
| 233 | pr = pulls.find((p) => p.issue === issue || new RegExp(`#${issue}\\b`).test(p.body ?? ""))?.number ?? null; |
| 234 | } |
| 235 | add("a pull request was opened", pr, pr ? `#${pr}` : "none found for the issue"); |
| 236 | if (pr) { |
| 237 | const { data } = await api(`/repos/${REPO}/pulls/${pr}`); |
| 238 | const p = data?.pull ?? data; |
| 239 | add("the pull request links back to the issue", p?.issue === issue || new RegExp(`#${issue}\\b`).test(p?.body ?? ""), `issue ${p?.issue ?? "none"}`); |
| 240 | add("the session is attached to the pull request", session.pull === pr, `session pull ${session.pull ?? "none"}`); |
| 241 | } |
| 242 | |
| 243 | // The Work pages, as the user sees them. |
| 244 | const page = async (path) => { |
| 245 | const response = await site(path); |
| 246 | return { status: response.status, html: response.ok ? await response.text() : "" }; |
| 247 | }; |
| 248 | const work = await page(`/${REPO}/work`); |
| 249 | add("the Work page shows the session", work.status === 200 && work.html.includes(session.id), `status ${work.status}${work.html.includes(session.id) ? "" : `, no ${session.id} on the page`}`); |
| 250 | const sessionPage = await page(`/${REPO}/work/sessions/${session.id}`); |
| 251 | add("the session's page loads", sessionPage.status === 200, `status ${sessionPage.status}`); |
| 252 | for (const child of children) add(`the session's page shows child ${child.id}`, sessionPage.html.includes(child.id) || (child.title && sessionPage.html.includes(child.title)), ""); |
| 253 | if (issue) { |
| 254 | const issuePage = await page(`/${REPO}/issues/${issue}`); |
| 255 | add("the issue's Work panel shows the session", issuePage.status === 200 && issuePage.html.includes(session.id), `status ${issuePage.status}`); |
| 256 | } |
| 257 | return checks; |
| 258 | } |
| 259 | |
| 260 | // --- main ------------------------------------------------------------------------------ |
| 261 | |
| 262 | async function main() { |
| 263 | const out = join(HERE, "runs", `sessions-${new Date().toISOString().replace(/[:.]/g, "-")}`); |
| 264 | const pre = await preflight(); |
| 265 | say(`Sessions scenario on ${REPO} (${VERIFY ? "verify only" : LIVE ? "live" : "dry run"})\n\nPreflight:`); |
| 266 | for (const r of pre.rows) say(` ${r.ok ? "ok " : r.blocking ? "FAIL" : "gap "} ${r.name}: ${r.detail}`); |
| 267 | |
| 268 | if (VERIFY) { |
| 269 | if (!flags.session) throw new Error("--verify needs --session ses_…"); |
| 270 | const checks = await verify({ sessionId: String(flags.session), issue: flags.issue ? Number(flags.issue) : undefined, pull: flags.pull ? Number(flags.pull) : undefined }); |
| 271 | return report(out, { mode: "verify", preflight: pre.rows, checks }); |
| 272 | } |
| 273 | |
| 274 | const issue = await testIssue(); |
| 275 | say(`\nTest issue: ${issue ? `#${issue}` : `none open yet ("${TASK.title}"); --live opens it`}`); |
| 276 | say(`Prompt: ${prompt(issue ?? "N")}`); |
| 277 | if (!LIVE) { |
| 278 | say("\nDry run: nothing was changed. Run with --live once the preflight passes."); |
| 279 | return; |
| 280 | } |
| 281 | if (!pre.ready) throw new Error("Preflight failed; see above. Nothing was run."); |
| 282 | |
| 283 | mkdirSync(out, { recursive: true }); |
| 284 | const scratch = mkdtempSync(join(tmpdir(), "g1t-sessions-")); |
| 285 | try { |
| 286 | const clone = spawnSync("git", ["clone", "-q", `${SITE}/${REPO}.git`, join(scratch, "repo")], { env: childEnv(), encoding: "utf8" }); |
| 287 | if (clone.status !== 0) throw new Error(`clone failed: ${redact(clone.stderr).slice(0, 400)}`); |
| 288 | say(`Cloned into ${join(scratch, "repo")}; running Claude Code…`); |
| 289 | const run = await runClaude({ cwd: join(scratch, "repo"), issue, pluginDir: pre.pluginDir, out }); |
| 290 | writeFileSync(join(out, "stream.jsonl"), redact(run.events.map((e) => JSON.stringify(e)).join("\n"))); |
| 291 | const stream = readStream(run.events); |
| 292 | say(`Claude Code exited ${run.code}: session ${stream.sessionId}, ${stream.subagents.length} subagents, ${stream.result?.turns ?? "?"} turns, $${stream.result?.cost_usd?.toFixed?.(2) ?? "?"}`); |
| 293 | // The plugin's hooks record in the background: give them a moment. |
| 294 | await new Promise((done) => setTimeout(done, 8000)); |
| 295 | const sessions = await mcp("session", { action: "list", repo: REPO }).catch(() => []); |
| 296 | const mine = (Array.isArray(sessions) ? sessions : []).find((s) => s.harness_session_id === stream.sessionId); |
| 297 | const checks = await verify({ sessionId: mine?.id, harnessSessionId: stream.sessionId, issue, subagents: stream.subagents }); |
| 298 | report(out, { mode: "live", preflight: pre.rows, issue, stream, exit: run.code, stderr: run.stderr, checks }); |
| 299 | } finally { |
| 300 | if (!flags.keep) rmSync(scratch, { recursive: true, force: true }); |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | function report(out, result) { |
| 305 | mkdirSync(out, { recursive: true }); |
| 306 | writeFileSync(join(out, "result.json"), redact(JSON.stringify(result, null, 2))); |
| 307 | const passed = result.checks.filter((c) => c.pass).length; |
| 308 | say(`\nChecks ${passed}/${result.checks.length}:`); |
| 309 | for (const c of result.checks) say(` ${c.pass ? "ok " : "FAIL"} ${c.name}${c.detail ? `: ${c.detail}` : ""}`); |
| 310 | say(`\nSaved ${join(out, "result.json")}`); |
| 311 | process.exitCode = passed === result.checks.length ? 0 : 1; |
| 312 | } |
| 313 | |
| 314 | main().catch((error) => { |
| 315 | console.error(redact(error?.stack ?? String(error))); |
| 316 | process.exit(1); |
| 317 | }); |