Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
388 linesCodeBlameRaw
1#!/usr/bin/env node
2// Sessions from the user's own harness, end to end: Claude Code, headless,
3// with g1t's plugin, works a real issue in the test repository and spawns a
4// subagent; then g1t must show it: a session linked to the issue, the
5// subagent as a child session with its task and result, the pull request
6// linking back, and the Work pages showing all of it.
7//
8// node bench/ask/sessions.mjs dry run: preflight and the plan; changes nothing
9// node bench/ask/sessions.mjs --live open (or reuse) the test issue, run Claude Code, verify
10// node bench/ask/sessions.mjs --verify --session ses_… [--issue N] [--pull N]
11// only the checks, on a session that already ran
12//
13// Options: --repo flagon-io/automation-lab (the test repository; it must be
14// one meant for tests), --budget 3 (USD cap for Claude Code), --max-turns 40,
15// --installed (use the installed g1t plugin instead of --plugin-dir
16// plugins/claude-code), --keep (keep the scratch clone).
17//
18// Needs: the `claude` CLI; the `g1t` CLI on PATH (or G1T_BIN), which the
19// plugin records through; G1T_TOKEN (or its export in ~/.bashrc); the
20// judge's key file .secrets/anthropic-key.txt, given to Claude Code as
21// ANTHROPIC_API_KEY in its own environment only. Nothing is printed or
22// written with a credential in it: the MCP config names ${G1T_TOKEN} and
23// git gets its header through GIT_CONFIG_* variables of the child.
24//
25// Results land in bench/ask/runs/sessions-<timestamp>/ (git-ignored).
26
27import { spawn, spawnSync } from "node:child_process";
28import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
29import { tmpdir } from "node:os";
30import { delimiter, join } from "node:path";
31
32import { HERE, ROOT, SITE, anthropicKeyValue, api, apiAll, apiTokenValue, loadCredentials, mcp, parseArgs, redact, site, webTokenValue } from "./lib.mjs";
33
34const { flags } = parseArgs(process.argv.slice(2));
35const REPO = String(flags.repo ?? "flagon-io/automation-lab");
36const LIVE = Boolean(flags.live);
37const VERIFY = Boolean(flags.verify);
38const say = (line = "") => console.log(redact(line));
39
40/** Issue the scenario works on: small, real, and only in the test repository. */
41const TASK = {
42 title: "[bench] sessions scenario: add abs() to math.js",
43 body: "Add `abs(x)` to math.js: the absolute value of a number, exported like the other functions, with a test next to the others.\n\nThis issue is opened by g1t's sessions bench (bench/ask/sessions.mjs) to check that sessions, subagents and pull requests show on the Work page. Its pull request can be merged or closed.",
44};
45
46const prompt = (issue) =>
47 [
48 `You are working on issue #${issue} in ${REPO} (this checkout).`,
49 "First use the Agent tool with the Explore agent to find where the math functions and their tests live and how they are exported; wait for its answer.",
50 `Then claim the issue, make the change on a branch named ${issue}-abs, run the tests if there are any, commit, push the branch,`,
51 `and open a pull request on g1t (the g1t MCP server's pull_request tool) that closes #${issue}. Keep the change small. Do not merge it.`,
52 ].join(" ");
53
54// --- preflight ------------------------------------------------------------------
55
56function which(name) {
57 const names = process.platform === "win32" ? [`${name}.exe`, `${name}.cmd`, name] : [name];
58 for (const dir of (process.env.PATH ?? process.env.Path ?? "").split(delimiter)) {
59 for (const n of names) if (dir && existsSync(join(dir, n))) return join(dir, n);
60 }
61 return null;
62}
63
64async function preflight() {
65 const creds = loadCredentials();
66 const rows = [];
67 const add = (name, ok, detail, blocking = true) => rows.push({ name, ok: Boolean(ok), detail, blocking });
68
69 const claude = spawnSync("claude", ["--version"], { encoding: "utf8" });
70 add("claude CLI", claude.status === 0, claude.status === 0 ? claude.stdout.trim() : "not on PATH");
71
72 const g1tBin = process.env.G1T_BIN ?? which("g1t");
73 add(
74 "g1t CLI (the plugin records through it)",
75 g1tBin,
76 g1tBin ?? "not on PATH and no G1T_BIN: install it from g1t.sh/download or `cargo build -p g1t` and set G1T_BIN",
77 );
78
79 const pluginDir = join(ROOT, "plugins", "claude-code");
80 if (flags.installed) {
81 const list = spawnSync("claude", ["plugin", "list"], { encoding: "utf8" });
82 add(
83 "g1t plugin installed",
84 /\bg1t\b/.test(list.stdout ?? ""),
85 "claude plugin marketplace add https://g1t.sh/flagon-io/g1t-plugins.git && claude plugin install g1t@g1t",
86 );
87 } else add("g1t plugin (--plugin-dir)", existsSync(join(pluginDir, ".claude-plugin", "plugin.json")), pluginDir);
88
89 const hooks = existsSync(join(pluginDir, "hooks", "hooks.json")) ? JSON.parse(readFileSync(join(pluginDir, "hooks", "hooks.json"), "utf8")) : {};
90 const subagentHooks = ["SubagentStart", "SubagentStop"].filter((h) => hooks.hooks?.[h]);
91 add(
92 "plugin records subagents (SubagentStart/SubagentStop hooks)",
93 subagentHooks.length === 2,
94 subagentHooks.length ? subagentHooks.join(", ") : "the plugin has no subagent hooks yet",
95 false,
96 );
97
98 add("G1T_TOKEN", creds.api, creds.api ? "found" : "export G1T_TOKEN or put it in ~/.bashrc");
99 add("site token (.credentials/g1t_token.txt)", creds.web, creds.web ? "found" : "missing");
100 add("Anthropic key (.secrets/anthropic-key.txt)", creds.anthropic, creds.anthropic ? "found" : "missing");
101
102 const repo = await api(`/repos/${REPO}`);
103 const description = repo.data?.description ?? repo.data?.repository?.description ?? "";
104 add(
105 `test repository ${REPO}`,
106 repo.status === 200 && /test|scratch|lab/i.test(`${REPO} ${description}`),
107 repo.status === 200 ? `"${description}"` : `API ${repo.status}`,
108 );
109
110 // Does the server know child sessions yet? The session tool's schema says.
111 const schema = await mcpSchema("session");
112 const parentField = schema ? ["parent_id", "parent_session_id", "parent", "agent_id"].find((f) => schema.includes(`"${f}"`)) : null;
113 add(
114 "server: sessions take a parent (child sessions)",
115 parentField,
116 parentField ? `session tool has ${parentField}` : "the session tool has no parent field yet",
117 false,
118 );
119
120 return { rows, ready: rows.filter((r) => r.blocking).every((r) => r.ok), pluginDir, g1tBin };
121}
122
123async function mcpSchema(tool) {
124 try {
125 const response = await fetch("https://mcp.g1t.sh", {
126 method: "POST",
127 headers: {
128 authorization: `Bearer ${apiTokenValue()}`,
129 "content-type": "application/json",
130 accept: "application/json, text/event-stream",
131 "user-agent": "g1t-ask-bench/1",
132 },
133 body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }),
134 });
135 const json = await response.json();
136 const found = json.result?.tools?.find((t) => t.name === tool);
137 return found ? JSON.stringify(found.inputSchema) : null;
138 } catch {
139 return null;
140 }
141}
142
143// --- the run ---------------------------------------------------------------------
144
145async function testIssue() {
146 const all = await apiAll(`/repos/${REPO}/issues?state=open`, 300);
147 const open = all.find((i) => i.title === TASK.title);
148 if (open) return open.number;
149 if (!LIVE) return null;
150 const created = await api(`/repos/${REPO}/issues`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(TASK) });
151 const number = created.data?.number ?? created.data?.issue?.number;
152 if (!number) throw new Error(`could not open the test issue (API ${created.status})`);
153 return number;
154}
155
156function childEnv(extra = {}) {
157 const env = { ...process.env, ...extra };
158 // The judge's key, for this child only; never the parent's own.
159 env.ANTHROPIC_API_KEY = anthropicKeyValue();
160 delete env.ANTHROPIC_AUTH_TOKEN;
161 env.G1T_TOKEN = apiTokenValue();
162 // git's header for g1t.sh, without a credential helper or a file.
163 const user = process.env.G1T_USER ?? "syntaqx";
164 env.GIT_CONFIG_COUNT = "1";
165 env.GIT_CONFIG_KEY_0 = `http.${SITE}/.extraheader`;
166 env.GIT_CONFIG_VALUE_0 = `Authorization: Basic ${Buffer.from(`${user}:${webTokenValue()}`).toString("base64")}`;
167 env.GIT_TERMINAL_PROMPT = "0";
168 env.GCM_INTERACTIVE = "never";
169 return env;
170}
171
172function runClaude({ cwd, issue, pluginDir, out }) {
173 const mcpConfig = join(out, "mcp.json");
174 // The token stays in the environment: Claude Code expands ${G1T_TOKEN}.
175 writeFileSync(
176 mcpConfig,
177 JSON.stringify({ mcpServers: { g1t: { type: "http", url: "https://mcp.g1t.sh", headers: { Authorization: "Bearer ${G1T_TOKEN}" } } } }, null, 2),
178 );
179 const args = [
180 "-p",
181 prompt(issue),
182 "--output-format",
183 "stream-json",
184 "--verbose",
185 "--mcp-config",
186 mcpConfig,
187 "--permission-mode",
188 "acceptEdits",
189 "--allowedTools",
190 "Bash(git:*) Bash(npm:*) Bash(node:*) Bash(g1t:*) Read Edit Write Glob Grep Agent Task mcp__g1t",
191 "--max-budget-usd",
192 String(flags.budget ?? 3),
193 "--max-turns",
194 String(flags["max-turns"] ?? 40),
195 ];
196 if (!flags.installed) args.push("--plugin-dir", pluginDir);
197 return new Promise((done) => {
198 const child = spawn("claude", args, { cwd, env: childEnv(), windowsHide: true });
199 const events = [];
200 let pending = "";
201 child.stdout.on("data", (chunk) => {
202 pending += chunk;
203 const lines = pending.split("\n");
204 pending = lines.pop() ?? "";
205 for (const line of lines) {
206 try {
207 events.push(JSON.parse(line));
208 } catch {}
209 }
210 });
211 let stderr = "";
212 child.stderr.on("data", (c) => (stderr += c));
213 child.on("close", (code) => done({ code, events, stderr: redact(stderr).slice(-4000) }));
214 });
215}
216
217/** What the stream says happened: Claude Code's session id, subagents it ran, the PR it opened. */
218function readStream(events) {
219 const init = events.find((e) => e.type === "system" && e.subtype === "init");
220 const subagents = [];
221 for (const e of events) {
222 for (const block of e.message?.content ?? []) {
223 if (block.type === "tool_use" && (block.name === "Agent" || block.name === "Task")) {
224 subagents.push({
225 id: block.id,
226 type: block.input?.subagent_type,
227 description: block.input?.description,
228 prompt: String(block.input?.prompt ?? "").slice(0, 400),
229 });
230 }
231 }
232 }
233 const result = events.findLast((e) => e.type === "result");
234 return {
235 sessionId: init?.session_id ?? result?.session_id ?? null,
236 subagents,
237 result: result
238 ? { subtype: result.subtype, cost_usd: result.total_cost_usd, turns: result.num_turns, text: String(result.result ?? "").slice(0, 2000) }
239 : null,
240 };
241}
242
243// --- the checks ----------------------------------------------------------------------
244
245async function verify({ sessionId, harnessSessionId, issue, pull, subagents = [] }) {
246 const checks = [];
247 const add = (name, pass, detail) => checks.push({ name, pass: Boolean(pass), detail });
248 const sessions = await mcp("session", { action: "list", repo: REPO }).catch((e) => ({ error: e.message }));
249 const list = Array.isArray(sessions) ? sessions : [];
250 const session = list.find((s) => s.id === sessionId) ?? list.find((s) => harnessSessionId && s.harness_session_id === harnessSessionId) ?? null;
251 add(
252 "a session exists for the run",
253 session,
254 session ? `${session.id} (${session.harness}, ${session.state})` : `none of ${list.length} sessions on ${REPO} is ${sessionId ?? harnessSessionId}`,
255 );
256 if (!session) return checks;
257 issue ??= session.issue;
258 add("the session is linked to the issue", issue && session.issue === issue, `session issue ${session.issue ?? "none"}, expected ${issue ?? "?"}`);
259
260 // Children: whatever the server calls the parent link.
261 const parentOf = (s) => s.parent_id ?? s.parent_session_id ?? s.parent?.id ?? s.parent ?? null;
262 const children = list.filter((s) => parentOf(s) === session.id);
263 add(
264 "subagents appear as child sessions",
265 children.length > 0,
266 children.length
267 ? `${children.length}: ${children.map((c) => c.id).join(", ")}`
268 : `no session names ${session.id} as its parent${list.some((s) => parentOf(s)) ? "" : " (no session has a parent field at all)"}`,
269 );
270 if (subagents.length)
271 add("one child per subagent the run spawned", children.length >= subagents.length, `${children.length} children for ${subagents.length} subagents`);
272 const detail = await mcp("session", { action: "get", repo: REPO, id: session.id }).catch(() => null);
273 for (const child of children) {
274 const task = child.title ?? child.task ?? child.description;
275 const result = child.result ?? child.summary ?? child.end_reason;
276 add(
277 `child ${child.id} has its task and result`,
278 task && result,
279 `task ${task ? `"${String(task).slice(0, 80)}"` : "missing"}; result ${result ? `"${String(result).slice(0, 80)}"` : "missing"}`,
280 );
281 }
282 const entries = detail?.entries ?? [];
283 add(
284 "the session recorded tool calls",
285 entries.some((e) => e.kind === "tool_call"),
286 `${entries.length} entries`,
287 );
288
289 // The pull request, and its link back to the issue and session.
290 let pr = pull ?? session.pull ?? null;
291 if (!pr && issue) {
292 const pulls = await apiAll(`/repos/${REPO}/pulls?state=all`, 100);
293 pr = pulls.find((p) => p.issue === issue || new RegExp(`#${issue}\\b`).test(p.body ?? ""))?.number ?? null;
294 }
295 add("a pull request was opened", pr, pr ? `#${pr}` : "none found for the issue");
296 if (pr) {
297 const { data } = await api(`/repos/${REPO}/pulls/${pr}`);
298 const p = data?.pull ?? data;
299 add("the pull request links back to the issue", p?.issue === issue || new RegExp(`#${issue}\\b`).test(p?.body ?? ""), `issue ${p?.issue ?? "none"}`);
300 add("the session is attached to the pull request", session.pull === pr, `session pull ${session.pull ?? "none"}`);
301 }
302
303 // The Work pages, as the user sees them.
304 const page = async (path) => {
305 const response = await site(path);
306 return { status: response.status, html: response.ok ? await response.text() : "" };
307 };
308 const work = await page(`/${REPO}/work`);
309 add(
310 "the Work page shows the session",
311 work.status === 200 && work.html.includes(session.id),
312 `status ${work.status}${work.html.includes(session.id) ? "" : `, no ${session.id} on the page`}`,
313 );
314 const sessionPage = await page(`/${REPO}/work/sessions/${session.id}`);
315 add("the session's page loads", sessionPage.status === 200, `status ${sessionPage.status}`);
316 for (const child of children)
317 add(`the session's page shows child ${child.id}`, sessionPage.html.includes(child.id) || (child.title && sessionPage.html.includes(child.title)), "");
318 if (issue) {
319 const issuePage = await page(`/${REPO}/issues/${issue}`);
320 add("the issue's Work panel shows the session", issuePage.status === 200 && issuePage.html.includes(session.id), `status ${issuePage.status}`);
321 }
322 return checks;
323}
324
325// --- main ------------------------------------------------------------------------------
326
327async function main() {
328 const out = join(HERE, "runs", `sessions-${new Date().toISOString().replace(/[:.]/g, "-")}`);
329 const pre = await preflight();
330 say(`Sessions scenario on ${REPO} (${VERIFY ? "verify only" : LIVE ? "live" : "dry run"})\n\nPreflight:`);
331 for (const r of pre.rows) say(` ${r.ok ? "ok " : r.blocking ? "FAIL" : "gap "} ${r.name}: ${r.detail}`);
332
333 if (VERIFY) {
334 if (!flags.session) throw new Error("--verify needs --session ses_…");
335 const checks = await verify({
336 sessionId: String(flags.session),
337 issue: flags.issue ? Number(flags.issue) : undefined,
338 pull: flags.pull ? Number(flags.pull) : undefined,
339 });
340 return report(out, { mode: "verify", preflight: pre.rows, checks });
341 }
342
343 const issue = await testIssue();
344 say(`\nTest issue: ${issue ? `#${issue}` : `none open yet ("${TASK.title}"); --live opens it`}`);
345 say(`Prompt: ${prompt(issue ?? "N")}`);
346 if (!LIVE) {
347 say("\nDry run: nothing was changed. Run with --live once the preflight passes.");
348 return;
349 }
350 if (!pre.ready) throw new Error("Preflight failed; see above. Nothing was run.");
351
352 mkdirSync(out, { recursive: true });
353 const scratch = mkdtempSync(join(tmpdir(), "g1t-sessions-"));
354 try {
355 const clone = spawnSync("git", ["clone", "-q", `${SITE}/${REPO}.git`, join(scratch, "repo")], { env: childEnv(), encoding: "utf8" });
356 if (clone.status !== 0) throw new Error(`clone failed: ${redact(clone.stderr).slice(0, 400)}`);
357 say(`Cloned into ${join(scratch, "repo")}; running Claude Code…`);
358 const run = await runClaude({ cwd: join(scratch, "repo"), issue, pluginDir: pre.pluginDir, out });
359 writeFileSync(join(out, "stream.jsonl"), redact(run.events.map((e) => JSON.stringify(e)).join("\n")));
360 const stream = readStream(run.events);
361 say(
362 `Claude Code exited ${run.code}: session ${stream.sessionId}, ${stream.subagents.length} subagents, ${stream.result?.turns ?? "?"} turns, $${stream.result?.cost_usd?.toFixed?.(2) ?? "?"}`,
363 );
364 // The plugin's hooks record in the background: give them a moment.
365 await new Promise((done) => setTimeout(done, 8000));
366 const sessions = await mcp("session", { action: "list", repo: REPO }).catch(() => []);
367 const mine = (Array.isArray(sessions) ? sessions : []).find((s) => s.harness_session_id === stream.sessionId);
368 const checks = await verify({ sessionId: mine?.id, harnessSessionId: stream.sessionId, issue, subagents: stream.subagents });
369 report(out, { mode: "live", preflight: pre.rows, issue, stream, exit: run.code, stderr: run.stderr, checks });
370 } finally {
371 if (!flags.keep) rmSync(scratch, { recursive: true, force: true });
372 }
373}
374
375function report(out, result) {
376 mkdirSync(out, { recursive: true });
377 writeFileSync(join(out, "result.json"), redact(JSON.stringify(result, null, 2)));
378 const passed = result.checks.filter((c) => c.pass).length;
379 say(`\nChecks ${passed}/${result.checks.length}:`);
380 for (const c of result.checks) say(` ${c.pass ? "ok " : "FAIL"} ${c.name}${c.detail ? `: ${c.detail}` : ""}`);
381 say(`\nSaved ${join(out, "result.json")}`);
382 process.exitCode = passed === result.checks.length ? 0 : 1;
383}
384
385main().catch((error) => {
386 console.error(redact(error?.stack ?? String(error)));
387 process.exit(1);
388});