g1t/bench/reviewbench/agent/agent.mjs
Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Research: shipping CSS, and a ReviewBench harness for g1t's reviewer | 1 | // g1t's pull request reviewer behind the ReviewBench agent contract |
| 2 | // (https://github.com/review-bench/ReviewBench/blob/main/AGENT_CONTRACT.md). | |
| 3 | // | |
| 4 | // Runs inside the image built from bench/reviewbench/Dockerfile, which is | |
| 5 | // the g1t sandbox's base image (same Claude Code CLI, same toolchains). | |
| 6 | // It does what crates/runner/src/review.rs does after its clone: write the | |
| 7 | // change to /work/change.diff, run Claude Code headless with the same | |
| 8 | // flags as crates/runner/src/harness.rs, read /work/review.json, and apply | |
| 9 | // the same filters services/work/src/reviews.rs applies when it records a | |
| 10 | // review. Then it writes the review's line comments as findings. | |
| 11 | // | |
| 12 | // Configuration (all optional): | |
| 13 | // ANTHROPIC_MODEL or RB_CONFIG_MODEL model id (production review route: claude-sonnet-5-5) | |
| 14 | // ANTHROPIC_BASE_URL g1t's model proxy (MODELS_URL/anthropic) or a gateway | |
| 15 | // ANTHROPIC_API_KEY provider key, or a model-proxy session token | |
| 16 | // RB_CONFIG_BUDGET_USD per-PR cost cap (--max-budget-usd), default 5 | |
| 17 | // RB_CONFIG_VARIANT prompt variant: "production" (default) or a file in variants/ | |
| 18 | ||
| 19 | import { spawn } from "node:child_process"; | |
| 20 | import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs"; | |
| 21 | ||
| 22 | import { buildPrompt } from "./prompt-lib.mjs"; | |
| 23 | ||
| 24 | const REPO = "/work/repo"; | |
| 25 | const DIFF_FILE = "/work/change.diff"; | |
| 26 | const REVIEW_FILE = "/work/review.json"; | |
| 27 | const MAX_TURNS = "80"; // harness.rs | |
| 28 | const MAX_REVIEW_COMMENTS = 30; // reviews.rs | |
| 29 | const MAX_REVIEW_CHARS = 20_000; // reviews.rs | |
| 30 | ||
| 31 | const env = (name, fallback) => process.env[name] ?? fallback; | |
| 32 | const out = env("RB_OUT", "/work/out/findings.json"); | |
| g1t is one name: its agent's work, commits and comments show as @g1t, and nobody can claim g1t or g1t-agent | 33 | const agent = env("RB_AGENT", "g1t"); |
| Research: shipping CSS, and a ReviewBench harness for g1t's reviewer | 34 | const pr = JSON.parse(readFileSync(env("RB_PR_JSON", "/work/pr/pr.json"), "utf8")); |
| 35 | const model = env("RB_CONFIG_MODEL", env("ANTHROPIC_MODEL", "claude-sonnet-5-5")); | |
| 36 | const budget = Number(env("RB_CONFIG_BUDGET_USD", "5")); | |
| 37 | const variant = env("RB_CONFIG_VARIANT", "production"); | |
| 38 | ||
| 39 | // The run's settings, printed so ReviewBench can see each label took effect. | |
| 40 | console.log(`model=${model} budget_usd=${budget} variant=${variant}`); | |
| 41 | ||
| 42 | function write(findings, usage) { | |
| 43 | const body = { | |
| 44 | pr: { repo: pr.repo ?? `https://github.com/${env("RB_NWO")}`, pr_number: Number(env("RB_PR_NUMBER", pr.pr_number)), base: env("RB_BASE", pr.base), head: env("RB_HEAD", pr.head) }, | |
| 45 | agent, | |
| 46 | findings, | |
| 47 | usage, | |
| 48 | }; | |
| 49 | writeFileSync(out, JSON.stringify(body, null, 2)); | |
| 50 | } | |
| 51 | ||
| 52 | /** Paths the change touches, as the work service knows them (pull.files). */ | |
| 53 | function changedPaths(diff) { | |
| 54 | const paths = new Set(); | |
| 55 | for (const line of diff.split("\n")) { | |
| 56 | const m = line.match(/^\+\+\+ b\/(.+)$/) ?? line.match(/^--- a\/(.+)$/); | |
| 57 | if (m && m[1] !== "/dev/null") paths.add(m[1]); | |
| 58 | } | |
| 59 | return paths; | |
| 60 | } | |
| 61 | ||
| 62 | function runClaude(prompt) { | |
| 63 | return new Promise((resolve, reject) => { | |
| 64 | const args = ["--print", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", MAX_TURNS, "--dangerously-skip-permissions", "--model", model]; | |
| 65 | if (budget > 0) args.push("--max-budget-usd", budget.toFixed(2)); | |
| 66 | const child = spawn("claude", args, { cwd: REPO, stdio: ["ignore", "pipe", "inherit"], env: { ...process.env, ANTHROPIC_MODEL: model } }); | |
| 67 | let result = null; | |
| 68 | let buffer = ""; | |
| 69 | child.stdout.on("data", (chunk) => { | |
| 70 | buffer += chunk; | |
| 71 | let at; | |
| 72 | while ((at = buffer.indexOf("\n")) >= 0) { | |
| 73 | const line = buffer.slice(0, at); | |
| 74 | buffer = buffer.slice(at + 1); | |
| 75 | try { | |
| 76 | const event = JSON.parse(line); | |
| 77 | if (event.type === "result") result = event; | |
| 78 | } catch {} | |
| 79 | } | |
| 80 | }); | |
| 81 | child.on("error", reject); | |
| 82 | child.on("close", (code) => (result ? resolve(result) : reject(new Error(`claude exited ${code} without a result`)))); | |
| 83 | }); | |
| 84 | } | |
| 85 | ||
| 86 | const started = Date.now(); | |
| 87 | copyFileSync(env("RB_DIFF", "/work/pr/diff.patch"), DIFF_FILE); | |
| 88 | const diff = readFileSync(DIFF_FILE, "utf8"); | |
| 89 | if (!diff.trim()) { | |
| 90 | // review.rs bails on an empty change; the benchmark still wants a file. | |
| 91 | write([], { time_in_ms: Date.now() - started }); | |
| 92 | process.exit(0); | |
| 93 | } | |
| 94 | ||
| 95 | const instructionsFile = variant === "production" ? new URL("./instructions.txt", import.meta.url) : new URL(`./variants/${variant}.txt`, import.meta.url); | |
| 96 | const prompt = buildPrompt(pr, readFileSync(instructionsFile, "utf8")); | |
| 97 | const result = await runClaude(prompt); | |
| 98 | ||
| 99 | let review = null; | |
| 100 | if (existsSync(REVIEW_FILE)) { | |
| 101 | try { | |
| 102 | review = JSON.parse(readFileSync(REVIEW_FILE, "utf8")); | |
| 103 | } catch {} | |
| 104 | } | |
| 105 | // review.rs: no file means the agent's answer is the review, with no line comments. | |
| 106 | const comments = Array.isArray(review?.comments) ? review.comments : []; | |
| 107 | const known = changedPaths(diff); | |
| 108 | const findings = comments | |
| 109 | .filter((c) => typeof c?.body === "string" && c.body.trim() && typeof c.path === "string") | |
| 110 | // reviews.rs: a line in a file the pull request does not change is dropped. | |
| 111 | .filter((c) => known.size === 0 || known.has(c.path.replace(/^\.\//, ""))) | |
| 112 | .slice(0, MAX_REVIEW_COMMENTS) | |
| 113 | .map((c) => { | |
| 114 | const line = Number.isInteger(c.line) && c.line > 0 ? c.line : 1; | |
| 115 | const end = Number.isInteger(c.end_line) && c.end_line >= line ? c.end_line : line; | |
| 116 | return { file: c.path.replace(/^\.\//, ""), start_line: line, end_line: end, message: c.body.trim().slice(0, MAX_REVIEW_CHARS), producer: agent }; | |
| 117 | }); | |
| 118 | ||
| 119 | // Kept beside the findings for analysis: what production would also have | |
| 120 | // posted (verdict, summary) and what the run cost. | |
| 121 | writeFileSync(out.replace(/\.json$/, ".g1t.json"), JSON.stringify({ verdict: review?.verdict ?? null, body: review?.body ?? result.result ?? "", dropped: comments.length - findings.length, cost_usd: result.total_cost_usd, turns: result.num_turns, model }, null, 2)); | |
| 122 | write(findings, { time_in_ms: Date.now() - started, ...(typeof result.total_cost_usd === "number" ? { cost_usd: result.total_cost_usd } : {}) }); |