Sign in

flagon-io/g1t

public

g1t/bench/reviewbench/agent/agent.mjs

122 lines5,925 bytesCodeBlame
1// g1t's pull request reviewer behind the ReviewBench agent contract
2// (https://github.com/review-bench/ReviewBench/blob/main/AGENT_CONTRACT.md).
3//
4// Runs inside the image built from bench/reviewbench/Dockerfile, which is
5// the g1t sandbox's base image (same Claude Code CLI, same toolchains).
6// It does what crates/runner/src/review.rs does after its clone: write the
7// change to /work/change.diff, run Claude Code headless with the same
8// flags as crates/runner/src/harness.rs, read /work/review.json, and apply
9// the same filters services/work/src/reviews.rs applies when it records a
10// review. Then it writes the review's line comments as findings.
11//
12// Configuration (all optional):
13// ANTHROPIC_MODEL or RB_CONFIG_MODEL model id (production review route: claude-sonnet-5-5)
14// ANTHROPIC_BASE_URL g1t's model proxy (MODELS_URL/anthropic) or a gateway
15// ANTHROPIC_API_KEY provider key, or a model-proxy session token
16// RB_CONFIG_BUDGET_USD per-PR cost cap (--max-budget-usd), default 5
17// RB_CONFIG_VARIANT prompt variant: "production" (default) or a file in variants/
18
19import { spawn } from "node:child_process";
20import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
21
22import { buildPrompt } from "./prompt-lib.mjs";
23
24const REPO = "/work/repo";
25const DIFF_FILE = "/work/change.diff";
26const REVIEW_FILE = "/work/review.json";
27const MAX_TURNS = "80"; // harness.rs
28const MAX_REVIEW_COMMENTS = 30; // reviews.rs
29const MAX_REVIEW_CHARS = 20_000; // reviews.rs
30
31const env = (name, fallback) => process.env[name] ?? fallback;
32const out = env("RB_OUT", "/work/out/findings.json");
33const agent = env("RB_AGENT", "g1t-agent");
34const pr = JSON.parse(readFileSync(env("RB_PR_JSON", "/work/pr/pr.json"), "utf8"));
35const model = env("RB_CONFIG_MODEL", env("ANTHROPIC_MODEL", "claude-sonnet-5-5"));
36const budget = Number(env("RB_CONFIG_BUDGET_USD", "5"));
37const variant = env("RB_CONFIG_VARIANT", "production");
38
39// The run's settings, printed so ReviewBench can see each label took effect.
40console.log(`model=${model} budget_usd=${budget} variant=${variant}`);
41
42function write(findings, usage) {
43 const body = {
44 pr: { repo: pr.repo ?? `https://github.com/${env("RB_NWO")}`, pr_number: Number(env("RB_PR_NUMBER", pr.pr_number)), base: env("RB_BASE", pr.base), head: env("RB_HEAD", pr.head) },
45 agent,
46 findings,
47 usage,
48 };
49 writeFileSync(out, JSON.stringify(body, null, 2));
50}
51
52/** Paths the change touches, as the work service knows them (pull.files). */
53function changedPaths(diff) {
54 const paths = new Set();
55 for (const line of diff.split("\n")) {
56 const m = line.match(/^\+\+\+ b\/(.+)$/) ?? line.match(/^--- a\/(.+)$/);
57 if (m && m[1] !== "/dev/null") paths.add(m[1]);
58 }
59 return paths;
60}
61
62function runClaude(prompt) {
63 return new Promise((resolve, reject) => {
64 const args = ["--print", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", MAX_TURNS, "--dangerously-skip-permissions", "--model", model];
65 if (budget > 0) args.push("--max-budget-usd", budget.toFixed(2));
66 const child = spawn("claude", args, { cwd: REPO, stdio: ["ignore", "pipe", "inherit"], env: { ...process.env, ANTHROPIC_MODEL: model } });
67 let result = null;
68 let buffer = "";
69 child.stdout.on("data", (chunk) => {
70 buffer += chunk;
71 let at;
72 while ((at = buffer.indexOf("\n")) >= 0) {
73 const line = buffer.slice(0, at);
74 buffer = buffer.slice(at + 1);
75 try {
76 const event = JSON.parse(line);
77 if (event.type === "result") result = event;
78 } catch {}
79 }
80 });
81 child.on("error", reject);
82 child.on("close", (code) => (result ? resolve(result) : reject(new Error(`claude exited ${code} without a result`))));
83 });
84}
85
86const started = Date.now();
87copyFileSync(env("RB_DIFF", "/work/pr/diff.patch"), DIFF_FILE);
88const diff = readFileSync(DIFF_FILE, "utf8");
89if (!diff.trim()) {
90 // review.rs bails on an empty change; the benchmark still wants a file.
91 write([], { time_in_ms: Date.now() - started });
92 process.exit(0);
93}
94
95const instructionsFile = variant === "production" ? new URL("./instructions.txt", import.meta.url) : new URL(`./variants/${variant}.txt`, import.meta.url);
96const prompt = buildPrompt(pr, readFileSync(instructionsFile, "utf8"));
97const result = await runClaude(prompt);
98
99let review = null;
100if (existsSync(REVIEW_FILE)) {
101 try {
102 review = JSON.parse(readFileSync(REVIEW_FILE, "utf8"));
103 } catch {}
104}
105// review.rs: no file means the agent's answer is the review, with no line comments.
106const comments = Array.isArray(review?.comments) ? review.comments : [];
107const known = changedPaths(diff);
108const findings = comments
109 .filter((c) => typeof c?.body === "string" && c.body.trim() && typeof c.path === "string")
110 // reviews.rs: a line in a file the pull request does not change is dropped.
111 .filter((c) => known.size === 0 || known.has(c.path.replace(/^\.\//, "")))
112 .slice(0, MAX_REVIEW_COMMENTS)
113 .map((c) => {
114 const line = Number.isInteger(c.line) && c.line > 0 ? c.line : 1;
115 const end = Number.isInteger(c.end_line) && c.end_line >= line ? c.end_line : line;
116 return { file: c.path.replace(/^\.\//, ""), start_line: line, end_line: end, message: c.body.trim().slice(0, MAX_REVIEW_CHARS), producer: agent };
117 });
118
119// Kept beside the findings for analysis: what production would also have
120// posted (verdict, summary) and what the run cost.
121writeFileSync(out.replace(/\.json$/, ".g1t.json"), JSON.stringify({ verdict: review?.verdict ?? null, body: review?.body ?? result.result ?? "", dropped: comments.length - findings.length, cost_usd: result.total_cost_usd, turns: result.num_turns, model }, null, 2));
122write(findings, { time_in_ms: Date.now() - started, ...(typeof result.total_cost_usd === "number" ? { cost_usd: result.total_cost_usd } : {}) });