Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp

Commit

bench/reviewbench is gone: it benchmarked g1t's own pull request reviewer, reading its instructions from crates/runner/src/review.rs and its run from the runner's startReviewRun, both deleted with g1t's hosted agents, so it could no longer build or run. Ask g1t's quality bar, bench/ask, stays.

syntaqxcommitted Parentba67a3cBrowse files
8 files+0−5110/8 viewed
+0−1
1−.cache/
+0−17
1−# syntax=docker/dockerfile:1.7
2−#
3−# g1t's reviewer as a ReviewBench agent image: the g1t sandbox's base image
4−# (the Claude Code CLI and toolchains production reviews run with) plus the
5−# contract adapter in agent/. Built by `node bench/reviewbench/run.mjs build`,
6−# which passes BASE from services/runner/base.json and regenerates
7−# agent/instructions.txt from crates/runner/src/review.rs first.
8−#
9−# linux/amd64, as the contract requires.
10−
11−ARG BASE
12−FROM --platform=linux/amd64 ${BASE}
13−COPY agent/ /bench/
14−# /work/change.diff and /work/review.json are where the production prompt
15−# tells the agent to read and write; the benchmark mounts the rest of /work.
16−RUN mkdir -p /work && chmod 0777 /work
17−ENTRYPOINT ["node", "/bench/agent.mjs"]
+0−122
1−// g1t's pull request reviewer behind the ReviewBench agent contract
2−// (https://github.com/review-bench/ReviewBench/blob/main/AGENT_CONTRACT.md).
3−//
4−// Runs inside the image built from bench/reviewbench/Dockerfile, which is
5−// the g1t sandbox's base image (same Claude Code CLI, same toolchains).
6−// It does what crates/runner/src/review.rs does after its clone: write the
7−// change to /work/change.diff, run Claude Code headless with the same
8−// flags as crates/runner/src/harness.rs, read /work/review.json, and apply
9−// the same filters services/work/src/reviews.rs applies when it records a
10−// review. Then it writes the review's line comments as findings.
11−//
12−// Configuration (all optional):
13−// ANTHROPIC_MODEL or RB_CONFIG_MODEL model id (production review route: claude-sonnet-5-5)
14−// ANTHROPIC_BASE_URL g1t's model proxy (MODELS_URL/anthropic) or a gateway
15−// ANTHROPIC_API_KEY provider key, or a model-proxy session token
16−// RB_CONFIG_BUDGET_USD per-PR cost cap (--max-budget-usd), default 5
17−// RB_CONFIG_VARIANT prompt variant: "production" (default) or a file in variants/
18−
19−import { spawn } from "node:child_process";
20−import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
21−
22−import { buildPrompt } from "./prompt-lib.mjs";
23−
24−const REPO = "/work/repo";
25−const DIFF_FILE = "/work/change.diff";
26−const REVIEW_FILE = "/work/review.json";
27−const MAX_TURNS = "80"; // harness.rs
28−const MAX_REVIEW_COMMENTS = 30; // reviews.rs
29−const MAX_REVIEW_CHARS = 20_000; // reviews.rs
30−
31−const env = (name, fallback) => process.env[name] ?? fallback;
32−const out = env("RB_OUT", "/work/out/findings.json");
33−const agent = env("RB_AGENT", "g1t");
34−const pr = JSON.parse(readFileSync(env("RB_PR_JSON", "/work/pr/pr.json"), "utf8"));
35−const model = env("RB_CONFIG_MODEL", env("ANTHROPIC_MODEL", "claude-sonnet-5-5"));
36−const budget = Number(env("RB_CONFIG_BUDGET_USD", "5"));
37−const variant = env("RB_CONFIG_VARIANT", "production");
38−
39−// The run's settings, printed so ReviewBench can see each label took effect.
40−console.log(`model=${model} budget_usd=${budget} variant=${variant}`);
41−
42−function write(findings, usage) {
43− const body = {
44− pr: { repo: pr.repo ?? `https://github.com/${env("RB_NWO")}`, pr_number: Number(env("RB_PR_NUMBER", pr.pr_number)), base: env("RB_BASE", pr.base), head: env("RB_HEAD", pr.head) },
45− agent,
46− findings,
47− usage,
48− };
49− writeFileSync(out, JSON.stringify(body, null, 2));
50−}
51−
52−/** Paths the change touches, as the work service knows them (pull.files). */
53−function changedPaths(diff) {
54− const paths = new Set();
55− for (const line of diff.split("\n")) {
56− const m = line.match(/^\+\+\+ b\/(.+)$/) ?? line.match(/^--- a\/(.+)$/);
57− if (m && m[1] !== "/dev/null") paths.add(m[1]);
58− }
59− return paths;
60−}
61−
62−function runClaude(prompt) {
63− return new Promise((resolve, reject) => {
64− const args = ["--print", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", MAX_TURNS, "--dangerously-skip-permissions", "--model", model];
65− if (budget > 0) args.push("--max-budget-usd", budget.toFixed(2));
66− const child = spawn("claude", args, { cwd: REPO, stdio: ["ignore", "pipe", "inherit"], env: { ...process.env, ANTHROPIC_MODEL: model } });
67− let result = null;
68− let buffer = "";
69− child.stdout.on("data", (chunk) => {
70− buffer += chunk;
71− let at;
72− while ((at = buffer.indexOf("\n")) >= 0) {
73− const line = buffer.slice(0, at);
74− buffer = buffer.slice(at + 1);
75− try {
76− const event = JSON.parse(line);
77− if (event.type === "result") result = event;
78− } catch {}
79− }
80− });
81− child.on("error", reject);
82− child.on("close", (code) => (result ? resolve(result) : reject(new Error(`claude exited ${code} without a result`))));
83− });
84−}
85−
86−const started = Date.now();
87−copyFileSync(env("RB_DIFF", "/work/pr/diff.patch"), DIFF_FILE);
88−const diff = readFileSync(DIFF_FILE, "utf8");
89−if (!diff.trim()) {
90− // review.rs bails on an empty change; the benchmark still wants a file.
91− write([], { time_in_ms: Date.now() - started });
92− process.exit(0);
93−}
94−
95−const instructionsFile = variant === "production" ? new URL("./instructions.txt", import.meta.url) : new URL(`./variants/${variant}.txt`, import.meta.url);
96−const prompt = buildPrompt(pr, readFileSync(instructionsFile, "utf8"));
97−const result = await runClaude(prompt);
98−
99−let review = null;
100−if (existsSync(REVIEW_FILE)) {
101− try {
102− review = JSON.parse(readFileSync(REVIEW_FILE, "utf8"));
103− } catch {}
104−}
105−// review.rs: no file means the agent's answer is the review, with no line comments.
106−const comments = Array.isArray(review?.comments) ? review.comments : [];
107−const known = changedPaths(diff);
108−const findings = comments
109− .filter((c) => typeof c?.body === "string" && c.body.trim() && typeof c.path === "string")
110− // reviews.rs: a line in a file the pull request does not change is dropped.
111− .filter((c) => known.size === 0 || known.has(c.path.replace(/^\.\//, "")))
112− .slice(0, MAX_REVIEW_COMMENTS)
113− .map((c) => {
114− const line = Number.isInteger(c.line) && c.line > 0 ? c.line : 1;
115− const end = Number.isInteger(c.end_line) && c.end_line >= line ? c.end_line : line;
116− return { file: c.path.replace(/^\.\//, ""), start_line: line, end_line: end, message: c.body.trim().slice(0, MAX_REVIEW_CHARS), producer: agent };
117− });
118−
119−// Kept beside the findings for analysis: what production would also have
120−// posted (verdict, summary) and what the run cost.
121−writeFileSync(out.replace(/\.json$/, ".g1t.json"), JSON.stringify({ verdict: review?.verdict ?? null, body: review?.body ?? result.result ?? "", dropped: comments.length - findings.length, cost_usd: result.total_cost_usd, turns: result.num_turns, model }, null, 2));
122−write(findings, { time_in_ms: Date.now() - started, ...(typeof result.total_cost_usd === "number" ? { cost_usd: result.total_cost_usd } : {}) });
+0−15
1−You are reviewing a pull request. The repository is checked out in the current directory at the pull request's head. The change under review is in /work/change.diff; read it first, then read the surrounding code as needed. You may run the project's tests. Do not modify the repository.
2−
3−Judge whether the change does what it is for, whether it is correct, and whether it would break anything. Be specific and brief. Comment only on real problems or things a maintainer would want to know; do not praise, and do not restate the diff.
4−
5−Write your review to /work/review.json as JSON with exactly this shape:
6−
7−{
8− "verdict": "approve" or "request_changes",
9− "body": "A summary in Markdown: what you checked and your conclusion.",
10− "comments": [
11− { "path": "path/in/the/repository", "line": 12, "body": "What is wrong on this line and what to do about it." }
12− ]
13−}
14−
15−`line` is the line number in the file as it is after the change. `comments` may be empty. Use "request_changes" only if something must be fixed before merging. Then finish.
+0−8
1−// The part of ../prompt.mjs the image needs, without the path to the Rust
2−// source (the image only carries instructions.txt, generated from it).
3−
4−/** What production says about the pull request, then the instructions. */
5−export function buildPrompt(pr, instructions) {
6− const about = [`Pull request #${pr.pr_number}: ${pr.title ?? ""}`, pr.body ?? ""].filter((part) => part.trim()).join("\n\n");
7− return `${about}\n\n${instructions}`;
8−}
+0−21
1−You are reviewing a pull request. The repository is checked out in the current directory at the pull request's head. The change under review is in /work/change.diff. Do not modify the repository.
2−
3−Work in three passes.
4−
5−1. Map the change. Read the diff and list every file it touches. For each changed function, type or config, find its callers and the code it calls (grep for the names) and read them. Read the tests that cover the change, and the tests that should and do not.
6−
7−2. Look for problems, file by file, in this order: correctness (wrong logic, off-by-one, null and error paths, broken invariants, concurrency, resource leaks), security (input reaching queries, shells, paths, HTML or regexes; secrets; authorization), reliability (missing error handling, retries, timeouts, migrations that cannot run twice), API and compatibility (callers this change breaks, changed behavior without a version bump), tests (missing or ineffective tests for new behavior), performance (work in loops, unbounded growth), documentation that the change makes wrong, and accessibility in UI code. Where you can, run the project's tests or a quick check to confirm a suspicion.
8−
9−3. Check each candidate before keeping it. Re-read the exact lines. Drop it if it is not true of the code at head, if it is pure style, or if the change does not make it worse. Keep every distinct problem that survives; do not stop at the first few.
10−
11−Write your review to /work/review.json as JSON with exactly this shape:
12−
13−{
14− "verdict": "approve" or "request_changes",
15− "body": "A summary in Markdown: what you checked and your conclusion.",
16− "comments": [
17− { "path": "path/in/the/repository", "line": 12, "end_line": 14, "severity": "high" | "medium" | "low", "body": "What is wrong, why it matters, and what to do about it." }
18− ]
19−}
20−
21−`line` and `end_line` are line numbers in the file as it is after the change, spanning the problem. One comment per problem. Every problem you mention in the summary must also be a comment on its lines. `comments` may be empty. Use "request_changes" only if something must be fixed before merging. Then finish.
+0−49
1−// The review prompt, taken from the runner's own source so the benchmark
2−// measures what production runs. crates/runner/src/review.rs holds the
3−// instructions as a Rust string constant; services/runner/src/index.ts
4−// (startReviewRun) puts "what the pull request is" in front of them.
5−//
6−// Production also appends repository instructions (AGENTS.md, CLAUDE.md,
7−// .g1t/review.md); Claude Code reads CLAUDE.md from the checkout by
8−// itself, as it does in a trusted production run.
9−
10−import { readFileSync } from "node:fs";
11−import { fileURLToPath } from "node:url";
12−
13−const REVIEW_RS = fileURLToPath(new URL("../../crates/runner/src/review.rs", import.meta.url));
14−
15−/** The `INSTRUCTIONS` constant of review.rs, unescaped as rustc would. */
16−export function instructionsFromRust(source = readFileSync(REVIEW_RS, "utf8")) {
17− const match = source.match(/const INSTRUCTIONS: &str = "([\s\S]*?)";\n/);
18− if (!match) throw new Error("INSTRUCTIONS not found in review.rs; update bench/reviewbench/prompt.mjs");
19− return match[1]
20− // A backslash at the end of a line continues the string and eats the
21− // next line's leading whitespace.
22− .replace(/\\\n\s*/g, "")
23− .replace(/\\"/g, '"')
24− .replace(/\\n/g, "\n")
25− .replace(/\\\\/g, "\\");
26−}
27−
28−export { buildPrompt } from "./agent/prompt-lib.mjs";
29−
30−// `node prompt.mjs` writes instructions.txt for the image build, and
31−// `node prompt.mjs --check` fails when it has drifted from review.rs.
32−if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
33− const out = fileURLToPath(new URL("./agent/instructions.txt", import.meta.url));
34− const text = instructionsFromRust();
35− if (process.argv.includes("--check")) {
36− let current = "";
37− try {
38− current = readFileSync(out, "utf8");
39− } catch {}
40− if (current !== text) {
41− console.error("bench/reviewbench/agent/instructions.txt is out of date; run node bench/reviewbench/prompt.mjs");
42− process.exit(1);
43− }
44− } else {
45− const { writeFileSync } = await import("node:fs");
46− writeFileSync(out, text);
47− console.log(`wrote ${out} (${text.length} chars)`);
48− }
49−}
+0−278
1−#!/usr/bin/env node
2−// Runs g1t's pull request reviewer over ReviewBench and scores it with the
3−// benchmark's own judge.
4−//
5−// node bench/reviewbench/run.mjs setup clone ReviewBench (pinned) and install its judge
6−// node bench/reviewbench/run.mjs build build the agent image from the runner base image
7−// node bench/reviewbench/run.mjs estimate --set sample:50
8−// node bench/reviewbench/run.mjs review --set sample:50 [--seed 1] [--variant production] [--model claude-sonnet-5-5] --yes
9−// node bench/reviewbench/run.mjs judge --run <id> [--judge-model claude-sonnet-5] --yes
10−// node bench/reviewbench/run.mjs report --run <id> [--json]
11−//
12−// --set is test (the benchmark's 25), full (all 219) or sample:N (N of the
13−// 219, stratified by change size, reproducible with --seed).
14−//
15−// Model access, from the environment, passed into each container:
16−// ANTHROPIC_API_KEY a provider key, or
17−// ANTHROPIC_BASE_URL + ANTHROPIC_API_KEY g1t's model proxy (MODELS_URL/anthropic and a session token)
18−// The judge reads ANTHROPIC_API_KEY itself (through ReviewBench's pi registry).
19−//
20−// Nothing that spends money runs without --yes. Each review is capped by
21−// --budget-usd (default 5), and a run stops once --max-total-usd is spent.
22−//
23−// Needs docker, git, jq and bash (Git Bash on Windows) and Node 20+.
24−
25−import { execFileSync, spawnSync } from "node:child_process";
26−import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
27−import { dirname, join } from "node:path";
28−import { fileURLToPath } from "node:url";
29−
30−const HERE = dirname(fileURLToPath(import.meta.url));
31−const ROOT = join(HERE, "..", "..");
32−const CACHE = join(HERE, ".cache");
33−const RB = join(CACHE, "ReviewBench");
34−const RUNS = join(CACHE, "runs");
35−// The ReviewBench commit these numbers are comparable against.
36−const RB_COMMIT = "ceb0794a3768da6ef4a56e5311dfb4afd29e5dee";
37−const IMAGE = "g1t-reviewbench:dev";
38−// The model the official leaderboard is judged with.
39−const DEFAULT_JUDGE = "claude-sonnet-5";
40−// services/runner/wrangler.jsonc AGENT_ROUTES.review
41−const DEFAULT_MODEL = "claude-sonnet-5-5";
42−
43−// $ per million tokens: input, output, cache read, cache write (5 minute).
44−const PRICES = {
45− "claude-sonnet-5-5": [2, 10, 0.2, 2.5],
46− "claude-sonnet-5": [2, 10, 0.2, 2.5],
47− "claude-opus-5-5": [4, 20, 0.2, 5],
48− "claude-haiku-4-5": [1, 5, 0.1, 1.25],
49−};
50−
51−function args() {
52− const [command, ...rest] = process.argv.slice(2);
53− const flags = {};
54− for (let i = 0; i < rest.length; i++) {
55− const name = rest[i].replace(/^--/, "");
56− const next = rest[i + 1];
57− if (next === undefined || next.startsWith("--")) flags[name] = true;
58− else flags[name] = rest[++i];
59− }
60− return { command, flags };
61−}
62−
63−const run = (cmd, argv, opts = {}) => {
64− const result = spawnSync(cmd, argv, { stdio: "inherit", ...opts });
65− if (result.status !== 0) throw new Error(`${cmd} ${argv.join(" ")} exited ${result.status}`);
66−};
67−
68−function manifest() {
69− return JSON.parse(readFileSync(join(RB, "corpus", "manifest.json"), "utf8"));
70−}
71−
72−/** A seeded shuffle, so a sample is the same sample next week. */
73−function shuffle(items, seed) {
74− let state = seed >>> 0 || 1;
75− const random = () => ((state = (state * 1664525 + 1013904223) >>> 0) / 2 ** 32);
76− const copy = [...items];
77− for (let i = copy.length - 1; i > 0; i--) {
78− const j = Math.floor(random() * (i + 1));
79− [copy[i], copy[j]] = [copy[j], copy[i]];
80− }
81− return copy;
82−}
83−
84−/** Indices into the full manifest for --set. */
85−function select(set, seed) {
86− const all = manifest();
87− if (set === "full") return all.map((_, i) => i);
88− if (set === "test") {
89− const test = JSON.parse(readFileSync(join(RB, "corpus", "test", "test.json"), "utf8"));
90− const keys = new Set(test.map((p) => `${p.nwo}#${p.pr_number}`));
91− return all.flatMap((p, i) => (keys.has(`${p.nwo}#${p.pr_number}`) ? [i] : []));
92− }
93− const m = /^sample:(\d+)$/.exec(set);
94− if (!m) throw new Error(`--set is test, full or sample:N, not ${set}`);
95− const n = Number(m[1]);
96− // Stratified by change size, in the corpus's proportions, so a sample is
97− // not all small or all huge.
98− const bucket = (p) => {
99− const lines = p.lines_added + p.lines_removed;
100− return lines <= 200 ? 0 : lines <= 1000 ? 1 : 2;
101− };
102− const strata = [[], [], []];
103− all.forEach((p, i) => strata[bucket(p)].push(i));
104− const picked = [];
105− for (const stratum of strata) {
106− const share = Math.round((stratum.length / all.length) * n);
107− picked.push(...shuffle(stratum, seed).slice(0, share));
108− }
109− return picked.slice(0, n).sort((a, b) => a - b);
110−}
111−
112−/**
113− * What one review is expected to cost. An agentic review re-reads its
114− * growing context every turn, mostly from cache: turns and context grow
115− * with the size of the change. Calibrate against real review runs (the
116− * billing ledger records each run's cost) before trusting the absolute
117− * numbers.
118− */
119−function estimateOne(p, model) {
120− const [, output, cacheRead, cacheWrite] = PRICES[model] ?? PRICES[DEFAULT_MODEL];
121− const lines = p.lines_added + p.lines_removed;
122− const turns = Math.min(80, 12 + Math.round(Math.sqrt(lines) * 0.8) + Math.min(p.files_changed, 40) * 0.4);
123− const startContext = 18_000 + Math.min(lines, 6_000) * 12; // prompt, tools, the diff when read
124− const endContext = Math.min(startContext + turns * 2_500, 180_000); // files read along the way
125− const meanContext = (startContext + endContext) / 2;
126− // Each turn re-reads the context from cache and writes only what it added.
127− const fresh = endContext;
128− const cachedReads = Math.max(0, turns * meanContext - fresh);
129− const outTokens = turns * 450 + 2_500;
130− return (cachedReads * cacheRead + fresh * cacheWrite + outTokens * output) / 1e6;
131−}
132−
133−/**
134− * The judge: matching per file chunk plus classifying every unmatched
135− * finding with read access to the repository (a few tool calls each).
136− */
137−function estimateJudge(p, model, findingsPerPr = 6) {
138− const [input, output, cacheRead] = PRICES[model] ?? PRICES[DEFAULT_JUDGE];
139− const matchCalls = Math.max(1, Math.ceil(findingsPerPr / 3));
140− const unmatched = findingsPerPr * 0.5;
141− const tokensIn = matchCalls * 6_000 + unmatched * 5 * 12_000;
142− const tokensOut = matchCalls * 1_500 + unmatched * 5 * 600;
143− return (tokensIn * 0.5 * input + tokensIn * 0.5 * cacheRead + tokensOut * output) / 1e6;
144−}
145−
146−function estimate(indices, model, judgeModel, rounds = 1) {
147− const all = manifest();
148− let review = 0;
149− let judge = 0;
150− for (const i of indices) {
151− review += estimateOne(all[i], model);
152− judge += estimateJudge(all[i], judgeModel);
153− }
154− return { prs: indices.length, rounds, review_usd: review * rounds, judge_usd: judge * rounds, total_usd: (review + judge) * rounds };
155−}
156−
157−function setup() {
158− mkdirSync(CACHE, { recursive: true });
159− if (!existsSync(join(RB, ".git"))) run("git", ["clone", "https://github.com/review-bench/ReviewBench.git", RB]);
160− run("git", ["-C", RB, "fetch", "-q", "origin", RB_COMMIT]);
161− run("git", ["-C", RB, "checkout", "-q", "--detach", RB_COMMIT]);
162− run("npm", ["ci", "--no-audit", "--no-fund"], { cwd: RB, shell: process.platform === "win32" });
163−}
164−
165−function build() {
166− run(process.execPath, [join(HERE, "prompt.mjs")]);
167− const base = JSON.parse(readFileSync(join(ROOT, "services", "runner", "base.json"), "utf8"));
168− run("docker", ["build", "--platform", "linux/amd64", "--build-arg", `BASE=${base.image}@${base.digest}`, "-t", IMAGE, HERE]);
169−}
170−
171−function review(flags) {
172− if (!flags.yes) throw new Error("review spends model credit; run estimate first, then pass --yes");
173− const set = flags.set ?? "sample:50";
174− const seed = Number(flags.seed ?? 1);
175− const model = flags.model ?? DEFAULT_MODEL;
176− const variant = flags.variant ?? "production";
177− const budget = Number(flags["budget-usd"] ?? 5);
178− const maxTotal = Number(flags["max-total-usd"] ?? 150);
179− const indices = select(set, seed);
180− const id = flags.run ?? `${new Date().toISOString().slice(0, 10)}-${set.replace(":", "")}-${variant}-${model}`;
181− const dir = join(RUNS, id);
182− mkdirSync(dir, { recursive: true });
183− writeFileSync(join(dir, "run.json"), JSON.stringify({ id, set, seed, model, variant, budget_usd: budget, indices, reviewbench: RB_COMMIT, g1t: execFileSync("git", ["-C", ROOT, "rev-parse", "HEAD"]).toString().trim(), started_at: new Date().toISOString() }, null, 2));
184− const env = { ...process.env, RB_CONFIG_MODEL: model, RB_CONFIG_VARIANT: variant, RB_CONFIG_BUDGET_USD: String(budget), TRY_AGENT_WORK: join(dir, ".work") };
185− const pass = ["-e", "RB_CONFIG_MODEL", "-e", "RB_CONFIG_VARIANT", "-e", "RB_CONFIG_BUDGET_USD", "-e", "ANTHROPIC_API_KEY"];
186− if (process.env.ANTHROPIC_BASE_URL) pass.push("-e", "ANTHROPIC_BASE_URL");
187− let spent = 0;
188− for (const i of indices) {
189− if (spent >= maxTotal) {
190− console.error(`stopped: spent $${spent.toFixed(2)} of --max-total-usd ${maxTotal}`);
191− break;
192− }
193− spawnSync("bash", [join(RB, "scripts", "try-agent.sh"), IMAGE, "--set", "full", "--pr", String(i), ...pass], { cwd: dir, env, stdio: "inherit" });
194− spent = sidecars(dir).reduce((sum, s) => sum + (s.cost_usd ?? 0), 0);
195− }
196− console.log(`run ${id}: ${readdirSync(join(dir, "findings")).length} of ${indices.length} reviewed, $${spent.toFixed(2)} spent`);
197−}
198−
199−/** What production would have posted beside the findings, and the cost. */
200−function sidecars(dir) {
201− const out = join(dir, ".work", "out");
202− if (!existsSync(out)) return [];
203− return readdirSync(out).flatMap((key) => {
204− const file = join(out, key, "findings.g1t.json");
205− return existsSync(file) ? [{ key, ...JSON.parse(readFileSync(file, "utf8")) }] : [];
206− });
207−}
208−
209−function judge(flags) {
210− if (!flags.yes) throw new Error("judging spends model credit; pass --yes");
211− const dir = join(RUNS, flags.run);
212− const judgeModel = flags["judge-model"] ?? DEFAULT_JUDGE;
213− run("npm", ["run", "judge", "--", "--candidate", join(dir, "findings"), "--golden", join(RB, "golden"), "--manifest", join(RB, "corpus", "manifest.json"), "--provider", "anthropic", "--model", judgeModel, "--output", join(dir, "scoring", "results.json"), "--repo-dir", join(CACHE, "judge-repos"), "--concurrency", String(flags.concurrency ?? 4)], { cwd: RB, shell: process.platform === "win32" });
214−}
215−
216−function report(flags) {
217− const dir = join(RUNS, flags.run);
218− const meta = JSON.parse(readFileSync(join(dir, "run.json"), "utf8"));
219− const results = JSON.parse(readFileSync(join(dir, "scoring", "results.json"), "utf8"));
220− const side = sidecars(dir);
221− const cost = side.reduce((sum, s) => sum + (s.cost_usd ?? 0), 0);
222− const summary = {
223− run: meta.id,
224− date: meta.started_at.slice(0, 10),
225− g1t: meta.g1t,
226− model: meta.model,
227− variant: meta.variant,
228− set: meta.set,
229− prs: side.length,
230− findings: readdirSync(join(dir, "findings")).reduce((sum, f) => sum + JSON.parse(readFileSync(join(dir, "findings", f), "utf8")).findings.length, 0),
231− review_cost_usd: Number(cost.toFixed(2)),
232− request_changes: side.filter((s) => s.verdict === "request_changes").length,
233− dropped_outside_change: side.reduce((sum, s) => sum + (s.dropped ?? 0), 0),
234− // The judge's own aggregate block, as ReviewBench writes it.
235− metrics: results.metrics ?? results.summary ?? results,
236− };
237− if (flags.json) {
238− console.log(JSON.stringify(summary));
239− return;
240− }
241− console.log(`## ReviewBench: ${summary.run}\n`);
242− console.log(`g1t ${summary.g1t.slice(0, 8)}, ${summary.model}, prompt ${summary.variant}, ${summary.set} (${summary.prs} PRs), review cost $${summary.review_cost_usd}\n`);
243− console.log("```json\n" + JSON.stringify(summary.metrics, null, 2) + "\n```");
244−}
245−
246−const { command, flags } = args();
247−try {
248− switch (command) {
249− case "setup":
250− setup();
251− break;
252− case "build":
253− build();
254− break;
255− case "estimate": {
256− const indices = select(flags.set ?? "sample:50", Number(flags.seed ?? 1));
257− const model = flags.model ?? DEFAULT_MODEL;
258− const judgeModel = flags["judge-model"] ?? DEFAULT_JUDGE;
259− console.log(JSON.stringify({ set: flags.set ?? "sample:50", model, judge: judgeModel, ...estimate(indices, model, judgeModel, Number(flags.rounds ?? 1)) }, (k, v) => (typeof v === "number" ? Number(v.toFixed(2)) : v), 2));
260− break;
261− }
262− case "review":
263− review(flags);
264− break;
265− case "judge":
266− judge(flags);
267− break;
268− case "report":
269− report(flags);
270− break;
271− default:
272− console.error(readFileSync(fileURLToPath(import.meta.url), "utf8").split("\n").slice(1, 22).join("\n"));
273− process.exit(2);
274− }
275−} catch (error) {
276− console.error(String(error.message ?? error));
277− process.exit(1);
278−}

Comments

Sign in to comment.