| 1 | #!/usr/bin/env node |
| 2 | // Ask g1t's quality bar: asks every question in questions.json on the live |
| 3 | // Ask endpoint as the user, records the whole answer, and grades it against |
| 4 | // ground truth fetched at the same time, with deterministic checks and an |
| 5 | // LLM judge. |
| 6 | // |
| 7 | // node bench/ask/run.mjs ask, grade, judge, report (one pass) |
| 8 | // node bench/ask/run.mjs --only id,id some questions |
| 9 | // node bench/ask/run.mjs --dry-run fetch ground truth and print the plan; asks nothing |
| 10 | // node bench/ask/run.mjs --no-judge deterministic checks only |
| 11 | // node bench/ask/run.mjs grade --run <dir> grade a saved run again (and judge, unless --no-judge, which keeps the saved verdicts; |
| 12 | // --refresh-truth fetches ground truth again) |
| 13 | // node bench/ask/run.mjs report --run <dir> write the report again |
| 14 | // node bench/ask/run.mjs fixtures make the fixtures the suite needs (writes to the test repository) |
| 15 | // |
| 16 | // Options: --workspace flagon-io, --pace 3500 (ms between questions; Ask |
| 17 | // allows 20 a minute per person), --label text (kept in the run's folder name). |
| 18 | // |
| 19 | // Credentials, read in-process and never printed or written: |
| 20 | // G1T_TOKEN (or its export in ~/.bashrc) g1t's API and MCP, for ground truth |
| 21 | // .credentials/g1t_token.txt g1t.sh as the user, for Ask and link checks |
| 22 | // .secrets/anthropic-key.txt the judge |
| 23 | // The latter two are read from the main checkout, so this runs from a worktree too. |
| 24 | // |
| 25 | // Runs land in bench/ask/runs/<timestamp>/ (git-ignored): one JSON per |
| 26 | // question (truth, every event, checks, verdict), summary.json and report.md. |
| 27 | |
| 28 | import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; |
| 29 | import { basename, join, resolve } from "node:path"; |
| 30 | |
| 31 | import { BAR, DIMENSIONS, JUDGE_MODEL, deterministic, judge, summarize } from "./grade.mjs"; |
| 32 | import { HERE, ask, loadCredentials, parseArgs, redact, sleep } from "./lib.mjs"; |
| 33 | import { ensureFixtures, resolveTruth } from "./truth.mjs"; |
| 34 | |
| 35 | const { command, flags } = parseArgs(process.argv.slice(2)); |
| 36 | const suite = JSON.parse(readFileSync(join(HERE, "questions.json"), "utf8")); |
| 37 | const WORKSPACE = flags.workspace ?? suite.workspace; |
| 38 | process.env.ASK_WORKSPACE ??= WORKSPACE; |
| 39 | |
| 40 | const write = (path, value) => writeFileSync(path, redact(typeof value === "string" ? value : JSON.stringify(value, null, 2))); |
| 41 | const say = (line) => console.log(redact(line)); |
| 42 | |
| 43 | function selected() { |
| 44 | const only = typeof flags.only === "string" ? new Set(flags.only.split(",")) : null; |
| 45 | return suite.questions.filter((q) => !only || only.has(q.id)); |
| 46 | } |
| 47 | |
| 48 | function stamp() { |
| 49 | return new Date().toISOString().replace(/[:.]/g, "-").replace(/-\d{3}Z$/, "Z"); |
| 50 | } |
| 51 | |
| 52 | async function gradeOne(question, truth, answer, { useJudge }) { |
| 53 | const checks = await deterministic(question, truth, answer); |
| 54 | const verdict = useJudge ? await judge(question, truth, answer) : null; |
| 55 | return { checks, judge: verdict }; |
| 56 | } |
| 57 | |
| 58 | async function runSuite() { |
| 59 | const creds = loadCredentials(); |
| 60 | if (!creds.api) throw new Error("No G1T_TOKEN: export it, or source ~/.bashrc."); |
| 61 | if (!creds.web) throw new Error("No site token: .credentials/g1t_token.txt is missing."); |
| 62 | const dry = Boolean(flags["dry-run"]); |
| 63 | const useJudge = !flags["no-judge"] && !dry; |
| 64 | if (useJudge && !creds.anthropic) throw new Error("No judge key: .secrets/anthropic-key.txt is missing (or pass --no-judge)."); |
| 65 | const pace = Number(flags.pace ?? 3500); |
| 66 | const questions = selected(); |
| 67 | const dir = dry ? null : join(HERE, "runs", `${stamp()}${flags.label ? `-${String(flags.label).replace(/[^\w-]+/g, "-")}` : ""}`); |
| 68 | if (dir) mkdirSync(join(dir, "questions"), { recursive: true }); |
| 69 | say(`${dry ? "Dry run" : "Run"}: ${questions.length} questions in ${WORKSPACE}${useJudge ? `, judged by ${JUDGE_MODEL}` : ""}${dir ? `, into ${dir}` : ""}`); |
| 70 | |
| 71 | const results = []; |
| 72 | for (const [index, question] of questions.entries()) { |
| 73 | const truth = await resolveTruth(question.truth); |
| 74 | const page = question.page === "$truth" ? truth.page : question.page; |
| 75 | const messages = question.messages ?? [{ role: "user", content: question.question }]; |
| 76 | const label = `[${index + 1}/${questions.length}] ${question.id}`; |
| 77 | if (truth.skip) { |
| 78 | say(`${label}: skipped (${truth.skip})`); |
| 79 | results.push({ id: question.id, category: question.category, skipped: truth.skip, truth }); |
| 80 | continue; |
| 81 | } |
| 82 | if (dry) { |
| 83 | say(`${label}: ${messages.at(-1).content}\n page: ${JSON.stringify(page ?? {})}\n truth: ${truth.digest.split("\n").slice(0, 6).join("\n ")}\n require: ${JSON.stringify([...(truth.require_any ?? []), ...(question.require_any ?? [])]).slice(0, 300)}`); |
| 84 | continue; |
| 85 | } |
| 86 | if (index > 0) await sleep(pace); |
| 87 | const answer = await ask(WORKSPACE, { messages, page }); |
| 88 | const graded = await gradeOne(question, truth, answer, { useJudge }); |
| 89 | const result = { id: question.id, category: question.category, question: messages.at(-1).content, page: page ?? null, truth, answer, ...graded }; |
| 90 | results.push(result); |
| 91 | write(join(dir, "questions", `${question.id}.json`), result); |
| 92 | const passed = graded.checks.filter((c) => c.pass).length; |
| 93 | say(`${label}: ${answer.status} ${answer.total_ms} ms, checks ${passed}/${graded.checks.length}${graded.judge?.mean != null ? `, judge ${graded.judge.mean.toFixed(2)}` : graded.judge?.error ? `, ${graded.judge.error}` : ""}${answer.unknown_types.length ? `, new events: ${answer.unknown_types.join(",")}` : ""}`); |
| 94 | } |
| 95 | if (dry) return; |
| 96 | finish(dir, results); |
| 97 | } |
| 98 | |
| 99 | function finish(dir, results) { |
| 100 | const summary = summarize(results); |
| 101 | write(join(dir, "summary.json"), { ...summary, workspace: WORKSPACE, judge_model: JUDGE_MODEL, finished_at: new Date().toISOString() }); |
| 102 | write(join(dir, "report.md"), report(dir, results, summary)); |
| 103 | say(`\n${headline(summary)}\nReport: ${join(dir, "report.md")}`); |
| 104 | } |
| 105 | |
| 106 | function headline(s) { |
| 107 | const pct = (x) => `${(x * 100).toFixed(1)}%`; |
| 108 | return [ |
| 109 | `${s.pass ? "PASS" : "FAIL"} against the bar (checks >= ${pct(BAR.deterministic)}, judge >= ${BAR.judge_mean}, safety failures = ${BAR.safety_failures})`, |
| 110 | `deterministic checks ${s.checks.passed}/${s.checks.total} (${pct(s.checks.rate)}); judge mean ${s.judge.mean?.toFixed(2) ?? "-"} over ${s.judge.judged}; safety failures ${s.safety_failures}; answers with tool use ${s.tool_use.answers_with_tools}/${s.graded}`, |
| 111 | ].join("\n"); |
| 112 | } |
| 113 | |
| 114 | function report(dir, results, s) { |
| 115 | const pct = (x) => `${(x * 100).toFixed(1)}%`; |
| 116 | const ms = (x) => (x ? `${(x / 1000).toFixed(1)} s` : "-"); |
| 117 | const model = results.find((r) => r.answer?.meta)?.answer.meta; |
| 118 | const lines = [ |
| 119 | `# Ask g1t bench: ${basename(dir)}`, |
| 120 | "", |
| 121 | `**${s.pass ? "PASS" : "FAIL"}** · workspace \`${WORKSPACE}\` · answered by ${model ? `${model.provider_label} \`${model.model}\`` : "?"} · judged by \`${JUDGE_MODEL}\``, |
| 122 | "", |
| 123 | "| Measure | Result | Bar |", |
| 124 | "| --- | --- | --- |", |
| 125 | `| Deterministic checks | ${s.checks.passed}/${s.checks.total} (${pct(s.checks.rate)}) | ≥ ${pct(BAR.deterministic)} |`, |
| 126 | `| Judge mean (1–5) | ${s.judge.mean?.toFixed(2) ?? "-"} over ${s.judge.judged} | ≥ ${BAR.judge_mean} |`, |
| 127 | `| Safety failures | ${s.safety_failures} | ${BAR.safety_failures} |`, |
| 128 | `| Answers that used tools | ${s.tool_use.answers_with_tools}/${s.graded} | |`, |
| 129 | `| Time to first text, p50 / p90 | ${ms(s.latency.first_text_ms?.p50)} / ${ms(s.latency.first_text_ms?.p90)} | |`, |
| 130 | `| Total time, p50 / p90 / max | ${ms(s.latency.total_ms?.p50)} / ${ms(s.latency.total_ms?.p90)} / ${ms(s.latency.total_ms?.max)} | |`, |
| 131 | s.skipped ? `| Skipped | ${s.skipped} | |` : null, |
| 132 | s.tool_use.unknown_event_types.length ? `| Event types the bench does not know | ${s.tool_use.unknown_event_types.join(", ")} | |` : null, |
| 133 | "", |
| 134 | "## Judge by dimension", |
| 135 | "", |
| 136 | `| ${DIMENSIONS.join(" | ")} |`, |
| 137 | `| ${DIMENSIONS.map(() => "---").join(" | ")} |`, |
| 138 | `| ${DIMENSIONS.map((d) => s.judge.by_dimension[d]?.toFixed(2) ?? "-").join(" | ")} |`, |
| 139 | "", |
| 140 | "## Questions", |
| 141 | "", |
| 142 | "| Question | Checks | Judge | First text | Total | Tools | Failed checks |", |
| 143 | "| --- | --- | --- | --- | --- | --- | --- |", |
| 144 | ...results.map((r) => { |
| 145 | if (r.skipped) return `| ${r.id} | skipped | | | | | ${r.skipped} |`; |
| 146 | const failed = r.checks.filter((c) => !c.pass); |
| 147 | return `| ${r.id} | ${r.checks.length - failed.length}/${r.checks.length} | ${r.judge?.mean?.toFixed(2) ?? (r.judge?.error ? "error" : "-")} | ${ms(r.answer.first_text_ms)} | ${ms(r.answer.total_ms)} | ${r.answer.tool_events.length} | ${failed.map((c) => `${c.safety ? "**SAFETY** " : ""}${c.name}`).join(", ").replace(/\|/g, "\\|")} |`; |
| 148 | }), |
| 149 | "", |
| 150 | "## Answers", |
| 151 | "", |
| 152 | ]; |
| 153 | for (const r of results) { |
| 154 | if (r.skipped) continue; |
| 155 | lines.push(`### ${r.id}`, "", `> ${r.question.replace(/\n/g, " ")}`, ""); |
| 156 | if (r.page) lines.push(`Page: \`${JSON.stringify(r.page)}\``, ""); |
| 157 | lines.push("<details><summary>Answer</summary>", "", r.answer.text || `_(no answer: ${r.answer.error ?? "empty"})_`, "", "</details>", ""); |
| 158 | lines.push("Checks:", ...r.checks.map((c) => `- ${c.pass ? "✅" : "❌"} ${c.safety ? "(safety) " : ""}${c.name}: ${c.detail}`), ""); |
| 159 | if (r.judge && !r.judge.error) { |
| 160 | lines.push(`Judge ${r.judge.mean.toFixed(2)}${r.judge.safety_ok ? "" : " · **unsafe**"}: ${r.judge.summary}`, ""); |
| 161 | lines.push(...DIMENSIONS.map((d) => `- ${d} ${r.judge[d].score}: ${r.judge[d].rationale}`), ""); |
| 162 | } else if (r.judge?.error) lines.push(`Judge: ${r.judge.error}`, ""); |
| 163 | lines.push("<details><summary>Ground truth</summary>", "", "```", r.truth.digest, "```", "", "</details>", ""); |
| 164 | } |
| 165 | return lines.filter((l) => l !== null).join("\n"); |
| 166 | } |
| 167 | |
| 168 | async function regrade() { |
| 169 | const dir = resolve(String(flags.run ?? "")); |
| 170 | if (!flags.run || !existsSync(join(dir, "questions"))) throw new Error("--run <dir> with a questions/ folder"); |
| 171 | loadCredentials(); |
| 172 | const useJudge = !flags["no-judge"] && command === "grade"; |
| 173 | const byId = new Map(suite.questions.map((q) => [q.id, q])); |
| 174 | const results = []; |
| 175 | for (const file of readdirSync(join(dir, "questions")).filter((f) => f.endsWith(".json"))) { |
| 176 | const saved = JSON.parse(readFileSync(join(dir, "questions", file), "utf8")); |
| 177 | const question = byId.get(saved.id) ?? saved; |
| 178 | if (command === "grade") { |
| 179 | // --refresh-truth fetches ground truth again (for changed resolvers); it is no longer the moment of asking. |
| 180 | if (flags["refresh-truth"] && question.truth) saved.truth = await resolveTruth(question.truth); |
| 181 | const kept = saved.judge; |
| 182 | Object.assign(saved, await gradeOne({ ...question, page: saved.page }, saved.truth, saved.answer, { useJudge })); |
| 183 | if (!useJudge) saved.judge = kept; |
| 184 | write(join(dir, "questions", file), saved); |
| 185 | say(`${saved.id}: checks ${saved.checks.filter((c) => c.pass).length}/${saved.checks.length}${saved.judge?.mean != null ? `, judge ${saved.judge.mean.toFixed(2)}` : ""}`); |
| 186 | } |
| 187 | results.push(saved); |
| 188 | } |
| 189 | const order = new Map(suite.questions.map((q, i) => [q.id, i])); |
| 190 | results.sort((a, b) => (order.get(a.id) ?? 99) - (order.get(b.id) ?? 99)); |
| 191 | finish(dir, results); |
| 192 | } |
| 193 | |
| 194 | const commands = { |
| 195 | async undefined() { |
| 196 | await runSuite(); |
| 197 | }, |
| 198 | async run() { |
| 199 | await runSuite(); |
| 200 | }, |
| 201 | grade: regrade, |
| 202 | report: regrade, |
| 203 | async fixtures() { |
| 204 | loadCredentials(); |
| 205 | say(JSON.stringify(await ensureFixtures())); |
| 206 | }, |
| 207 | }; |
| 208 | |
| 209 | const handler = commands[command]; |
| 210 | if (!handler) { |
| 211 | console.error(`Unknown command ${command}. See the top of bench/ask/run.mjs.`); |
| 212 | process.exit(2); |
| 213 | } |
| 214 | handler().catch((error) => { |
| 215 | console.error(redact(error?.stack ?? String(error))); |
| 216 | process.exit(1); |
| 217 | }); |