Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Pricing says it plainly: models at the provider's price, the agent rate for what g1t runs around every model call (the gateway, secrets, routing, context and pass-through to your own provider, so your own keys too), everything else at cost plus 20%, your own runners free, no seats; Security and quality comes with the plan with no separate fee, and live activations end. Each agent has an effort setting, Auto to Max, with what a typical task has cost at each level, and Spend's Spend less, keep quality suggests a lower level only when the agent's own past work shows quality held, to apply or dismiss. The pricing, spend and agents guides say how. | 1 | import assert from "node:assert/strict"; |
| 2 | import { test } from "node:test"; | |
| 3 | ||
| 4 | import { MIN_SESSIONS, type SessionOutcome, currentLevel, effortCostsOf, judge, median, wilsonLower, wording } from "./recommend.ts"; | |
| 5 | ||
| 6 | /** `n` sessions at a level, `accepted` of them accepted, each costing `micros`. */ | |
| 7 | function at(effort: SessionOutcome["effort"], n: number, accepted: number, micros: number | ((i: number) => number)): SessionOutcome[] { | |
| 8 | return Array.from({ length: n }, (_, i) => ({ effort, accepted: i < accepted, charged_micros: typeof micros === "function" ? micros(i) : micros })); | |
| 9 | } | |
| 10 | ||
| 11 | test("a cheaper level that holds up on the agent's own work is recommended, with the saving from measured costs", () => { | |
| 12 | const outcomes = [...at("high", 20, 19, 480_000), ...at("medium", 12, 12, 210_000)]; | |
| 13 | const judged = judge("high", outcomes, 28); | |
| 14 | assert.equal(judged.kind, "recommend"); | |
| 15 | if (judged.kind !== "recommend") return; | |
| 16 | assert.equal(judged.to, "medium"); | |
| 17 | assert.equal(judged.current.sessions, 20); | |
| 18 | assert.equal(judged.cheaper.accepted, 12); | |
| 19 | // (480,000 - 210,000) × 20 sessions / 28 days × 30 days. | |
| 20 | assert.equal(judged.saving_month_micros, Math.round(270_000 * (20 / 28) * 30)); | |
| 21 | const words = wording(judged, "Reviewer"); | |
| 22 | assert.equal(words.title, "Run Reviewer at Medium effort"); | |
| 23 | assert.match(words.reason, /12 of its 12 sessions .* against 19 of 20 at High; a typical one cost \$0\.21 instead of \$0\.48/); | |
| 24 | }); | |
| 25 | ||
| 26 | test("too little history says so, and recommends nothing", () => { | |
| 27 | const judged = judge("high", [...at("high", 20, 20, 400_000), ...at("medium", 3, 3, 100_000)]); | |
| 28 | assert.equal(judged.kind, "thin"); | |
| 29 | if (judged.kind !== "thin") return; | |
| 30 | const words = wording(judged, "Reviewer"); | |
| 31 | assert.match(words.reason, new RegExp(`20 at High and 3 at Medium.*at least ${MIN_SESSIONS} at each`)); | |
| 32 | assert.equal(judge("high", at("high", 25, 25, 400_000)).kind, "thin", "never ran cheaper: nothing measured to compare"); | |
| 33 | }); | |
| 34 | ||
| 35 | test("a cheaper level that does worse, or saves too little, is not suggested", () => { | |
| 36 | // Acceptance falls from 95% to 70%. | |
| 37 | assert.equal(judge("high", [...at("high", 20, 19, 480_000), ...at("medium", 20, 14, 200_000)]).kind, "none"); | |
| 38 | // Same quality, but a typical session costs 95% as much. | |
| 39 | assert.equal(judge("high", [...at("high", 20, 20, 400_000), ...at("medium", 20, 20, 380_000)]).kind, "none"); | |
| 40 | }); | |
| 41 | ||
| 42 | test("nothing below low, and nothing with no history", () => { | |
| 43 | assert.equal(judge("low", at("low", 30, 30, 10_000)).kind, "none"); | |
| 44 | assert.equal(judge("high", []).kind, "none"); | |
| 45 | }); | |
| 46 | ||
| 47 | test("auto is judged at the level most of its sessions ran at", () => { | |
| 48 | const outcomes = [...at("medium", 15, 15, 300_000), ...at("high", 4, 3, 600_000), ...at("low", 11, 11, 90_000)]; | |
| 49 | assert.equal(currentLevel("auto", outcomes), "medium"); | |
| 50 | const judged = judge("auto", outcomes); | |
| 51 | assert.equal(judged.kind, "recommend"); | |
| 52 | if (judged.kind === "recommend") { | |
| 53 | assert.equal(judged.from, "auto"); | |
| 54 | assert.equal(judged.to, "low"); | |
| 55 | } | |
| 56 | }); | |
| 57 | ||
| 58 | test("the pessimistic estimate stops a lucky small sample", () => { | |
| 59 | assert.ok(wilsonLower(10, 10) < 1); | |
| 60 | assert.ok(wilsonLower(10, 10) > 0.8); | |
| 61 | assert.equal(wilsonLower(0, 0), 0); | |
| 62 | }); | |
| 63 | ||
| 64 | test("what each level cost is measured, and empty where it never ran", () => { | |
| 65 | const costs = effortCostsOf("reviewer", "high", [...at("high", 3, 2, (i) => [100, 300, 200][i]), ...at("low", 1, 1, 40)]); | |
| 66 | assert.deepEqual( | |
| 67 | costs.levels.map((l) => [l.effort, l.sessions, l.typical_micros]), | |
| 68 | [ | |
| 69 | ["low", 1, 40], | |
| 70 | ["medium", 0, null], | |
| 71 | ["high", 3, 200], | |
| 72 | ["max", 0, null], | |
| 73 | ], | |
| 74 | ); | |
| 75 | assert.equal(costs.levels[2].accepted_share, 2 / 3); | |
| 76 | assert.equal(median([1, 2, 3, 4]), 3); | |
| 77 | }); |
