g1t/services/runner/src/gate.test.ts
Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look | 1 | import assert from "node:assert/strict"; |
| 2 | import { test } from "node:test"; | |
| 3 | ||
| 4 | import { | |
| 5 | type ComputeEntitlements, | |
| 6 | type ComputeKind, | |
| 7 | type PlanMemory, | |
| 8 | ComputeGate, | |
| 9 | MODEL_ESTIMATE_MICROS, | |
| 10 | WAITING_PREFIX, | |
| 11 | actualMicros, | |
| 12 | agentEstimateMicros, | |
| 13 | alwaysPasses, | |
| 14 | embeddingEstimateMicros, | |
| 15 | isWaiting, | |
| 16 | issueCapReached, | |
| 17 | localRefusal, | |
| 18 | lowerCap, | |
| 19 | onBillingError, | |
| 20 | readEntitlements, | |
| 21 | refusalCode, | |
| 22 | refusalMessage, | |
| 23 | runBudgetUsd, | |
| 24 | runMinutes, | |
| 25 | sandboxEstimateMicros, | |
| 26 | slotFree, | |
| 27 | waitingMessage, | |
| 28 | } from "../../../packages/contracts/src/compute.ts"; | |
| 29 | import { BUILD_HOSTS, buildHosts, withPlanLimits } from "./egress.ts"; | |
| 30 | import { WAITING, handleMention } from "./mentions.ts"; | |
| 31 | ||
| 32 | const repo = { namespace: "acme", name: "web" }; | |
| 33 | ||
| 34 | function ent(change: Partial<ComputeEntitlements> = {}): ComputeEntitlements { | |
| 35 | return { | |
| 36 | plan: "paid", | |
| 37 | compute: true, | |
| 38 | trialMicrosLeft: 0, | |
| 39 | trialVerified: true, | |
| 40 | firstMonth: false, | |
| 41 | maxConcurrentAgents: 10, | |
| 42 | maxRunMinutes: 0, | |
| 43 | runCapMicros: 2_000_000, | |
| 44 | issueCapMicros: 10_000_000, | |
| 45 | ceilingMicros: 100_000_000, | |
| 46 | exposureMicros: 0, | |
| 47 | paused: null, | |
| 48 | ...change, | |
| 49 | }; | |
| 50 | } | |
| 51 | ||
| 52 | /** A billing service that answers as told, and records what it was asked. */ | |
| 53 | function billing(answers: { | |
| 54 | entitlements?: unknown; | |
| 55 | reserve?: unknown; | |
| 56 | fail?: Set<string>; | |
| 57 | }) { | |
| 58 | const asked: { method: string; body: Record<string, unknown> }[] = []; | |
| 59 | const binding = { | |
| 60 | async fetch(url: string, init?: RequestInit): Promise<Response> { | |
| 61 | const method = url.split("/rpc/")[1]; | |
| 62 | asked.push({ method, body: JSON.parse(String(init?.body ?? "{}")) }); | |
| 63 | if (answers.fail?.has(method)) return new Response("down", { status: 503 }); | |
| 64 | if (method === "entitlements") return Response.json(answers.entitlements ?? null); | |
| 65 | if (method === "reserve") return Response.json(answers.reserve ?? { ok: true, value: { id: "rsv_1", paidBy: "credit" } }); | |
| 66 | if (method === "settle") return Response.json({ ok: true, value: true }); | |
| 67 | if (method === "prices") return Response.json({ prices: [{ meter: "sandbox_second", costMicros: 20 }] }); | |
| 68 | return new Response("not found", { status: 404 }); | |
| 69 | }, | |
| 70 | }; | |
| 71 | return { binding, asked }; | |
| 72 | } | |
| 73 | ||
| 74 | /** A plan memory that remembers nothing unless told. */ | |
| 75 | function memory(plan: ComputeEntitlements["plan"] | null = null): PlanMemory { | |
| 76 | return { get: async () => plan, put: async () => undefined }; | |
| 77 | } | |
| 78 | ||
| 79 | const quiet = () => undefined; | |
| 80 | ||
| 81 | function wire(e: ComputeEntitlements) { | |
| 82 | // As billing serialises it: camelCase. | |
| 83 | return { workspace: "acme", ...e }; | |
| 84 | } | |
| 85 | ||
| 86 | const request = (kind: ComputeKind, isPublic = false) => ({ | |
| 87 | workspace: "acme", | |
| 88 | repo, | |
| 89 | public: isPublic, | |
| 90 | kind, | |
| 91 | estimateMicros: 100_000, | |
| 92 | }); | |
| 93 | ||
| 94 | // ---- Gating decisions --------------------------------------------------------------- | |
| 95 | ||
| 96 | test("a paid workspace's start is reserved and goes ahead", async () => { | |
| 97 | const { binding, asked } = billing({ entitlements: wire(ent()) }); | |
| 98 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); | |
| 99 | assert.equal(admitted.ok, true); | |
| 100 | assert.deepEqual(admitted.ok && admitted.reservation, { id: "rsv_1", paidBy: "credit" }); | |
| 101 | const reserve = asked.find((call) => call.method === "reserve")!; | |
| 102 | assert.deepEqual(reserve.body, { workspace: "acme", repo, public: false, kind: "agent", estimateMicros: 100_000 }); | |
| 103 | }); | |
| 104 | ||
| 105 | test("billing's refusal is shown in its own words", async () => { | |
| 106 | for (const code of ["not_paid", "trial_used", "limit", "oss_pool_empty"] as const) { | |
| 107 | const { binding } = billing({ | |
| 108 | entitlements: wire(ent({ plan: "free", compute: false })), | |
| 109 | reserve: { ok: false, error: { code, message: `Refused: ${code}. /acme/-/billing` } }, | |
| 110 | }); | |
| 111 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("check", true)); | |
| 112 | assert.equal(admitted.ok, false, code); | |
| 113 | assert.equal(!admitted.ok && admitted.code, code); | |
| 114 | assert.equal(!admitted.ok && admitted.message, `Refused: ${code}. /acme/-/billing`); | |
| 115 | } | |
| 116 | }); | |
| 117 | ||
| 118 | test("a refusal without words gets g1t's, with what to do and where", async () => { | |
| 119 | const { binding } = billing({ | |
| 120 | entitlements: wire(ent({ plan: "free", compute: false })), | |
| 121 | reserve: { ok: false, error: { code: "not_paid", message: "" } }, | |
| 122 | }); | |
| 123 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); | |
| 124 | assert.equal(!admitted.ok && admitted.message, refusalMessage("not_paid", "acme", "agent")); | |
| 125 | assert.match(refusalMessage("not_paid", "acme", "agent"), /^Agents need a paid workspace\. Start the \$20 plan or try it with \$5 of free usage after a card check: \/acme\/-\/billing$/); | |
| 126 | }); | |
| 127 | ||
| 128 | test("a paused workspace starts nothing, and billing is not asked to reserve", async () => { | |
| 129 | const { binding, asked } = billing({ entitlements: wire(ent({ paused: "A spend spike is waiting for an owner." })) }); | |
| 130 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("workflow")); | |
| 131 | assert.equal(!admitted.ok && admitted.code, "paused"); | |
| 132 | assert.match(!admitted.ok ? admitted.message : "", /A spend spike is waiting for an owner/); | |
| 133 | assert.equal(asked.some((call) => call.method === "reserve"), false); | |
| 134 | }); | |
| 135 | ||
| 136 | test("internal and enterprise plans pass even when billing refuses", async () => { | |
| 137 | for (const plan of ["internal", "enterprise"] as const) { | |
| 138 | const { binding } = billing({ | |
| 139 | entitlements: wire(ent({ plan })), | |
| 140 | reserve: { ok: false, error: { code: "limit", message: "Over." } }, | |
| 141 | }); | |
| 142 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); | |
| 143 | assert.equal(admitted.ok, true, plan); | |
| 144 | } | |
| 145 | assert.equal(alwaysPasses("internal") && alwaysPasses("enterprise") && !alwaysPasses("paid"), true); | |
| 146 | }); | |
| 147 | ||
| 148 | test("billing down: free workspaces fail closed, paying ones go on", async () => { | |
| 149 | const down = new Set(["entitlements", "reserve"]); | |
| 150 | // Nothing known about the workspace: treated as free. | |
| 151 | let gate = new ComputeGate(billing({ fail: down }).binding, memory(null), quiet); | |
| 152 | let admitted = await gate.admit(request("agent")); | |
| 153 | assert.equal(!admitted.ok && admitted.code, "billing_unavailable"); | |
| 154 | // Last seen paid: goes ahead, without a reservation. | |
| 155 | gate = new ComputeGate(billing({ fail: down }).binding, memory("paid"), quiet); | |
| 156 | admitted = await gate.admit(request("agent")); | |
| 157 | assert.deepEqual(admitted, { ok: true, reservation: null, entitlements: null }); | |
| 158 | // Entitlements answer, reserve errors: the plan decides. | |
| 159 | gate = new ComputeGate(billing({ entitlements: wire(ent({ plan: "free", compute: false, trialMicrosLeft: 5_000_000 })), fail: new Set(["reserve"]) }).binding, memory(), quiet); | |
| 160 | admitted = await gate.admit(request("agent")); | |
| 161 | assert.equal(admitted.ok, false); | |
| 162 | gate = new ComputeGate(billing({ entitlements: wire(ent()), fail: new Set(["reserve"]) }).binding, memory(), quiet); | |
| 163 | assert.equal((await gate.admit(request("agent"))).ok, true); | |
| 164 | assert.equal(onBillingError("free"), "refuse"); | |
| 165 | assert.equal(onBillingError(null), "refuse"); | |
| 166 | assert.equal(onBillingError("paid"), "allow"); | |
| 167 | }); | |
| 168 | ||
| 169 | test("an answer that is not a refusal counts as billing being down", async () => { | |
| 170 | const { binding } = billing({ | |
| 171 | entitlements: wire(ent({ plan: "free", compute: false })), | |
| 172 | reserve: { ok: false, error: { code: "invalid", message: "Unknown method" } }, | |
| 173 | }); | |
| 174 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); | |
| 175 | assert.equal(!admitted.ok && admitted.code, "billing_unavailable"); | |
| 176 | assert.equal(refusalCode({ code: "payment_required" }), "not_paid"); | |
| 177 | assert.equal(refusalCode({ code: "conflict", reason: "trial_used" }), "trial_used"); | |
| 178 | assert.equal(refusalCode({ code: "invalid" }), null); | |
| 179 | }); | |
| 180 | ||
| 181 | test("entitlements read in either case, and not at all without a plan", () => { | |
| 182 | const snake = readEntitlements({ plan: "free", trial_micros_left: 5, trial_verified: true, max_concurrent_agents: 2, paused: "" }); | |
| 183 | assert.equal(snake?.trialMicrosLeft, 5); | |
| 184 | assert.equal(snake?.maxConcurrentAgents, 2); | |
| 185 | assert.equal(snake?.paused, null); | |
| 186 | assert.equal(readEntitlements({ team: true })?.plan, undefined); | |
| 187 | assert.equal(readEntitlements({ ok: true, value: { plan: "paid" } })?.plan, "paid"); | |
| 188 | }); | |
| 189 | ||
| 190 | test("before they try: what a free workspace can start, by kind and repository", () => { | |
| 191 | const free = ent({ plan: "free", compute: false, trialVerified: false }); | |
| 192 | assert.equal(localRefusal(free, "agent", false), "not_paid"); | |
| 193 | assert.equal(localRefusal(free, "check", true), "not_paid", "the pool needs a card check"); | |
| 194 | const verified = ent({ plan: "free", compute: false, trialVerified: true, trialMicrosLeft: 0 }); | |
| 195 | // The open-source path: public repositories' checks, workflows and queue. | |
| 196 | for (const kind of ["check", "workflow", "queue"] as const) assert.equal(localRefusal(verified, kind, true), null, kind); | |
| 197 | for (const kind of ["agent", "deploy", "embedding"] as const) assert.equal(localRefusal(verified, kind, true), "trial_used", kind); | |
| 198 | assert.equal(localRefusal(verified, "check", false), "trial_used"); | |
| 199 | const trial = ent({ plan: "free", compute: true, trialVerified: true, trialMicrosLeft: 4_000_000 }); | |
| 200 | assert.equal(localRefusal(trial, "agent", false), null); | |
| 201 | assert.equal(localRefusal(ent(), "deploy", false), null); | |
| 202 | assert.equal(localRefusal(ent({ paused: "Held." }), "agent", false), "paused"); | |
| 203 | }); | |
| 204 | ||
| 205 | // ---- Estimates ------------------------------------------------------------------------ | |
| 206 | ||
| 207 | test("an agent's estimate is its model's average plus its sandbox for its time cap", () => { | |
| 208 | assert.equal(sandboxEstimateMicros(60, 25), 90_000); | |
| 209 | assert.equal(agentEstimateMicros("implement", 90, 25), 100_000 + 135_000); | |
| 210 | assert.equal(agentEstimateMicros("review", 30, 25), 70_000 + 45_000); | |
| 211 | assert.equal(agentEstimateMicros("plan", 30, 20), 100_000 + 36_000); | |
| 212 | // The workspace's own provider pays for the model. | |
| 213 | assert.equal(agentEstimateMicros("implement", 90, 25, true), 135_000); | |
| 214 | assert.equal(MODEL_ESTIMATE_MICROS.implement, 100_000); | |
| 215 | assert.equal(embeddingEstimateMicros(1_000_000), 67_000); | |
| 216 | assert.equal(sandboxEstimateMicros(-5, 25), 0); | |
| 217 | }); | |
| 218 | ||
| 219 | test("what work cost: its seconds, plus its model in dollars", () => { | |
| 220 | assert.equal(actualMicros(600, 20), 12_000); | |
| 221 | assert.equal(actualMicros(600, 20, 0.094), 12_000 + 94_000); | |
| 222 | assert.equal(actualMicros(10, 20, Number.NaN), 200); | |
| 223 | }); | |
| 224 | ||
| 225 | test("the gate reads the sandbox price from the price book", async () => { | |
| 226 | const gate = new ComputeGate(billing({}).binding, memory(), quiet); | |
| 227 | assert.equal(await gate.microsPerSecond(), 20); | |
| 228 | const down = new ComputeGate(billing({ fail: new Set(["prices"]) }).binding, memory(), quiet); | |
| 229 | assert.equal(await down.microsPerSecond(), 25); | |
| 230 | }); | |
| 231 | ||
| 232 | // ---- Caps ------------------------------------------------------------------------------- | |
| 233 | ||
| 234 | test("caps are the lower of the guardrails' and the plan's", () => { | |
| 235 | assert.equal(lowerCap(90, 60), 60); | |
| 236 | assert.equal(lowerCap(30, 60), 30); | |
| 237 | assert.equal(lowerCap(null, 60), 60); | |
| 238 | assert.equal(lowerCap(0, 0), null); | |
| 239 | assert.equal(runMinutes(90, ent({ maxRunMinutes: 60 })), 60); | |
| 240 | assert.equal(runMinutes(45, ent({ maxRunMinutes: 0 })), 45); | |
| 241 | assert.equal(runBudgetUsd(5, ent({ runCapMicros: 2_000_000 })), 2); | |
| 242 | assert.equal(runBudgetUsd(1, ent({ runCapMicros: 2_000_000 })), 1); | |
| 243 | assert.equal(runBudgetUsd(null, ent({ runCapMicros: 0 })), null); | |
| 244 | const guard = { policy: { budgetUsd: 5 } as never, minutes: 90 }; | |
| 245 | const limited = withPlanLimits(guard as never, { minutes: 60, budgetUsd: 2 }); | |
| 246 | assert.equal(limited.minutes, 60); | |
| 247 | assert.equal((limited.policy as { budgetUsd: number }).budgetUsd, 2); | |
| 248 | const unlimited = withPlanLimits(guard as never, { minutes: null, budgetUsd: null }); | |
| 249 | assert.equal(unlimited.minutes, 90); | |
| 250 | assert.equal((unlimited.policy as { budgetUsd: number }).budgetUsd, 5); | |
| 251 | }); | |
| 252 | ||
| 253 | test("agents at once: a run waits for a free slot past the plan's cap", () => { | |
| 254 | const first = ent({ firstMonth: true, maxConcurrentAgents: 2 }); | |
| 255 | assert.equal(slotFree(1, first), true); | |
| 256 | assert.equal(slotFree(2, first), false); | |
| 257 | assert.equal(slotFree(50, ent({ plan: "internal", maxConcurrentAgents: 2 })), true); | |
| 258 | assert.equal(slotFree(50, ent({ maxConcurrentAgents: 0 })), true); | |
| 259 | assert.equal(slotFree(50, null), true); | |
| 260 | const said = waitingMessage(2); | |
| 261 | assert.equal(isWaiting(said), true); | |
| 262 | assert.match(said, /runs 2 agents at a time/); | |
| 263 | assert.equal(isWaiting("Agents need a paid workspace."), false); | |
| 264 | // mentions.ts keeps its own copy, to have no runtime imports. | |
| 265 | assert.equal(WAITING, WAITING_PREFIX); | |
| 266 | }); | |
| 267 | ||
| 268 | test("an issue's agents stop at its cap, with a way to raise it", () => { | |
| 269 | assert.equal(issueCapReached(9_999_999, ent(), 12), null); | |
| 270 | const capped = issueCapReached(10_000_000, ent(), 12); | |
| 271 | assert.match(capped ?? "", /#12 have spent \$10\.00, its cap of \$10\.00/); | |
| 272 | assert.equal(issueCapReached(50_000_000, ent({ issueCapMicros: 0 }), 12), null); | |
| 273 | assert.equal(issueCapReached(50_000_000, ent({ plan: "internal" }), 12), null); | |
| 274 | assert.match(refusalMessage("issue_cap", "acme", "agent", capped), /raise the cap per issue: \/acme\/-\/billing#caps$/); | |
| 275 | }); | |
| 276 | ||
| 277 | // ---- Builds' network --------------------------------------------------------------------- | |
| 278 | ||
| 279 | test("builds reach registries and git hosts, and no mining pool", () => { | |
| 280 | for (const host of ["registry.npmjs.org", "codeload.github.com", "nodejs.org", "crates.io"]) { | |
| 281 | assert.ok(BUILD_HOSTS.includes(host), host); | |
| 282 | } | |
| 283 | assert.ok(buildHosts("deploy").includes("api.cloudflare.com")); | |
| 284 | assert.ok(!buildHosts("actions").includes("api.cloudflare.com")); | |
| 285 | for (const host of buildHosts("deploy")) { | |
| 286 | assert.doesNotMatch(host, /pool|xmr|monero|nicehash|^\*/, host); | |
| 287 | } | |
| 288 | }); | |
| 289 | ||
| 290 | // ---- Mentions -------------------------------------------------------------------------------- | |
| 291 | ||
| 292 | test("a mention whose run waits for a slot says so, and is not a failure", async () => { | |
| 293 | const replies: string[] = []; | |
| 294 | const recorded: string[] = []; | |
| 295 | const job = { | |
| 296 | commentId: "cmt_1", | |
| 297 | actor: { id: "u1", username: "ana" }, | |
| 298 | repo, | |
| 299 | number: 3, | |
| 300 | member: true, | |
| 301 | pull: null, | |
| 302 | intent: "work", | |
| 303 | issueOpen: true, | |
| 304 | workingPull: null, | |
| 305 | body: "@g1t-agent take this", | |
| 306 | defaultBranch: "main", | |
| 307 | } as never; | |
| 308 | await handleMention(job, { | |
| 309 | mentions: { replyMention: async (_id: string, text: string) => (replies.push(text), true) } as never, | |
| 310 | refusal: async () => null, | |
| 311 | assign: async () => ({ ok: false, error: { code: "conflict", message: waitingMessage(2) } }), | |
| 312 | revise: async () => null, | |
| 313 | review: async () => ({ ok: true, value: true }), | |
| 314 | answer: async () => ({ ok: true, value: true }), | |
| 315 | message: async () => ({ ok: true, value: true }), | |
| 316 | record: async (_job, why) => void recorded.push(why), | |
| 317 | }); | |
| 318 | assert.equal(recorded.length, 0); | |
| 319 | assert.equal(replies.length, 1); | |
| 320 | assert.match(replies[0], /^@ana, Waiting for a free slot/); | |
| 321 | }); |