| 1 | import assert from "node:assert/strict"; |
| 2 | import { test } from "node:test"; |
| 3 | |
| 4 | import { |
| 5 | type ComputeEntitlements, |
| 6 | type ComputeKind, |
| 7 | type PlanMemory, |
| 8 | ComputeGate, |
| 9 | MODEL_ESTIMATE_MICROS, |
| 10 | actualMicros, |
| 11 | agentEstimateMicros, |
| 12 | alwaysPasses, |
| 13 | embeddingEstimateMicros, |
| 14 | entitlementsKeptSeconds, |
| 15 | isWaiting, |
| 16 | issueCapReached, |
| 17 | localRefusal, |
| 18 | lowerCap, |
| 19 | onBillingError, |
| 20 | readEntitlements, |
| 21 | refusalCode, |
| 22 | refusalMessage, |
| 23 | runBudgetUsd, |
| 24 | runMinutes, |
| 25 | sandboxEstimateMicros, |
| 26 | slotFree, |
| 27 | waitingMessage, |
| 28 | } from "../../../packages/contracts/src/compute.ts"; |
| 29 | import { BUILD_HOSTS, buildHosts, dockerFor, withPlanLimits } from "./egress.ts"; |
| 30 | |
| 31 | const repo = { namespace: "acme", name: "web" }; |
| 32 | |
| 33 | function ent(change: Partial<ComputeEntitlements> = {}): ComputeEntitlements { |
| 34 | return { |
| 35 | plan: "paid", |
| 36 | compute: true, |
| 37 | trialMicrosLeft: 0, |
| 38 | trialVerified: true, |
| 39 | firstMonth: false, |
| 40 | maxConcurrentAgents: 10, |
| 41 | maxRunMinutes: 0, |
| 42 | runCapMicros: 2_000_000, |
| 43 | issueCapMicros: 10_000_000, |
| 44 | ceilingMicros: 100_000_000, |
| 45 | exposureMicros: 0, |
| 46 | paused: null, |
| 47 | ...change, |
| 48 | }; |
| 49 | } |
| 50 | |
| 51 | /** A billing service that answers as told, and records what it was asked. */ |
| 52 | function billing(answers: { |
| 53 | entitlements?: unknown; |
| 54 | reserve?: unknown; |
| 55 | fail?: Set<string>; |
| 56 | }) { |
| 57 | const asked: { method: string; body: Record<string, unknown> }[] = []; |
| 58 | const binding = { |
| 59 | async fetch(url: string, init?: RequestInit): Promise<Response> { |
| 60 | const method = url.split("/rpc/")[1]; |
| 61 | asked.push({ method, body: JSON.parse(String(init?.body ?? "{}")) }); |
| 62 | if (answers.fail?.has(method)) return new Response("down", { status: 503 }); |
| 63 | if (method === "entitlements") return Response.json(answers.entitlements ?? null); |
| 64 | if (method === "reserve") return Response.json(answers.reserve ?? { ok: true, value: { id: "rsv_1", paidBy: "credit" } }); |
| 65 | if (method === "settle") return Response.json({ ok: true, value: true }); |
| 66 | if (method === "prices") return Response.json({ prices: [{ meter: "sandbox_second", costMicros: 20 }] }); |
| 67 | return new Response("not found", { status: 404 }); |
| 68 | }, |
| 69 | }; |
| 70 | return { binding, asked }; |
| 71 | } |
| 72 | |
| 73 | /** A plan memory that remembers nothing unless told. */ |
| 74 | function memory(plan: ComputeEntitlements["plan"] | null = null): PlanMemory { |
| 75 | return { get: async () => plan, put: async () => undefined }; |
| 76 | } |
| 77 | |
| 78 | const quiet = () => undefined; |
| 79 | |
| 80 | function wire(e: ComputeEntitlements) { |
| 81 | // As billing serialises it: camelCase. |
| 82 | return { workspace: "acme", ...e }; |
| 83 | } |
| 84 | |
| 85 | const request = (kind: ComputeKind, isPublic = false) => ({ |
| 86 | workspace: "acme", |
| 87 | repo, |
| 88 | public: isPublic, |
| 89 | kind, |
| 90 | estimateMicros: 100_000, |
| 91 | }); |
| 92 | |
| 93 | // ---- Gating decisions --------------------------------------------------------------- |
| 94 | |
| 95 | test("a paid workspace's start is reserved and goes ahead", async () => { |
| 96 | const { binding, asked } = billing({ entitlements: wire(ent()) }); |
| 97 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); |
| 98 | assert.equal(admitted.ok, true); |
| 99 | assert.deepEqual(admitted.ok && admitted.reservation, { id: "rsv_1", paidBy: "credit" }); |
| 100 | const reserve = asked.find((call) => call.method === "reserve")!; |
| 101 | assert.deepEqual(reserve.body, { workspace: "acme", repo, public: false, kind: "agent", estimateMicros: 100_000 }); |
| 102 | }); |
| 103 | |
| 104 | test("billing's refusal is shown in its own words", async () => { |
| 105 | for (const code of ["not_paid", "trial_used", "limit", "oss_pool_empty"] as const) { |
| 106 | const { binding } = billing({ |
| 107 | entitlements: wire(ent({ plan: "free", compute: false })), |
| 108 | reserve: { ok: false, error: { code, message: `Refused: ${code}. /acme/-/billing` } }, |
| 109 | }); |
| 110 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("check", true)); |
| 111 | assert.equal(admitted.ok, false, code); |
| 112 | assert.equal(!admitted.ok && admitted.code, code); |
| 113 | assert.equal(!admitted.ok && admitted.message, `Refused: ${code}. /acme/-/billing`); |
| 114 | } |
| 115 | }); |
| 116 | |
| 117 | test("a refusal without words gets g1t's, with what to do and where", async () => { |
| 118 | const { binding } = billing({ |
| 119 | entitlements: wire(ent({ plan: "free", compute: false })), |
| 120 | reserve: { ok: false, error: { code: "not_paid", message: "" } }, |
| 121 | }); |
| 122 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); |
| 123 | assert.equal(!admitted.ok && admitted.message, refusalMessage("not_paid", "acme", "agent")); |
| 124 | assert.match(refusalMessage("not_paid", "acme", "agent"), /^Agents need a paid workspace\. Start the \$20 plan or try it with \$5 of free usage after a card check: \/acme\/-\/billing$/); |
| 125 | }); |
| 126 | |
| 127 | test("a paused workspace starts nothing, and billing is not asked to reserve", async () => { |
| 128 | const { binding, asked } = billing({ entitlements: wire(ent({ paused: "A spend spike is waiting for an owner." })) }); |
| 129 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("workflow")); |
| 130 | assert.equal(!admitted.ok && admitted.code, "paused"); |
| 131 | assert.match(!admitted.ok ? admitted.message : "", /A spend spike is waiting for an owner/); |
| 132 | assert.equal(asked.some((call) => call.method === "reserve"), false); |
| 133 | }); |
| 134 | |
| 135 | test("an answer that holds compute back is kept seconds, one that lets it through half a minute", () => { |
| 136 | assert.equal(entitlementsKeptSeconds(ent()), 30); |
| 137 | assert.equal(entitlementsKeptSeconds(ent({ paused: "A spend spike is waiting for an owner." })), 3); |
| 138 | assert.equal(entitlementsKeptSeconds(ent({ plan: "free", compute: false })), 3); |
| 139 | // A free workspace with compute (its trial) runs, so it is kept as long. |
| 140 | assert.equal(entitlementsKeptSeconds(ent({ plan: "free", compute: true })), 30); |
| 141 | }); |
| 142 | |
| 143 | test("internal and enterprise plans pass even when billing refuses", async () => { |
| 144 | for (const plan of ["internal", "enterprise"] as const) { |
| 145 | const { binding } = billing({ |
| 146 | entitlements: wire(ent({ plan })), |
| 147 | reserve: { ok: false, error: { code: "limit", message: "Over." } }, |
| 148 | }); |
| 149 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); |
| 150 | assert.equal(admitted.ok, true, plan); |
| 151 | } |
| 152 | assert.equal(alwaysPasses("internal") && alwaysPasses("enterprise") && !alwaysPasses("paid"), true); |
| 153 | }); |
| 154 | |
| 155 | test("a pause from billing holds for every plan: g1t's own budget and its daily breaker", async () => { |
| 156 | for (const plan of ["internal", "enterprise", "paid"] as const) { |
| 157 | const message = "flagon-io's monthly budget for g1t's own agents is used up ($150.00 of $150.00 this month at cost), so new runs wait."; |
| 158 | const { binding, asked } = billing({ entitlements: wire(ent({ plan })), reserve: { ok: false, error: { code: "paused", message } } }); |
| 159 | const admitted = await new ComputeGate(binding, memory(), quiet).admit({ ...request("agent"), hostedModel: true }); |
| 160 | assert.equal(!admitted.ok && admitted.code, "paused", plan); |
| 161 | assert.equal(!admitted.ok && admitted.message, message); |
| 162 | // Billing hears whether the run is on g1t's hosted models. |
| 163 | assert.equal(asked.find((call) => call.method === "reserve")!.body.hostedModel, true); |
| 164 | } |
| 165 | // Billing's pause reason reads cleanly inside g1t's sentence. |
| 166 | assert.equal( |
| 167 | refusalMessage("paused", "acme", "agent", "Its budget is used up."), |
| 168 | "g1t paused compute for this workspace: Its budget is used up. Contact support@g1t.sh to have it looked at.", |
| 169 | ); |
| 170 | }); |
| 171 | |
| 172 | test("billing down: free workspaces fail closed, paying ones go on", async () => { |
| 173 | const down = new Set(["entitlements", "reserve"]); |
| 174 | // Nothing known about the workspace: treated as free. |
| 175 | let gate = new ComputeGate(billing({ fail: down }).binding, memory(null), quiet); |
| 176 | let admitted = await gate.admit(request("agent")); |
| 177 | assert.equal(!admitted.ok && admitted.code, "billing_unavailable"); |
| 178 | // Last seen paid: goes ahead, without a reservation. |
| 179 | gate = new ComputeGate(billing({ fail: down }).binding, memory("paid"), quiet); |
| 180 | admitted = await gate.admit(request("agent")); |
| 181 | assert.deepEqual(admitted, { ok: true, reservation: null, entitlements: null }); |
| 182 | // Entitlements answer, reserve errors: the plan decides. |
| 183 | gate = new ComputeGate(billing({ entitlements: wire(ent({ plan: "free", compute: false, trialMicrosLeft: 5_000_000 })), fail: new Set(["reserve"]) }).binding, memory(), quiet); |
| 184 | admitted = await gate.admit(request("agent")); |
| 185 | assert.equal(admitted.ok, false); |
| 186 | gate = new ComputeGate(billing({ entitlements: wire(ent()), fail: new Set(["reserve"]) }).binding, memory(), quiet); |
| 187 | assert.equal((await gate.admit(request("agent"))).ok, true); |
| 188 | assert.equal(onBillingError("free"), "refuse"); |
| 189 | assert.equal(onBillingError(null), "refuse"); |
| 190 | assert.equal(onBillingError("paid"), "allow"); |
| 191 | }); |
| 192 | |
| 193 | test("an answer that is not a refusal counts as billing being down", async () => { |
| 194 | const { binding } = billing({ |
| 195 | entitlements: wire(ent({ plan: "free", compute: false })), |
| 196 | reserve: { ok: false, error: { code: "invalid", message: "Unknown method" } }, |
| 197 | }); |
| 198 | const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent")); |
| 199 | assert.equal(!admitted.ok && admitted.code, "billing_unavailable"); |
| 200 | assert.equal(refusalCode({ code: "payment_required" }), "not_paid"); |
| 201 | assert.equal(refusalCode({ code: "conflict", reason: "trial_used" }), "trial_used"); |
| 202 | assert.equal(refusalCode({ code: "invalid" }), null); |
| 203 | }); |
| 204 | |
| 205 | test("entitlements read in either case, and not at all without a plan", () => { |
| 206 | const snake = readEntitlements({ plan: "free", trial_micros_left: 5, trial_verified: true, max_concurrent_agents: 2, paused: "" }); |
| 207 | assert.equal(snake?.trialMicrosLeft, 5); |
| 208 | assert.equal(snake?.maxConcurrentAgents, 2); |
| 209 | assert.equal(snake?.paused, null); |
| 210 | assert.equal(readEntitlements({ team: true })?.plan, undefined); |
| 211 | assert.equal(readEntitlements({ ok: true, value: { plan: "paid" } })?.plan, "paid"); |
| 212 | }); |
| 213 | |
| 214 | test("before they try: what a free workspace can start, by kind and repository", () => { |
| 215 | const free = ent({ plan: "free", compute: false, trialVerified: false }); |
| 216 | assert.equal(localRefusal(free, "agent", false), "not_paid"); |
| 217 | assert.equal(localRefusal(free, "check", true), "not_paid", "the pool needs a card check"); |
| 218 | const verified = ent({ plan: "free", compute: false, trialVerified: true, trialMicrosLeft: 0 }); |
| 219 | // The open-source path: public repositories' checks, workflows and queue. |
| 220 | for (const kind of ["check", "workflow", "queue"] as const) assert.equal(localRefusal(verified, kind, true), null, kind); |
| 221 | for (const kind of ["agent", "deploy", "embedding"] as const) assert.equal(localRefusal(verified, kind, true), "trial_used", kind); |
| 222 | assert.equal(localRefusal(verified, "check", false), "trial_used"); |
| 223 | const trial = ent({ plan: "free", compute: true, trialVerified: true, trialMicrosLeft: 4_000_000 }); |
| 224 | assert.equal(localRefusal(trial, "agent", false), null); |
| 225 | assert.equal(localRefusal(ent(), "deploy", false), null); |
| 226 | assert.equal(localRefusal(ent({ paused: "Held." }), "agent", false), "paused"); |
| 227 | }); |
| 228 | |
| 229 | // ---- Estimates ------------------------------------------------------------------------ |
| 230 | |
| 231 | test("an agent's estimate is its model's average plus its sandbox for its time cap", () => { |
| 232 | assert.equal(sandboxEstimateMicros(60, 25), 90_000); |
| 233 | assert.equal(agentEstimateMicros("implement", 90, 25), 100_000 + 135_000); |
| 234 | assert.equal(agentEstimateMicros("review", 30, 25), 70_000 + 45_000); |
| 235 | assert.equal(agentEstimateMicros("plan", 30, 20), 100_000 + 36_000); |
| 236 | // The workspace's own provider pays for the model. |
| 237 | assert.equal(agentEstimateMicros("implement", 90, 25, true), 135_000); |
| 238 | assert.equal(MODEL_ESTIMATE_MICROS.implement, 100_000); |
| 239 | assert.equal(embeddingEstimateMicros(1_000_000), 67_000); |
| 240 | assert.equal(sandboxEstimateMicros(-5, 25), 0); |
| 241 | }); |
| 242 | |
| 243 | test("what work cost: its seconds, plus its model in dollars", () => { |
| 244 | assert.equal(actualMicros(600, 20), 12_000); |
| 245 | assert.equal(actualMicros(600, 20, 0.094), 12_000 + 94_000); |
| 246 | assert.equal(actualMicros(10, 20, Number.NaN), 200); |
| 247 | }); |
| 248 | |
| 249 | test("the gate reads the sandbox price from the price book", async () => { |
| 250 | const gate = new ComputeGate(billing({}).binding, memory(), quiet); |
| 251 | assert.equal(await gate.microsPerSecond(), 20); |
| 252 | const down = new ComputeGate(billing({ fail: new Set(["prices"]) }).binding, memory(), quiet); |
| 253 | assert.equal(await down.microsPerSecond(), 25); |
| 254 | }); |
| 255 | |
| 256 | // ---- Caps ------------------------------------------------------------------------------- |
| 257 | |
| 258 | test("caps are the lower of the guardrails' and the plan's", () => { |
| 259 | assert.equal(lowerCap(90, 60), 60); |
| 260 | assert.equal(lowerCap(30, 60), 30); |
| 261 | assert.equal(lowerCap(null, 60), 60); |
| 262 | assert.equal(lowerCap(0, 0), null); |
| 263 | assert.equal(runMinutes(90, ent({ maxRunMinutes: 60 })), 60); |
| 264 | assert.equal(runMinutes(45, ent({ maxRunMinutes: 0 })), 45); |
| 265 | assert.equal(runBudgetUsd(5, ent({ runCapMicros: 2_000_000 })), 2); |
| 266 | assert.equal(runBudgetUsd(1, ent({ runCapMicros: 2_000_000 })), 1); |
| 267 | assert.equal(runBudgetUsd(null, ent({ runCapMicros: 0 })), null); |
| 268 | const guard = { policy: { budgetUsd: 5 } as never, minutes: 90 }; |
| 269 | const limited = withPlanLimits(guard as never, { minutes: 60, budgetUsd: 2 }); |
| 270 | assert.equal(limited.minutes, 60); |
| 271 | assert.equal((limited.policy as { budgetUsd: number }).budgetUsd, 2); |
| 272 | const unlimited = withPlanLimits(guard as never, { minutes: null, budgetUsd: null }); |
| 273 | assert.equal(unlimited.minutes, 90); |
| 274 | assert.equal((unlimited.policy as { budgetUsd: number }).budgetUsd, 5); |
| 275 | }); |
| 276 | |
| 277 | test("agents at once: a run waits for a free slot past the plan's cap", () => { |
| 278 | const first = ent({ firstMonth: true, maxConcurrentAgents: 2 }); |
| 279 | assert.equal(slotFree(1, first), true); |
| 280 | assert.equal(slotFree(2, first), false); |
| 281 | assert.equal(slotFree(50, ent({ plan: "internal", maxConcurrentAgents: 2 })), true); |
| 282 | assert.equal(slotFree(50, ent({ maxConcurrentAgents: 0 })), true); |
| 283 | assert.equal(slotFree(50, null), true); |
| 284 | const said = waitingMessage(2); |
| 285 | assert.equal(isWaiting(said), true); |
| 286 | assert.match(said, /runs 2 agents at a time/); |
| 287 | assert.equal(isWaiting("Agents need a paid workspace."), false); |
| 288 | }); |
| 289 | |
| 290 | test("an issue's agents stop at its cap, with a way to raise it", () => { |
| 291 | assert.equal(issueCapReached(9_999_999, ent(), 12), null); |
| 292 | const capped = issueCapReached(10_000_000, ent(), 12); |
| 293 | assert.match(capped ?? "", /#12 have spent \$10\.00, its cap of \$10\.00/); |
| 294 | assert.equal(issueCapReached(50_000_000, ent({ issueCapMicros: 0 }), 12), null); |
| 295 | assert.equal(issueCapReached(50_000_000, ent({ plan: "internal" }), 12), null); |
| 296 | assert.match(refusalMessage("issue_cap", "acme", "agent", capped), /raise the cap per issue: \/acme\/-\/billing#caps$/); |
| 297 | }); |
| 298 | |
| 299 | // ---- Builds' network --------------------------------------------------------------------- |
| 300 | |
| 301 | test("builds reach registries and git hosts, and no mining pool", () => { |
| 302 | for (const host of ["registry.npmjs.org", "codeload.github.com", "nodejs.org", "crates.io"]) { |
| 303 | assert.ok(BUILD_HOSTS.includes(host), host); |
| 304 | } |
| 305 | // A job's Docker Engine pulls from Docker Hub (through its mirror) and Quay. |
| 306 | for (const host of ["registry-1.docker.io", "auth.docker.io", "production.cloudflare.docker.com", "mirror.gcr.io", "ghcr.io", "quay.io"]) { |
| 307 | assert.ok(BUILD_HOSTS.includes(host), host); |
| 308 | } |
| 309 | assert.ok(buildHosts("deploy").includes("api.cloudflare.com")); |
| 310 | // A container build pushes its image to Cloudflare's registry, from a sandbox or a job. |
| 311 | assert.ok(buildHosts("deploy").includes("registry.cloudflare.com")); |
| 312 | assert.ok(buildHosts("actions").includes("registry.cloudflare.com")); |
| 313 | assert.ok(!buildHosts("actions").includes("api.cloudflare.com")); |
| 314 | for (const host of buildHosts("deploy")) { |
| 315 | assert.doesNotMatch(host, /pool|xmr|monero|nicehash|^\*/, host); |
| 316 | } |
| 317 | }); |
| 318 | |
| 319 | test("workflow jobs get Docker unless the operator turns it off", () => { |
| 320 | assert.equal(dockerFor(undefined), "on"); |
| 321 | assert.equal(dockerFor("on"), "on"); |
| 322 | assert.equal(dockerFor(" OFF "), "off"); |
| 323 | }); |