Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/runner/src/gate.test.ts

338 lines15,994 bytesCodeBlame
1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import {
5 type ComputeEntitlements,
6 type ComputeKind,
7 type PlanMemory,
8 ComputeGate,
9 MODEL_ESTIMATE_MICROS,
10 WAITING_PREFIX,
11 actualMicros,
12 agentEstimateMicros,
13 alwaysPasses,
14 embeddingEstimateMicros,
15 isWaiting,
16 issueCapReached,
17 localRefusal,
18 lowerCap,
19 onBillingError,
20 readEntitlements,
21 refusalCode,
22 refusalMessage,
23 runBudgetUsd,
24 runMinutes,
25 sandboxEstimateMicros,
26 slotFree,
27 waitingMessage,
28} from "../../../packages/contracts/src/compute.ts";
29import { BUILD_HOSTS, buildHosts, withPlanLimits } from "./egress.ts";
30import { WAITING, handleMention } from "./mentions.ts";
31
32const repo = { namespace: "acme", name: "web" };
33
34function ent(change: Partial<ComputeEntitlements> = {}): ComputeEntitlements {
35 return {
36 plan: "paid",
37 compute: true,
38 trialMicrosLeft: 0,
39 trialVerified: true,
40 firstMonth: false,
41 maxConcurrentAgents: 10,
42 maxRunMinutes: 0,
43 runCapMicros: 2_000_000,
44 issueCapMicros: 10_000_000,
45 ceilingMicros: 100_000_000,
46 exposureMicros: 0,
47 paused: null,
48 ...change,
49 };
50}
51
52/** A billing service that answers as told, and records what it was asked. */
53function billing(answers: {
54 entitlements?: unknown;
55 reserve?: unknown;
56 fail?: Set<string>;
57}) {
58 const asked: { method: string; body: Record<string, unknown> }[] = [];
59 const binding = {
60 async fetch(url: string, init?: RequestInit): Promise<Response> {
61 const method = url.split("/rpc/")[1];
62 asked.push({ method, body: JSON.parse(String(init?.body ?? "{}")) });
63 if (answers.fail?.has(method)) return new Response("down", { status: 503 });
64 if (method === "entitlements") return Response.json(answers.entitlements ?? null);
65 if (method === "reserve") return Response.json(answers.reserve ?? { ok: true, value: { id: "rsv_1", paidBy: "credit" } });
66 if (method === "settle") return Response.json({ ok: true, value: true });
67 if (method === "prices") return Response.json({ prices: [{ meter: "sandbox_second", costMicros: 20 }] });
68 return new Response("not found", { status: 404 });
69 },
70 };
71 return { binding, asked };
72}
73
74/** A plan memory that remembers nothing unless told. */
75function memory(plan: ComputeEntitlements["plan"] | null = null): PlanMemory {
76 return { get: async () => plan, put: async () => undefined };
77}
78
79const quiet = () => undefined;
80
81function wire(e: ComputeEntitlements) {
82 // As billing serialises it: camelCase.
83 return { workspace: "acme", ...e };
84}
85
86const request = (kind: ComputeKind, isPublic = false) => ({
87 workspace: "acme",
88 repo,
89 public: isPublic,
90 kind,
91 estimateMicros: 100_000,
92});
93
94// ---- Gating decisions ---------------------------------------------------------------
95
96test("a paid workspace's start is reserved and goes ahead", async () => {
97 const { binding, asked } = billing({ entitlements: wire(ent()) });
98 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
99 assert.equal(admitted.ok, true);
100 assert.deepEqual(admitted.ok && admitted.reservation, { id: "rsv_1", paidBy: "credit" });
101 const reserve = asked.find((call) => call.method === "reserve")!;
102 assert.deepEqual(reserve.body, { workspace: "acme", repo, public: false, kind: "agent", estimateMicros: 100_000 });
103});
104
105test("billing's refusal is shown in its own words", async () => {
106 for (const code of ["not_paid", "trial_used", "limit", "oss_pool_empty"] as const) {
107 const { binding } = billing({
108 entitlements: wire(ent({ plan: "free", compute: false })),
109 reserve: { ok: false, error: { code, message: `Refused: ${code}. /acme/-/billing` } },
110 });
111 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("check", true));
112 assert.equal(admitted.ok, false, code);
113 assert.equal(!admitted.ok && admitted.code, code);
114 assert.equal(!admitted.ok && admitted.message, `Refused: ${code}. /acme/-/billing`);
115 }
116});
117
118test("a refusal without words gets g1t's, with what to do and where", async () => {
119 const { binding } = billing({
120 entitlements: wire(ent({ plan: "free", compute: false })),
121 reserve: { ok: false, error: { code: "not_paid", message: "" } },
122 });
123 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
124 assert.equal(!admitted.ok && admitted.message, refusalMessage("not_paid", "acme", "agent"));
125 assert.match(refusalMessage("not_paid", "acme", "agent"), /^Agents need a paid workspace\. Start the \$20 plan or try it with \$5 of free usage after a card check: \/acme\/-\/billing$/);
126});
127
128test("a paused workspace starts nothing, and billing is not asked to reserve", async () => {
129 const { binding, asked } = billing({ entitlements: wire(ent({ paused: "A spend spike is waiting for an owner." })) });
130 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("workflow"));
131 assert.equal(!admitted.ok && admitted.code, "paused");
132 assert.match(!admitted.ok ? admitted.message : "", /A spend spike is waiting for an owner/);
133 assert.equal(asked.some((call) => call.method === "reserve"), false);
134});
135
136test("internal and enterprise plans pass even when billing refuses", async () => {
137 for (const plan of ["internal", "enterprise"] as const) {
138 const { binding } = billing({
139 entitlements: wire(ent({ plan })),
140 reserve: { ok: false, error: { code: "limit", message: "Over." } },
141 });
142 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
143 assert.equal(admitted.ok, true, plan);
144 }
145 assert.equal(alwaysPasses("internal") && alwaysPasses("enterprise") && !alwaysPasses("paid"), true);
146});
147
148test("a pause from billing holds for every plan: g1t's own budget and its daily breaker", async () => {
149 for (const plan of ["internal", "enterprise", "paid"] as const) {
150 const message = "flagon-io's monthly budget for g1t's own agents is used up ($150.00 of $150.00 this month at cost), so new runs wait.";
151 const { binding, asked } = billing({ entitlements: wire(ent({ plan })), reserve: { ok: false, error: { code: "paused", message } } });
152 const admitted = await new ComputeGate(binding, memory(), quiet).admit({ ...request("agent"), hostedModel: true });
153 assert.equal(!admitted.ok && admitted.code, "paused", plan);
154 assert.equal(!admitted.ok && admitted.message, message);
155 // Billing hears whether the run is on g1t's hosted models.
156 assert.equal(asked.find((call) => call.method === "reserve")!.body.hostedModel, true);
157 }
158 // Billing's pause reason reads cleanly inside g1t's sentence.
159 assert.equal(
160 refusalMessage("paused", "acme", "agent", "Its budget is used up."),
161 "g1t paused compute for this workspace: Its budget is used up. Contact support@g1t.sh to have it looked at.",
162 );
163});
164
165test("billing down: free workspaces fail closed, paying ones go on", async () => {
166 const down = new Set(["entitlements", "reserve"]);
167 // Nothing known about the workspace: treated as free.
168 let gate = new ComputeGate(billing({ fail: down }).binding, memory(null), quiet);
169 let admitted = await gate.admit(request("agent"));
170 assert.equal(!admitted.ok && admitted.code, "billing_unavailable");
171 // Last seen paid: goes ahead, without a reservation.
172 gate = new ComputeGate(billing({ fail: down }).binding, memory("paid"), quiet);
173 admitted = await gate.admit(request("agent"));
174 assert.deepEqual(admitted, { ok: true, reservation: null, entitlements: null });
175 // Entitlements answer, reserve errors: the plan decides.
176 gate = new ComputeGate(billing({ entitlements: wire(ent({ plan: "free", compute: false, trialMicrosLeft: 5_000_000 })), fail: new Set(["reserve"]) }).binding, memory(), quiet);
177 admitted = await gate.admit(request("agent"));
178 assert.equal(admitted.ok, false);
179 gate = new ComputeGate(billing({ entitlements: wire(ent()), fail: new Set(["reserve"]) }).binding, memory(), quiet);
180 assert.equal((await gate.admit(request("agent"))).ok, true);
181 assert.equal(onBillingError("free"), "refuse");
182 assert.equal(onBillingError(null), "refuse");
183 assert.equal(onBillingError("paid"), "allow");
184});
185
186test("an answer that is not a refusal counts as billing being down", async () => {
187 const { binding } = billing({
188 entitlements: wire(ent({ plan: "free", compute: false })),
189 reserve: { ok: false, error: { code: "invalid", message: "Unknown method" } },
190 });
191 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
192 assert.equal(!admitted.ok && admitted.code, "billing_unavailable");
193 assert.equal(refusalCode({ code: "payment_required" }), "not_paid");
194 assert.equal(refusalCode({ code: "conflict", reason: "trial_used" }), "trial_used");
195 assert.equal(refusalCode({ code: "invalid" }), null);
196});
197
198test("entitlements read in either case, and not at all without a plan", () => {
199 const snake = readEntitlements({ plan: "free", trial_micros_left: 5, trial_verified: true, max_concurrent_agents: 2, paused: "" });
200 assert.equal(snake?.trialMicrosLeft, 5);
201 assert.equal(snake?.maxConcurrentAgents, 2);
202 assert.equal(snake?.paused, null);
203 assert.equal(readEntitlements({ team: true })?.plan, undefined);
204 assert.equal(readEntitlements({ ok: true, value: { plan: "paid" } })?.plan, "paid");
205});
206
207test("before they try: what a free workspace can start, by kind and repository", () => {
208 const free = ent({ plan: "free", compute: false, trialVerified: false });
209 assert.equal(localRefusal(free, "agent", false), "not_paid");
210 assert.equal(localRefusal(free, "check", true), "not_paid", "the pool needs a card check");
211 const verified = ent({ plan: "free", compute: false, trialVerified: true, trialMicrosLeft: 0 });
212 // The open-source path: public repositories' checks, workflows and queue.
213 for (const kind of ["check", "workflow", "queue"] as const) assert.equal(localRefusal(verified, kind, true), null, kind);
214 for (const kind of ["agent", "deploy", "embedding"] as const) assert.equal(localRefusal(verified, kind, true), "trial_used", kind);
215 assert.equal(localRefusal(verified, "check", false), "trial_used");
216 const trial = ent({ plan: "free", compute: true, trialVerified: true, trialMicrosLeft: 4_000_000 });
217 assert.equal(localRefusal(trial, "agent", false), null);
218 assert.equal(localRefusal(ent(), "deploy", false), null);
219 assert.equal(localRefusal(ent({ paused: "Held." }), "agent", false), "paused");
220});
221
222// ---- Estimates ------------------------------------------------------------------------
223
224test("an agent's estimate is its model's average plus its sandbox for its time cap", () => {
225 assert.equal(sandboxEstimateMicros(60, 25), 90_000);
226 assert.equal(agentEstimateMicros("implement", 90, 25), 100_000 + 135_000);
227 assert.equal(agentEstimateMicros("review", 30, 25), 70_000 + 45_000);
228 assert.equal(agentEstimateMicros("plan", 30, 20), 100_000 + 36_000);
229 // The workspace's own provider pays for the model.
230 assert.equal(agentEstimateMicros("implement", 90, 25, true), 135_000);
231 assert.equal(MODEL_ESTIMATE_MICROS.implement, 100_000);
232 assert.equal(embeddingEstimateMicros(1_000_000), 67_000);
233 assert.equal(sandboxEstimateMicros(-5, 25), 0);
234});
235
236test("what work cost: its seconds, plus its model in dollars", () => {
237 assert.equal(actualMicros(600, 20), 12_000);
238 assert.equal(actualMicros(600, 20, 0.094), 12_000 + 94_000);
239 assert.equal(actualMicros(10, 20, Number.NaN), 200);
240});
241
242test("the gate reads the sandbox price from the price book", async () => {
243 const gate = new ComputeGate(billing({}).binding, memory(), quiet);
244 assert.equal(await gate.microsPerSecond(), 20);
245 const down = new ComputeGate(billing({ fail: new Set(["prices"]) }).binding, memory(), quiet);
246 assert.equal(await down.microsPerSecond(), 25);
247});
248
249// ---- Caps -------------------------------------------------------------------------------
250
251test("caps are the lower of the guardrails' and the plan's", () => {
252 assert.equal(lowerCap(90, 60), 60);
253 assert.equal(lowerCap(30, 60), 30);
254 assert.equal(lowerCap(null, 60), 60);
255 assert.equal(lowerCap(0, 0), null);
256 assert.equal(runMinutes(90, ent({ maxRunMinutes: 60 })), 60);
257 assert.equal(runMinutes(45, ent({ maxRunMinutes: 0 })), 45);
258 assert.equal(runBudgetUsd(5, ent({ runCapMicros: 2_000_000 })), 2);
259 assert.equal(runBudgetUsd(1, ent({ runCapMicros: 2_000_000 })), 1);
260 assert.equal(runBudgetUsd(null, ent({ runCapMicros: 0 })), null);
261 const guard = { policy: { budgetUsd: 5 } as never, minutes: 90 };
262 const limited = withPlanLimits(guard as never, { minutes: 60, budgetUsd: 2 });
263 assert.equal(limited.minutes, 60);
264 assert.equal((limited.policy as { budgetUsd: number }).budgetUsd, 2);
265 const unlimited = withPlanLimits(guard as never, { minutes: null, budgetUsd: null });
266 assert.equal(unlimited.minutes, 90);
267 assert.equal((unlimited.policy as { budgetUsd: number }).budgetUsd, 5);
268});
269
270test("agents at once: a run waits for a free slot past the plan's cap", () => {
271 const first = ent({ firstMonth: true, maxConcurrentAgents: 2 });
272 assert.equal(slotFree(1, first), true);
273 assert.equal(slotFree(2, first), false);
274 assert.equal(slotFree(50, ent({ plan: "internal", maxConcurrentAgents: 2 })), true);
275 assert.equal(slotFree(50, ent({ maxConcurrentAgents: 0 })), true);
276 assert.equal(slotFree(50, null), true);
277 const said = waitingMessage(2);
278 assert.equal(isWaiting(said), true);
279 assert.match(said, /runs 2 agents at a time/);
280 assert.equal(isWaiting("Agents need a paid workspace."), false);
281 // mentions.ts keeps its own copy, to have no runtime imports.
282 assert.equal(WAITING, WAITING_PREFIX);
283});
284
285test("an issue's agents stop at its cap, with a way to raise it", () => {
286 assert.equal(issueCapReached(9_999_999, ent(), 12), null);
287 const capped = issueCapReached(10_000_000, ent(), 12);
288 assert.match(capped ?? "", /#12 have spent \$10\.00, its cap of \$10\.00/);
289 assert.equal(issueCapReached(50_000_000, ent({ issueCapMicros: 0 }), 12), null);
290 assert.equal(issueCapReached(50_000_000, ent({ plan: "internal" }), 12), null);
291 assert.match(refusalMessage("issue_cap", "acme", "agent", capped), /raise the cap per issue: \/acme\/-\/billing#caps$/);
292});
293
294// ---- Builds' network ---------------------------------------------------------------------
295
296test("builds reach registries and git hosts, and no mining pool", () => {
297 for (const host of ["registry.npmjs.org", "codeload.github.com", "nodejs.org", "crates.io"]) {
298 assert.ok(BUILD_HOSTS.includes(host), host);
299 }
300 assert.ok(buildHosts("deploy").includes("api.cloudflare.com"));
301 assert.ok(!buildHosts("actions").includes("api.cloudflare.com"));
302 for (const host of buildHosts("deploy")) {
303 assert.doesNotMatch(host, /pool|xmr|monero|nicehash|^\*/, host);
304 }
305});
306
307// ---- Mentions --------------------------------------------------------------------------------
308
309test("a mention whose run waits for a slot says so, and is not a failure", async () => {
310 const replies: string[] = [];
311 const recorded: string[] = [];
312 const job = {
313 commentId: "cmt_1",
314 actor: { id: "u1", username: "ana" },
315 repo,
316 number: 3,
317 member: true,
318 pull: null,
319 intent: "work",
320 issueOpen: true,
321 workingPull: null,
322 body: "@g1t-agent take this",
323 defaultBranch: "main",
324 } as never;
325 await handleMention(job, {
326 mentions: { replyMention: async (_id: string, text: string) => (replies.push(text), true) } as never,
327 refusal: async () => null,
328 assign: async () => ({ ok: false, error: { code: "conflict", message: waitingMessage(2) } }),
329 revise: async () => null,
330 review: async () => ({ ok: true, value: true }),
331 answer: async () => ({ ok: true, value: true }),
332 message: async () => ({ ok: true, value: true }),
333 record: async (_job, why) => void recorded.push(why),
334 });
335 assert.equal(recorded.length, 0);
336 assert.equal(replies.length, 1);
337 assert.match(replies[0], /^@ana, Waiting for a free slot/);
338});