Skip to content
Sign up

flagon-io/g1t

public
0
323 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

A package's latest version is its highest, not the last one published1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import {
5 type ComputeEntitlements,
6 type ComputeKind,
7 type PlanMemory,
8 ComputeGate,
9 MODEL_ESTIMATE_MICROS,
10 actualMicros,
11 agentEstimateMicros,
12 alwaysPasses,
13 embeddingEstimateMicros,
Merge branch 'worktree-agent-a985c01f43538af01'14 entitlementsKeptSeconds,
A package's latest version is its highest, not the last one published15 isWaiting,
16 issueCapReached,
17 localRefusal,
18 lowerCap,
19 onBillingError,
20 readEntitlements,
21 refusalCode,
22 refusalMessage,
23 runBudgetUsd,
24 runMinutes,
25 sandboxEstimateMicros,
26 slotFree,
27 waitingMessage,
28} from "../../../packages/contracts/src/compute.ts";
Merge branch 'main' into actions-toolkit-oidc-artifacts29import { BUILD_HOSTS, buildHosts, dockerFor, withPlanLimits } from "./egress.ts";
A package's latest version is its highest, not the last one published30
31const repo = { namespace: "acme", name: "web" };
32
33function ent(change: Partial<ComputeEntitlements> = {}): ComputeEntitlements {
34 return {
35 plan: "paid",
36 compute: true,
37 trialMicrosLeft: 0,
38 trialVerified: true,
39 firstMonth: false,
40 maxConcurrentAgents: 10,
41 maxRunMinutes: 0,
42 runCapMicros: 2_000_000,
43 issueCapMicros: 10_000_000,
44 ceilingMicros: 100_000_000,
45 exposureMicros: 0,
46 paused: null,
47 ...change,
48 };
49}
50
51/** A billing service that answers as told, and records what it was asked. */
52function billing(answers: {
53 entitlements?: unknown;
54 reserve?: unknown;
55 fail?: Set<string>;
56}) {
57 const asked: { method: string; body: Record<string, unknown> }[] = [];
58 const binding = {
59 async fetch(url: string, init?: RequestInit): Promise<Response> {
60 const method = url.split("/rpc/")[1];
61 asked.push({ method, body: JSON.parse(String(init?.body ?? "{}")) });
62 if (answers.fail?.has(method)) return new Response("down", { status: 503 });
63 if (method === "entitlements") return Response.json(answers.entitlements ?? null);
64 if (method === "reserve") return Response.json(answers.reserve ?? { ok: true, value: { id: "rsv_1", paidBy: "credit" } });
65 if (method === "settle") return Response.json({ ok: true, value: true });
66 if (method === "prices") return Response.json({ prices: [{ meter: "sandbox_second", costMicros: 20 }] });
67 return new Response("not found", { status: 404 });
68 },
69 };
70 return { binding, asked };
71}
72
73/** A plan memory that remembers nothing unless told. */
74function memory(plan: ComputeEntitlements["plan"] | null = null): PlanMemory {
75 return { get: async () => plan, put: async () => undefined };
76}
77
78const quiet = () => undefined;
79
80function wire(e: ComputeEntitlements) {
81 // As billing serialises it: camelCase.
82 return { workspace: "acme", ...e };
83}
84
85const request = (kind: ComputeKind, isPublic = false) => ({
86 workspace: "acme",
87 repo,
88 public: isPublic,
89 kind,
90 estimateMicros: 100_000,
91});
92
93// ---- Gating decisions ---------------------------------------------------------------
94
95test("a paid workspace's start is reserved and goes ahead", async () => {
96 const { binding, asked } = billing({ entitlements: wire(ent()) });
97 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
98 assert.equal(admitted.ok, true);
99 assert.deepEqual(admitted.ok && admitted.reservation, { id: "rsv_1", paidBy: "credit" });
100 const reserve = asked.find((call) => call.method === "reserve")!;
101 assert.deepEqual(reserve.body, { workspace: "acme", repo, public: false, kind: "agent", estimateMicros: 100_000 });
102});
103
104test("billing's refusal is shown in its own words", async () => {
105 for (const code of ["not_paid", "trial_used", "limit", "oss_pool_empty"] as const) {
106 const { binding } = billing({
107 entitlements: wire(ent({ plan: "free", compute: false })),
108 reserve: { ok: false, error: { code, message: `Refused: ${code}. /acme/-/billing` } },
109 });
110 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("check", true));
111 assert.equal(admitted.ok, false, code);
112 assert.equal(!admitted.ok && admitted.code, code);
113 assert.equal(!admitted.ok && admitted.message, `Refused: ${code}. /acme/-/billing`);
114 }
115});
116
117test("a refusal without words gets g1t's, with what to do and where", async () => {
118 const { binding } = billing({
119 entitlements: wire(ent({ plan: "free", compute: false })),
120 reserve: { ok: false, error: { code: "not_paid", message: "" } },
121 });
122 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
123 assert.equal(!admitted.ok && admitted.message, refusalMessage("not_paid", "acme", "agent"));
124 assert.match(refusalMessage("not_paid", "acme", "agent"), /^Agents need a paid workspace\. Start the \$20 plan or try it with \$5 of free usage after a card check: \/acme\/-\/billing$/);
125});
126
127test("a paused workspace starts nothing, and billing is not asked to reserve", async () => {
128 const { binding, asked } = billing({ entitlements: wire(ent({ paused: "A spend spike is waiting for an owner." })) });
129 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("workflow"));
130 assert.equal(!admitted.ok && admitted.code, "paused");
131 assert.match(!admitted.ok ? admitted.message : "", /A spend spike is waiting for an owner/);
132 assert.equal(asked.some((call) => call.method === "reserve"), false);
133});
134
Merge branch 'worktree-agent-a985c01f43538af01'135test("an answer that holds compute back is kept seconds, one that lets it through half a minute", () => {
136 assert.equal(entitlementsKeptSeconds(ent()), 30);
137 assert.equal(entitlementsKeptSeconds(ent({ paused: "A spend spike is waiting for an owner." })), 3);
138 assert.equal(entitlementsKeptSeconds(ent({ plan: "free", compute: false })), 3);
139 // A free workspace with compute (its trial) runs, so it is kept as long.
140 assert.equal(entitlementsKeptSeconds(ent({ plan: "free", compute: true })), 30);
141});
142
A package's latest version is its highest, not the last one published143test("internal and enterprise plans pass even when billing refuses", async () => {
144 for (const plan of ["internal", "enterprise"] as const) {
145 const { binding } = billing({
146 entitlements: wire(ent({ plan })),
147 reserve: { ok: false, error: { code: "limit", message: "Over." } },
148 });
149 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
150 assert.equal(admitted.ok, true, plan);
151 }
152 assert.equal(alwaysPasses("internal") && alwaysPasses("enterprise") && !alwaysPasses("paid"), true);
153});
154
155test("a pause from billing holds for every plan: g1t's own budget and its daily breaker", async () => {
156 for (const plan of ["internal", "enterprise", "paid"] as const) {
157 const message = "flagon-io's monthly budget for g1t's own agents is used up ($150.00 of $150.00 this month at cost), so new runs wait.";
158 const { binding, asked } = billing({ entitlements: wire(ent({ plan })), reserve: { ok: false, error: { code: "paused", message } } });
159 const admitted = await new ComputeGate(binding, memory(), quiet).admit({ ...request("agent"), hostedModel: true });
160 assert.equal(!admitted.ok && admitted.code, "paused", plan);
161 assert.equal(!admitted.ok && admitted.message, message);
162 // Billing hears whether the run is on g1t's hosted models.
163 assert.equal(asked.find((call) => call.method === "reserve")!.body.hostedModel, true);
164 }
165 // Billing's pause reason reads cleanly inside g1t's sentence.
166 assert.equal(
167 refusalMessage("paused", "acme", "agent", "Its budget is used up."),
168 "g1t paused compute for this workspace: Its budget is used up. Contact support@g1t.sh to have it looked at.",
169 );
170});
171
172test("billing down: free workspaces fail closed, paying ones go on", async () => {
173 const down = new Set(["entitlements", "reserve"]);
174 // Nothing known about the workspace: treated as free.
175 let gate = new ComputeGate(billing({ fail: down }).binding, memory(null), quiet);
176 let admitted = await gate.admit(request("agent"));
177 assert.equal(!admitted.ok && admitted.code, "billing_unavailable");
178 // Last seen paid: goes ahead, without a reservation.
179 gate = new ComputeGate(billing({ fail: down }).binding, memory("paid"), quiet);
180 admitted = await gate.admit(request("agent"));
181 assert.deepEqual(admitted, { ok: true, reservation: null, entitlements: null });
182 // Entitlements answer, reserve errors: the plan decides.
183 gate = new ComputeGate(billing({ entitlements: wire(ent({ plan: "free", compute: false, trialMicrosLeft: 5_000_000 })), fail: new Set(["reserve"]) }).binding, memory(), quiet);
184 admitted = await gate.admit(request("agent"));
185 assert.equal(admitted.ok, false);
186 gate = new ComputeGate(billing({ entitlements: wire(ent()), fail: new Set(["reserve"]) }).binding, memory(), quiet);
187 assert.equal((await gate.admit(request("agent"))).ok, true);
188 assert.equal(onBillingError("free"), "refuse");
189 assert.equal(onBillingError(null), "refuse");
190 assert.equal(onBillingError("paid"), "allow");
191});
192
193test("an answer that is not a refusal counts as billing being down", async () => {
194 const { binding } = billing({
195 entitlements: wire(ent({ plan: "free", compute: false })),
196 reserve: { ok: false, error: { code: "invalid", message: "Unknown method" } },
197 });
198 const admitted = await new ComputeGate(binding, memory(), quiet).admit(request("agent"));
199 assert.equal(!admitted.ok && admitted.code, "billing_unavailable");
200 assert.equal(refusalCode({ code: "payment_required" }), "not_paid");
201 assert.equal(refusalCode({ code: "conflict", reason: "trial_used" }), "trial_used");
202 assert.equal(refusalCode({ code: "invalid" }), null);
203});
204
205test("entitlements read in either case, and not at all without a plan", () => {
206 const snake = readEntitlements({ plan: "free", trial_micros_left: 5, trial_verified: true, max_concurrent_agents: 2, paused: "" });
207 assert.equal(snake?.trialMicrosLeft, 5);
208 assert.equal(snake?.maxConcurrentAgents, 2);
209 assert.equal(snake?.paused, null);
210 assert.equal(readEntitlements({ team: true })?.plan, undefined);
211 assert.equal(readEntitlements({ ok: true, value: { plan: "paid" } })?.plan, "paid");
212});
213
214test("before they try: what a free workspace can start, by kind and repository", () => {
215 const free = ent({ plan: "free", compute: false, trialVerified: false });
216 assert.equal(localRefusal(free, "agent", false), "not_paid");
217 assert.equal(localRefusal(free, "check", true), "not_paid", "the pool needs a card check");
218 const verified = ent({ plan: "free", compute: false, trialVerified: true, trialMicrosLeft: 0 });
219 // The open-source path: public repositories' checks, workflows and queue.
220 for (const kind of ["check", "workflow", "queue"] as const) assert.equal(localRefusal(verified, kind, true), null, kind);
221 for (const kind of ["agent", "deploy", "embedding"] as const) assert.equal(localRefusal(verified, kind, true), "trial_used", kind);
222 assert.equal(localRefusal(verified, "check", false), "trial_used");
223 const trial = ent({ plan: "free", compute: true, trialVerified: true, trialMicrosLeft: 4_000_000 });
224 assert.equal(localRefusal(trial, "agent", false), null);
225 assert.equal(localRefusal(ent(), "deploy", false), null);
226 assert.equal(localRefusal(ent({ paused: "Held." }), "agent", false), "paused");
227});
228
229// ---- Estimates ------------------------------------------------------------------------
230
231test("an agent's estimate is its model's average plus its sandbox for its time cap", () => {
232 assert.equal(sandboxEstimateMicros(60, 25), 90_000);
233 assert.equal(agentEstimateMicros("implement", 90, 25), 100_000 + 135_000);
234 assert.equal(agentEstimateMicros("review", 30, 25), 70_000 + 45_000);
235 assert.equal(agentEstimateMicros("plan", 30, 20), 100_000 + 36_000);
236 // The workspace's own provider pays for the model.
237 assert.equal(agentEstimateMicros("implement", 90, 25, true), 135_000);
238 assert.equal(MODEL_ESTIMATE_MICROS.implement, 100_000);
239 assert.equal(embeddingEstimateMicros(1_000_000), 67_000);
240 assert.equal(sandboxEstimateMicros(-5, 25), 0);
241});
242
243test("what work cost: its seconds, plus its model in dollars", () => {
244 assert.equal(actualMicros(600, 20), 12_000);
245 assert.equal(actualMicros(600, 20, 0.094), 12_000 + 94_000);
246 assert.equal(actualMicros(10, 20, Number.NaN), 200);
247});
248
249test("the gate reads the sandbox price from the price book", async () => {
250 const gate = new ComputeGate(billing({}).binding, memory(), quiet);
251 assert.equal(await gate.microsPerSecond(), 20);
252 const down = new ComputeGate(billing({ fail: new Set(["prices"]) }).binding, memory(), quiet);
253 assert.equal(await down.microsPerSecond(), 25);
254});
255
256// ---- Caps -------------------------------------------------------------------------------
257
258test("caps are the lower of the guardrails' and the plan's", () => {
259 assert.equal(lowerCap(90, 60), 60);
260 assert.equal(lowerCap(30, 60), 30);
261 assert.equal(lowerCap(null, 60), 60);
262 assert.equal(lowerCap(0, 0), null);
263 assert.equal(runMinutes(90, ent({ maxRunMinutes: 60 })), 60);
264 assert.equal(runMinutes(45, ent({ maxRunMinutes: 0 })), 45);
265 assert.equal(runBudgetUsd(5, ent({ runCapMicros: 2_000_000 })), 2);
266 assert.equal(runBudgetUsd(1, ent({ runCapMicros: 2_000_000 })), 1);
267 assert.equal(runBudgetUsd(null, ent({ runCapMicros: 0 })), null);
268 const guard = { policy: { budgetUsd: 5 } as never, minutes: 90 };
269 const limited = withPlanLimits(guard as never, { minutes: 60, budgetUsd: 2 });
270 assert.equal(limited.minutes, 60);
271 assert.equal((limited.policy as { budgetUsd: number }).budgetUsd, 2);
272 const unlimited = withPlanLimits(guard as never, { minutes: null, budgetUsd: null });
273 assert.equal(unlimited.minutes, 90);
274 assert.equal((unlimited.policy as { budgetUsd: number }).budgetUsd, 5);
275});
276
277test("agents at once: a run waits for a free slot past the plan's cap", () => {
278 const first = ent({ firstMonth: true, maxConcurrentAgents: 2 });
279 assert.equal(slotFree(1, first), true);
280 assert.equal(slotFree(2, first), false);
281 assert.equal(slotFree(50, ent({ plan: "internal", maxConcurrentAgents: 2 })), true);
282 assert.equal(slotFree(50, ent({ maxConcurrentAgents: 0 })), true);
283 assert.equal(slotFree(50, null), true);
284 const said = waitingMessage(2);
285 assert.equal(isWaiting(said), true);
286 assert.match(said, /runs 2 agents at a time/);
287 assert.equal(isWaiting("Agents need a paid workspace."), false);
288});
289
290test("an issue's agents stop at its cap, with a way to raise it", () => {
291 assert.equal(issueCapReached(9_999_999, ent(), 12), null);
292 const capped = issueCapReached(10_000_000, ent(), 12);
293 assert.match(capped ?? "", /#12 have spent \$10\.00, its cap of \$10\.00/);
294 assert.equal(issueCapReached(50_000_000, ent({ issueCapMicros: 0 }), 12), null);
295 assert.equal(issueCapReached(50_000_000, ent({ plan: "internal" }), 12), null);
296 assert.match(refusalMessage("issue_cap", "acme", "agent", capped), /raise the cap per issue: \/acme\/-\/billing#caps$/);
297});
298
299// ---- Builds' network ---------------------------------------------------------------------
300
301test("builds reach registries and git hosts, and no mining pool", () => {
302 for (const host of ["registry.npmjs.org", "codeload.github.com", "nodejs.org", "crates.io"]) {
303 assert.ok(BUILD_HOSTS.includes(host), host);
304 }
Merge branch 'main' into actions-toolkit-oidc-artifacts305 // A job's Docker Engine pulls from Docker Hub (through its mirror) and Quay.
306 for (const host of ["registry-1.docker.io", "auth.docker.io", "production.cloudflare.docker.com", "mirror.gcr.io", "ghcr.io", "quay.io"]) {
307 assert.ok(BUILD_HOSTS.includes(host), host);
308 }
A package's latest version is its highest, not the last one published309 assert.ok(buildHosts("deploy").includes("api.cloudflare.com"));
Merge GitHub pull request #8 (claude/cool-newton-7nm8r9): Docker deployments on g1t.page behind a switch that stays off until Cloudflare's container APIs are checked live, workflows a workspace manages in its repositories, tarball and zipball downloads, refs/pull/* kept to g1t alone, framework detection for g1t.page builds, and a workflow's event payload matching GitHub's.310 // A container build pushes its image to Cloudflare's registry, from a sandbox or a job.
311 assert.ok(buildHosts("deploy").includes("registry.cloudflare.com"));
312 assert.ok(buildHosts("actions").includes("registry.cloudflare.com"));
A package's latest version is its highest, not the last one published313 assert.ok(!buildHosts("actions").includes("api.cloudflare.com"));
314 for (const host of buildHosts("deploy")) {
315 assert.doesNotMatch(host, /pool|xmr|monero|nicehash|^\*/, host);
316 }
317});
318
Merge branch 'main' into actions-toolkit-oidc-artifacts319test("workflow jobs get Docker unless the operator turns it off", () => {
320 assert.equal(dockerFor(undefined), "on");
321 assert.equal(dockerFor("on"), "on");
322 assert.equal(dockerFor(" OFF "), "off");
A package's latest version is its highest, not the last one published323});

This file's history is long; its oldest lines are credited to the oldest commit read.