Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge platform pause and the hourly usage watcher: staff can pause compute, schedules, indexing or renders for everyone, the watcher emails on a breach and is never blind quietly, and the models proxy holds each run to its cap (billing 0051, integrations 0006) | 1 | /** |
| 2 | * g1t-wide pauses: staff (or billing's hourly usage watcher) can stop | |
| 3 | * whole kinds of work across the platform while unusual usage is looked | |
| 4 | * into. See docs/SPEND-GUARDRAILS.md and the billing service's | |
| 5 | * `platform.rs`. | |
| 6 | * | |
| 7 | * - `compute`: agents, sandboxes, Actions hosted jobs and builds. Billing's | |
| 8 | * `reserve` refuses them itself, so every `ComputeGate.admit` caller gets | |
| 9 | * it without asking here. | |
| 10 | * - `schedules`: Actions' cron-triggered runs, and the runner's sweep that | |
| 11 | * starts queued agents. | |
| 12 | * - `indexing`: context embeddings and backfills, search backfills. | |
| 13 | * - `renders`: social card rendering, which falls back to a static image. | |
| 14 | * | |
| 15 | * Read through billing's `platform_pause`, kept for 30 seconds in the | |
| 16 | * isolate: never a database read per request. When billing cannot say, | |
| 17 | * nothing is paused (fails open): a pause is pulled on purpose, and a | |
| 18 | * billing outage must not stop the platform with it. The failure is kept | |
| 19 | * for the same 30 seconds. | |
| 20 | * | |
| 21 | * Only type imports, so services' unit tests can load it on its own. | |
| 22 | */ | |
| 23 | import type { ServiceBinding } from "./clients"; | |
| 24 | ||
| 25 | export type PauseLevel = "compute" | "schedules" | "indexing" | "renders"; | |
| 26 | ||
| 27 | export const PAUSE_LEVELS: readonly PauseLevel[] = ["compute", "schedules", "indexing", "renders"]; | |
| 28 | ||
| 29 | /** Billing's `platform_pause`: which levels are paused now. */ | |
| 30 | export type PlatformPause = Record<PauseLevel, boolean>; | |
| 31 | ||
| 32 | /** How long an answer is kept in the isolate. */ | |
| 33 | export const PAUSE_KEPT_MS = 30_000; | |
| 34 | ||
| 35 | const NOTHING_PAUSED: PlatformPause = { compute: false, schedules: false, indexing: false, renders: false }; | |
| 36 | ||
| 37 | let kept: { value: PlatformPause; until: number } | null = null; | |
| 38 | ||
| 39 | /** Billing's answer as a pause, anything it does not say as not paused. */ | |
| 40 | export function readPause(answer: unknown): PlatformPause { | |
| 41 | const raw = answer && typeof answer === "object" ? (answer as Record<string, unknown>) : {}; | |
| 42 | return { | |
| 43 | compute: raw.compute === true, | |
| 44 | schedules: raw.schedules === true, | |
| 45 | indexing: raw.indexing === true, | |
| 46 | renders: raw.renders === true, | |
| 47 | }; | |
| 48 | } | |
| 49 | ||
| 50 | /** Every level, kept for 30 seconds. Nothing paused when billing cannot say. Never throws. */ | |
| 51 | export async function platformPause(billing: ServiceBinding | undefined, now = Date.now()): Promise<PlatformPause> { | |
| 52 | if (!billing) return NOTHING_PAUSED; | |
| 53 | if (kept && kept.until > now) return kept.value; | |
| 54 | let value = NOTHING_PAUSED; | |
| 55 | try { | |
| 56 | const response = await billing.fetch("https://service/rpc/platform_pause", { | |
| 57 | method: "POST", | |
| 58 | headers: { "content-type": "application/json" }, | |
| 59 | body: "{}", | |
| 60 | }); | |
| 61 | if (!response.ok) throw new Error(`platform_pause failed with status ${response.status}`); | |
| 62 | value = readPause(await response.json()); | |
| 63 | } catch (error) { | |
| 64 | console.error("platform pause unreadable, so nothing is paused", String(error)); | |
| 65 | } | |
| 66 | kept = { value, until: now + PAUSE_KEPT_MS }; | |
| 67 | return value; | |
| 68 | } | |
| 69 | ||
| 70 | /** Whether `level` is paused across g1t. */ | |
| 71 | export async function platformPaused(billing: ServiceBinding | undefined, level: PauseLevel): Promise<boolean> { | |
| 72 | return (await platformPause(billing))[level]; | |
| 73 | } | |
| 74 | ||
| 75 | /** For tests: forget the kept answer. */ | |
| 76 | export function forgetPlatformPause(): void { | |
| 77 | kept = null; | |
| 78 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.