| 1 | /** |
| 2 | * g1t-wide pauses: staff (or billing's hourly usage watcher) can stop |
| 3 | * whole kinds of work across the platform while unusual usage is looked |
| 4 | * into. See docs/SPEND-GUARDRAILS.md and the billing service's |
| 5 | * `platform.rs`. |
| 6 | * |
| 7 | * - `compute`: agents, sandboxes, Actions hosted jobs and builds. Billing's |
| 8 | * `reserve` refuses them itself, so every `ComputeGate.admit` caller gets |
| 9 | * it without asking here. |
| 10 | * - `schedules`: Actions' cron-triggered runs, and the runner's sweep that |
| 11 | * starts queued agents. |
| 12 | * - `indexing`: context embeddings and backfills, search backfills. |
| 13 | * - `renders`: social card rendering, which falls back to a static image. |
| 14 | * |
| 15 | * Read through billing's `platform_pause`, kept for 30 seconds in the |
| 16 | * isolate: never a database read per request. When billing cannot say, |
| 17 | * nothing is paused (fails open): a pause is pulled on purpose, and a |
| 18 | * billing outage must not stop the platform with it. The failure is kept |
| 19 | * for the same 30 seconds. |
| 20 | * |
| 21 | * Only type imports, so services' unit tests can load it on its own. |
| 22 | */ |
| 23 | import type { ServiceBinding } from "./clients"; |
| 24 | |
| 25 | export type PauseLevel = "compute" | "schedules" | "indexing" | "renders"; |
| 26 | |
| 27 | export const PAUSE_LEVELS: readonly PauseLevel[] = ["compute", "schedules", "indexing", "renders"]; |
| 28 | |
| 29 | /** Billing's `platform_pause`: which levels are paused now. */ |
| 30 | export type PlatformPause = Record<PauseLevel, boolean>; |
| 31 | |
| 32 | /** How long an answer is kept in the isolate. */ |
| 33 | export const PAUSE_KEPT_MS = 30_000; |
| 34 | |
| 35 | const NOTHING_PAUSED: PlatformPause = { compute: false, schedules: false, indexing: false, renders: false }; |
| 36 | |
| 37 | let kept: { value: PlatformPause; until: number } | null = null; |
| 38 | |
| 39 | /** Billing's answer as a pause, anything it does not say as not paused. */ |
| 40 | export function readPause(answer: unknown): PlatformPause { |
| 41 | const raw = answer && typeof answer === "object" ? (answer as Record<string, unknown>) : {}; |
| 42 | return { |
| 43 | compute: raw.compute === true, |
| 44 | schedules: raw.schedules === true, |
| 45 | indexing: raw.indexing === true, |
| 46 | renders: raw.renders === true, |
| 47 | }; |
| 48 | } |
| 49 | |
| 50 | /** Every level, kept for 30 seconds. Nothing paused when billing cannot say. Never throws. */ |
| 51 | export async function platformPause(billing: ServiceBinding | undefined, now = Date.now()): Promise<PlatformPause> { |
| 52 | if (!billing) return NOTHING_PAUSED; |
| 53 | if (kept && kept.until > now) return kept.value; |
| 54 | let value = NOTHING_PAUSED; |
| 55 | try { |
| 56 | const response = await billing.fetch("https://service/rpc/platform_pause", { |
| 57 | method: "POST", |
| 58 | headers: { "content-type": "application/json" }, |
| 59 | body: "{}", |
| 60 | }); |
| 61 | if (!response.ok) throw new Error(`platform_pause failed with status ${response.status}`); |
| 62 | value = readPause(await response.json()); |
| 63 | } catch (error) { |
| 64 | console.error("platform pause unreadable, so nothing is paused", String(error)); |
| 65 | } |
| 66 | kept = { value, until: now + PAUSE_KEPT_MS }; |
| 67 | return value; |
| 68 | } |
| 69 | |
| 70 | /** Whether `level` is paused across g1t. */ |
| 71 | export async function platformPaused(billing: ServiceBinding | undefined, level: PauseLevel): Promise<boolean> { |
| 72 | return (await platformPause(billing))[level]; |
| 73 | } |
| 74 | |
| 75 | /** For tests: forget the kept answer. */ |
| 76 | export function forgetPlatformPause(): void { |
| 77 | kept = null; |
| 78 | } |