| 1 | /** |
| 2 | * How a sandbox's Durable Object handles the end of its container without |
| 3 | * failing the invocation it happens in. The containers library calls |
| 4 | * `onStop` from the object's alarm: a stop hook that throws fails that |
| 5 | * alarm, which Cloudflare retries and counts as an error each time, and |
| 6 | * which runs the whole hook again. So the hook's calls to other services |
| 7 | * go through `tellStopped`, which tries twice and logs at error level, and |
| 8 | * never throws. Pure, so it is tested on its own. |
| 9 | */ |
| 10 | |
| 11 | /** How long `tellStopped` waits before its second try. */ |
| 12 | export const RETRY_AFTER_MS = 500; |
| 13 | |
| 14 | /** |
| 15 | * Tells another service that a sandbox stopped: `step` runs, and once more |
| 16 | * if it throws. Never throws: a step that fails twice is logged with |
| 17 | * `log` as `sandbox stop not reported`, with what and why, so it shows in |
| 18 | * the runner's logs at error level and the five-minute sweep picks the |
| 19 | * work up instead. Returns whether it got through. |
| 20 | */ |
| 21 | export async function tellStopped( |
| 22 | what: string, |
| 23 | step: () => Promise<unknown>, |
| 24 | options: { log?: (...data: unknown[]) => void; attempts?: number; waitMs?: number } = {}, |
| 25 | ): Promise<boolean> { |
| 26 | const log = options.log ?? console.error; |
| 27 | const attempts = Math.max(1, options.attempts ?? 2); |
| 28 | let last: unknown = null; |
| 29 | for (let attempt = 1; attempt <= attempts; attempt++) { |
| 30 | try { |
| 31 | await step(); |
| 32 | return true; |
| 33 | } catch (error) { |
| 34 | last = error; |
| 35 | if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, options.waitMs ?? RETRY_AFTER_MS)); |
| 36 | } |
| 37 | } |
| 38 | log("sandbox stop not reported", what, describeError(last)); |
| 39 | return false; |
| 40 | } |
| 41 | |
| 42 | /** |
| 43 | * A service binding's answer as a step's outcome: a 5xx (or 429) throws, so |
| 44 | * `tellStopped` tries again and logs it. A refusal is not a failure: the |
| 45 | * work already reported its end, and services say so with `ok: false` or a |
| 46 | * 4xx (the deployments service answers 404 for a build that has finished). |
| 47 | */ |
| 48 | export async function answered(what: string, response: Response): Promise<void> { |
| 49 | if (response.status < 500 && response.status !== 429) return; |
| 50 | const body = await response.text().catch(() => ""); |
| 51 | throw new Error(`${what} answered ${response.status}${body ? `: ${body.slice(0, 200)}` : ""}`); |
| 52 | } |
| 53 | |
| 54 | /** |
| 55 | * Whether a sandbox whose `sleepAfter` has passed is still inside its run's |
| 56 | * time cap, and so keeps running: the cap, not inactivity, ends a run (the |
| 57 | * runner never fetches the container, so to the library every sandbox looks |
| 58 | * idle). Without a cap or a start time, `sleepAfter` applies. |
| 59 | */ |
| 60 | export function withinTimeCap( |
| 61 | started: number | null | undefined, |
| 62 | capMinutes: number | null | undefined, |
| 63 | now: number, |
| 64 | graceSeconds: number, |
| 65 | ): boolean { |
| 66 | if (typeof started !== "number" || typeof capMinutes !== "number" || capMinutes <= 0) return false; |
| 67 | return now < started + (capMinutes * 60 + graceSeconds) * 1000; |
| 68 | } |
| 69 | |
| 70 | /** |
| 71 | * Whether the run a sandbox holds now has used its whole time cap, so a |
| 72 | * `timeUp` alarm may stop it. A sandbox's Durable Object is named for its |
| 73 | * work (`actions:{job}` and the like), and a re-run of that work starts in |
| 74 | * the same object; an alarm an earlier attempt left must not stop the |
| 75 | * attempt running now, which started later and has its own alarm. |
| 76 | */ |
| 77 | export function capReached(started: number | null | undefined, capMinutes: number | null | undefined, now: number): boolean { |
| 78 | if (typeof started !== "number" || typeof capMinutes !== "number" || capMinutes <= 0) return false; |
| 79 | return now >= started + capMinutes * 60 * 1000; |
| 80 | } |
| 81 | |
| 82 | /** |
| 83 | * Whether a sandbox's stop ends its work as a failure. A sandbox g1t |
| 84 | * stopped (at its time cap, say) can come back with exit code 0, since the |
| 85 | * container library reports a destroyed container as a clean exit; the |
| 86 | * reason g1t stopped it, not the exit code, says how it ended. |
| 87 | */ |
| 88 | export function stoppedShort(exitCode: number, why: string | null): boolean { |
| 89 | return exitCode !== 0 || why !== null; |
| 90 | } |
| 91 | |
| 92 | /** An error as one line, for a log: its message, or what was thrown. */ |
| 93 | export function describeError(error: unknown): string { |
| 94 | if (error instanceof Error) return error.message || error.name; |
| 95 | if (error === undefined) return "no reason given"; |
| 96 | if (typeof error === "string") return error; |
| 97 | try { |
| 98 | return JSON.stringify(error) ?? String(error); |
| 99 | } catch { |
| 100 | return String(error); |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | /** |
| 105 | * Sandbox time billing has not confirmed yet, kept in the sandbox's |
| 106 | * storage until it does. `usage` is fixed when the sandbox stopped, so a |
| 107 | * later try records the same seconds under the same reference, which |
| 108 | * billing records once whatever it is sent (`record_sandbox`). |
| 109 | */ |
| 110 | export type Owed<U extends { reference: string }> = { |
| 111 | usage: U; |
| 112 | /** Tries so far that billing did not confirm. */ |
| 113 | attempts: number; |
| 114 | /** When the sandbox stopped, in ms. */ |
| 115 | since: number; |
| 116 | }; |
| 117 | |
| 118 | /** The longest wait between tries: an hour. */ |
| 119 | export const OWED_MAX_DELAY_SECONDS = 60 * 60; |
| 120 | /** How long sandbox time is tried for before it is logged as lost: a week. */ |
| 121 | export const OWED_GIVE_UP_MS = 7 * 24 * 60 * 60 * 1000; |
| 122 | |
| 123 | /** The wait before the next try, after `attempts` failed ones: 30 seconds, doubling, at most an hour. */ |
| 124 | export function owedDelaySeconds(attempts: number): number { |
| 125 | return Math.min(OWED_MAX_DELAY_SECONDS, 30 * 2 ** Math.min(16, Math.max(0, attempts - 1))); |
| 126 | } |
| 127 | |
| 128 | /** |
| 129 | * Tries to record each owed usage once. Returns what billing confirmed |
| 130 | * (to forget), what to keep for another try (with its attempts counted), |
| 131 | * what has been tried for too long (to log and forget), and the wait before |
| 132 | * the next try, or null when nothing is kept. `record` resolves true only |
| 133 | * when billing confirmed it; a throw is a failure like any other. |
| 134 | */ |
| 135 | export async function recordOwed<U extends { reference: string }>( |
| 136 | owed: Owed<U>[], |
| 137 | record: (usage: U) => Promise<boolean>, |
| 138 | now: number, |
| 139 | ): Promise<{ recorded: Owed<U>[]; kept: Owed<U>[]; lost: Owed<U>[]; retryInSeconds: number | null }> { |
| 140 | const recorded: Owed<U>[] = []; |
| 141 | const kept: Owed<U>[] = []; |
| 142 | const lost: Owed<U>[] = []; |
| 143 | for (const entry of owed) { |
| 144 | const ok = await record(entry.usage).catch(() => false); |
| 145 | if (ok) { |
| 146 | recorded.push(entry); |
| 147 | continue; |
| 148 | } |
| 149 | const next = { ...entry, attempts: entry.attempts + 1 }; |
| 150 | if (now - entry.since >= OWED_GIVE_UP_MS) lost.push(next); |
| 151 | else kept.push(next); |
| 152 | } |
| 153 | const retryInSeconds = kept.length ? Math.min(...kept.map((entry) => owedDelaySeconds(entry.attempts))) : null; |
| 154 | return { recorded, kept, lost, retryInSeconds }; |
| 155 | } |