Skip to content
Sign in

flagon-io/g1t

public
0
219 linesCodeBlameRaw
1/**
2 * Running a part's check: each request it makes, all at once, with a short
3 * timeout. The answer's body is never read.
4 */
5import type { Check, Step } from "./components.ts";
6
7/** A check's raw outcome. */
8export type ProbeResult = {
9 ok: boolean;
10 /** How long it took, in milliseconds: the slowest of its requests. */
11 ms: number;
12 /** Why it failed, in a few words: "timed out", "HTTP 502". */
13 error?: string;
14 /** Why it worked but not well, when that is not just slowness. */
15 degraded?: string;
16 /**
17 * The Cloudflare data centre that answered, from the `cf-ray` header's
18 * suffix (`8c1f…-IAD`): where the check ran from, as far as g1t saw it.
19 */
20 colo?: string;
21 /** Slow at first and checked again at once (`confirmSlow`): the first try's time. */
22 first_ms?: number;
23};
24
25/** How one git store namespace answered lately: repos `store_health`. */
26export type StoreHealthRow = {
27 store: string;
28 calls: number;
29 errors: number;
30 rate_limited: number;
31 rejected: number;
32 ms_total: number;
33};
34
35export type StorageReport = { minutes: number; stores: StoreHealthRow[] };
36
37/** At least this many failed calls, and this share of them, before git storage is down. */
38const STORAGE_MIN_ERRORS = 5;
39const STORAGE_DOWN_SHARE = 0.25;
40
41/**
42 * What the git store's recent answers mean: down when a quarter or more of
43 * its calls failed (at least five), or calls were refused after repeated
44 * failures; degraded when it rate limited g1t; otherwise as fast as its
45 * mean call. Quiet is up.
46 */
47export function judgeStorage(report: StorageReport): ProbeResult {
48 const sum = (key: keyof Omit<StoreHealthRow, "store">) =>
49 report.stores.reduce((total, row) => total + (Number(row[key]) || 0), 0);
50 const calls = sum("calls");
51 const errors = sum("errors");
52 const limited = sum("rate_limited");
53 const rejected = sum("rejected");
54 const ms = calls > 0 ? sum("ms_total") / calls : 0;
55 if (rejected > 0) return { ok: false, ms, error: `calls refused after repeated failures (${rejected})` };
56 if (errors >= STORAGE_MIN_ERRORS && errors / Math.max(calls, 1) >= STORAGE_DOWN_SHARE) {
57 return { ok: false, ms, error: `${Math.round((100 * errors) / calls)}% of calls failed` };
58 }
59 if (limited > 0) return { ok: true, ms, degraded: `Rate limited ${limited} times in ${report.minutes} minutes` };
60 // A namespace served from the fallback store (`<namespace>@fallback`,
61 // repos src/fallback.rs): reads work from the last backup, writes wait.
62 const fallback = report.stores.filter((row) => row.store.endsWith("@fallback") && Number(row.calls) > 0);
63 if (fallback.length > 0) {
64 return { ok: true, ms, degraded: "Served from the backup store: reads work, pushes and merges wait" };
65 }
66 return { ok: true, ms };
67}
68
69/** No request waits longer than this. */
70export const TIMEOUT_MS = 5000;
71
72/** What every check but a page load says it is. */
73export const USER_AGENT = "g1t-status (+https://status.g1t.sh)";
74
75/**
76 * What a page load (`Step.browser`) says it is: a browser's user agent with
77 * `g1t-status/1.0 (+status.g1t.sh)` on the end, so it still says who it is.
78 *
79 * The site renders a page for a crawler in full before sending a byte (an
80 * `isbot` match makes apps/web's entry.server.tsx wait for `allReady`),
81 * and streams the shell first for a browser. USER_AGENT matches isbot (on
82 * "http", and "status/"), so with it Page speed timed a crawler's full
83 * render, while its budget is to the first byte. probe.test.ts checks this
84 * one against the isbot the site uses. Keep the name after "Safari/537.36",
85 * and keep "http" and "compatible;" out of it: isbot matches a URL, and
86 * "status/" inside a "compatible" comment.
87 */
88export const BROWSER_USER_AGENT =
89 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 g1t-status/1.0 (+status.g1t.sh)";
90
91type Fetch = (url: string, init: RequestInit) => Promise<Response>;
92
93class Timeout extends Error {}
94
95/** Runs `work`, timing it, and failing it after `timeoutMs`. */
96export async function timed(
97 work: (signal: AbortSignal) => Promise<true | string>,
98 timeoutMs = TIMEOUT_MS,
99 now: () => number = Date.now,
100): Promise<ProbeResult> {
101 const started = now();
102 const controller = new AbortController();
103 let timer: ReturnType<typeof setTimeout> | undefined;
104 const deadline = new Promise<never>((_, reject) => {
105 timer = setTimeout(() => {
106 controller.abort();
107 reject(new Timeout());
108 }, timeoutMs);
109 });
110 try {
111 const outcome = await Promise.race([work(controller.signal), deadline]);
112 const ms = now() - started;
113 return outcome === true ? { ok: true, ms } : { ok: false, ms, error: outcome };
114 } catch (error) {
115 const ms = now() - started;
116 return { ok: false, ms, error: error instanceof Timeout ? "timed out" : "could not connect" };
117 } finally {
118 clearTimeout(timer);
119 }
120}
121
122/** One request, answered with the status that means it works. */
123export async function step(fetcher: Fetch, { url, headers = {}, expect, browser = false }: Step, timeoutMs = TIMEOUT_MS): Promise<ProbeResult> {
124 let colo: string | null = null;
125 const result = await timed(async (signal) => {
126 const response = await fetcher(url, {
127 signal,
128 redirect: "manual",
129 headers: { "user-agent": browser ? BROWSER_USER_AGENT : USER_AGENT, "cache-control": "no-cache", ...headers },
130 });
131 // Timed to the answer's headers: the body is never read.
132 colo = coloOf(response.headers.get("cf-ray"));
133 await response.body?.cancel().catch(() => undefined);
134 const good = expect == null ? response.ok : response.status === expect;
135 return good || `HTTP ${response.status}`;
136 }, timeoutMs);
137 return colo ? { ...result, colo } : result;
138}
139
140/** The data centre in a `cf-ray` header: `8c1f2e3d4a5b6c7d-IAD` is `IAD`. */
141export function coloOf(ray: string | null | undefined): string | null {
142 const m = /-([A-Za-z]{3,4})$/.exec((ray ?? "").trim());
143 return m ? m[1]!.toUpperCase() : null;
144}
145
146/** A check's requests together: it works when every one does, and takes as long as the slowest. */
147export function combine(results: ProbeResult[]): ProbeResult {
148 const ms = Math.max(0, ...results.map((r) => r.ms));
149 const failed = results.find((r) => !r.ok);
150 const colo = results.find((r) => r.colo)?.colo;
151 const out: ProbeResult = failed ? { ok: false, ms, error: failed.error ?? "no answer" } : { ok: true, ms };
152 return colo ? { ...out, colo } : out;
153}
154
155/** Whether a result is only slow: it worked, with nothing else wrong, but over `slowMs`. */
156export function onlySlow(result: ProbeResult | null, slowMs: number): boolean {
157 return result != null && result.ok && !result.degraded && Math.round(result.ms) > slowMs;
158}
159
160/**
161 * A slow check, and the same check run again at once: the better of the
162 * two. One slow answer (a cold isolate, a cache refill, a busy moment on
163 * the path) does not count when the next answers in time; slow twice is
164 * slow, at the faster of the two times. A second try that failed does not
165 * make a slow check worse. `first_ms` keeps the first try's time.
166 */
167export function confirmSlow(first: ProbeResult, again: ProbeResult | null): ProbeResult {
168 if (!again || !again.ok || again.degraded) return { ...first, first_ms: first.ms };
169 const better = again.ms < first.ms ? again : first;
170 const colo = better.colo ?? first.colo ?? again.colo;
171 return { ...better, ...(colo ? { colo } : {}), first_ms: first.ms };
172}
173
174/** What runs a check. `billing` is null when there is no binding to it. */
175export type Probers = {
176 fetch: Fetch;
177 billing: (() => Promise<unknown>) | null;
178 /** Git storage's recent health, through the repos service; null when not bound. */
179 storage?: (() => Promise<StorageReport>) | null;
180 timeoutMs?: number;
181};
182
183/** Runs one part's check. Null for a part with no check. */
184export async function runCheck(check: Check, probers: Probers): Promise<ProbeResult | null> {
185 const timeoutMs = probers.timeoutMs ?? TIMEOUT_MS;
186 switch (check.kind) {
187 case "http":
188 return combine(await Promise.all(check.steps.map((s) => step(probers.fetch, s, timeoutMs))));
189 case "billing": {
190 const billing = probers.billing;
191 if (!billing) return null;
192 return timed(async () => ((await billing()) ? true : "no price book"), timeoutMs);
193 }
194 case "storage": {
195 const storage = probers.storage;
196 if (!storage) return null;
197 let report: StorageReport | null = null;
198 const asked = await timed(async () => {
199 report = await storage();
200 return true;
201 }, timeoutMs);
202 return report ? judgeStorage(report) : asked;
203 }
204 case "none":
205 return null;
206 }
207}
208
209/**
210 * Runs one part's check as the cron does: an address that answered, but
211 * slowly, is asked once more straight away (`confirmSlow`) before the slow
212 * answer counts. Checks through a binding are not repeated: git storage
213 * reports the minutes gone by, and asking twice says the same.
214 */
215export async function probe(check: Check, probers: Probers, slowMs: number): Promise<ProbeResult | null> {
216 const first = await runCheck(check, probers);
217 if (check.kind !== "http" || !first || !onlySlow(first, slowMs)) return first;
218 return confirmSlow(first, await runCheck(check, probers));
219}