Sign in

flagon-io/g1t

public

g1t/apps/status/src/probe.ts

152 lines5,817 bytesCodeBlame
1/**
2 * Running a part's check: each request it makes, all at once, with a short
3 * timeout. The answer's body is never read.
4 */
5import type { Check, Step } from "./components.ts";
6
7/** A check's raw outcome. */
8export type ProbeResult = {
9 ok: boolean;
10 /** How long it took, in milliseconds: the slowest of its requests. */
11 ms: number;
12 /** Why it failed, in a few words: "timed out", "HTTP 502". */
13 error?: string;
14 /** Why it worked but not well, when that is not just slowness. */
15 degraded?: string;
16};
17
18/** How one git store namespace answered lately: repos `store_health`. */
19export type StoreHealthRow = {
20 store: string;
21 calls: number;
22 errors: number;
23 rate_limited: number;
24 rejected: number;
25 ms_total: number;
26};
27
28export type StorageReport = { minutes: number; stores: StoreHealthRow[] };
29
30/** At least this many failed calls, and this share of them, before git storage is down. */
31const STORAGE_MIN_ERRORS = 5;
32const STORAGE_DOWN_SHARE = 0.25;
33
34/**
35 * What the git store's recent answers mean: down when a quarter or more of
36 * its calls failed (at least five), or calls were refused after repeated
37 * failures; degraded when it rate limited g1t; otherwise as fast as its
38 * mean call. Quiet is up.
39 */
40export function judgeStorage(report: StorageReport): ProbeResult {
41 const sum = (key: keyof Omit<StoreHealthRow, "store">) =>
42 report.stores.reduce((total, row) => total + (Number(row[key]) || 0), 0);
43 const calls = sum("calls");
44 const errors = sum("errors");
45 const limited = sum("rate_limited");
46 const rejected = sum("rejected");
47 const ms = calls > 0 ? sum("ms_total") / calls : 0;
48 if (rejected > 0) return { ok: false, ms, error: `calls refused after repeated failures (${rejected})` };
49 if (errors >= STORAGE_MIN_ERRORS && errors / Math.max(calls, 1) >= STORAGE_DOWN_SHARE) {
50 return { ok: false, ms, error: `${Math.round((100 * errors) / calls)}% of calls failed` };
51 }
52 if (limited > 0) return { ok: true, ms, degraded: `Rate limited ${limited} times in ${report.minutes} minutes` };
53 // A namespace served from the fallback store (`<namespace>@fallback`,
54 // repos src/fallback.rs): reads work from the last backup, writes wait.
55 const fallback = report.stores.filter((row) => row.store.endsWith("@fallback") && Number(row.calls) > 0);
56 if (fallback.length > 0) {
57 return { ok: true, ms, degraded: "Served from the backup store: reads work, pushes and merges wait" };
58 }
59 return { ok: true, ms };
60}
61
62/** No request waits longer than this. */
63export const TIMEOUT_MS = 5000;
64
65export const USER_AGENT = "g1t-status (+https://status.g1t.sh)";
66
67type Fetch = (url: string, init: RequestInit) => Promise<Response>;
68
69class Timeout extends Error {}
70
71/** Runs `work`, timing it, and failing it after `timeoutMs`. */
72export async function timed(
73 work: (signal: AbortSignal) => Promise<true | string>,
74 timeoutMs = TIMEOUT_MS,
75 now: () => number = Date.now,
76): Promise<ProbeResult> {
77 const started = now();
78 const controller = new AbortController();
79 let timer: ReturnType<typeof setTimeout> | undefined;
80 const deadline = new Promise<never>((_, reject) => {
81 timer = setTimeout(() => {
82 controller.abort();
83 reject(new Timeout());
84 }, timeoutMs);
85 });
86 try {
87 const outcome = await Promise.race([work(controller.signal), deadline]);
88 const ms = now() - started;
89 return outcome === true ? { ok: true, ms } : { ok: false, ms, error: outcome };
90 } catch (error) {
91 const ms = now() - started;
92 return { ok: false, ms, error: error instanceof Timeout ? "timed out" : "could not connect" };
93 } finally {
94 clearTimeout(timer);
95 }
96}
97
98/** One request, answered with the status that means it works. */
99export function step(fetcher: Fetch, { url, headers = {}, expect }: Step, timeoutMs = TIMEOUT_MS): Promise<ProbeResult> {
100 return timed(async (signal) => {
101 const response = await fetcher(url, {
102 signal,
103 redirect: "manual",
104 headers: { "user-agent": USER_AGENT, "cache-control": "no-cache", ...headers },
105 });
106 await response.body?.cancel().catch(() => undefined);
107 const good = expect == null ? response.ok : response.status === expect;
108 return good || `HTTP ${response.status}`;
109 }, timeoutMs);
110}
111
112/** A check's requests together: it works when every one does, and takes as long as the slowest. */
113export function combine(results: ProbeResult[]): ProbeResult {
114 const ms = Math.max(0, ...results.map((r) => r.ms));
115 const failed = results.find((r) => !r.ok);
116 return failed ? { ok: false, ms, error: failed.error ?? "no answer" } : { ok: true, ms };
117}
118
119/** What runs a check. `billing` is null when there is no binding to it. */
120export type Probers = {
121 fetch: Fetch;
122 billing: (() => Promise<unknown>) | null;
123 /** Git storage's recent health, through the repos service; null when not bound. */
124 storage?: (() => Promise<StorageReport>) | null;
125 timeoutMs?: number;
126};
127
128/** Runs one part's check. Null for a part with no check. */
129export async function runCheck(check: Check, probers: Probers): Promise<ProbeResult | null> {
130 const timeoutMs = probers.timeoutMs ?? TIMEOUT_MS;
131 switch (check.kind) {
132 case "http":
133 return combine(await Promise.all(check.steps.map((s) => step(probers.fetch, s, timeoutMs))));
134 case "billing": {
135 const billing = probers.billing;
136 if (!billing) return null;
137 return timed(async () => ((await billing()) ? true : "no price book"), timeoutMs);
138 }
139 case "storage": {
140 const storage = probers.storage;
141 if (!storage) return null;
142 let report: StorageReport | null = null;
143 const asked = await timed(async () => {
144 report = await storage();
145 return true;
146 }, timeoutMs);
147 return report ? judgeStorage(report) : asked;
148 }
149 case "none":
150 return null;
151 }
152}