Sign in

flagon-io/g1t

public

g1t/apps/status/src/probe.ts

152 lines5,817 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1/**
2 * Running a part's check: each request it makes, all at once, with a short
3 * timeout. The answer's body is never read.
4 */
5import type { Check, Step } from "./components.ts";
6
7/** A check's raw outcome. */
8export type ProbeResult = {
9 ok: boolean;
10 /** How long it took, in milliseconds: the slowest of its requests. */
11 ms: number;
12 /** Why it failed, in a few words: "timed out", "HTTP 502". */
13 error?: string;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily14 /** Why it worked but not well, when that is not just slowness. */
15 degraded?: string;
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas16};
17
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily18/** How one git store namespace answered lately: repos `store_health`. */
19export type StoreHealthRow = {
20 store: string;
21 calls: number;
22 errors: number;
23 rate_limited: number;
24 rejected: number;
25 ms_total: number;
26};
27
28export type StorageReport = { minutes: number; stores: StoreHealthRow[] };
29
30/** At least this many failed calls, and this share of them, before git storage is down. */
31const STORAGE_MIN_ERRORS = 5;
32const STORAGE_DOWN_SHARE = 0.25;
33
34/**
35 * What the git store's recent answers mean: down when a quarter or more of
36 * its calls failed (at least five), or calls were refused after repeated
37 * failures; degraded when it rate limited g1t; otherwise as fast as its
38 * mean call. Quiet is up.
39 */
40export function judgeStorage(report: StorageReport): ProbeResult {
41 const sum = (key: keyof Omit<StoreHealthRow, "store">) =>
42 report.stores.reduce((total, row) => total + (Number(row[key]) || 0), 0);
43 const calls = sum("calls");
44 const errors = sum("errors");
45 const limited = sum("rate_limited");
46 const rejected = sum("rejected");
47 const ms = calls > 0 ? sum("ms_total") / calls : 0;
48 if (rejected > 0) return { ok: false, ms, error: `calls refused after repeated failures (${rejected})` };
49 if (errors >= STORAGE_MIN_ERRORS && errors / Math.max(calls, 1) >= STORAGE_DOWN_SHARE) {
50 return { ok: false, ms, error: `${Math.round((100 * errors) / calls)}% of calls failed` };
51 }
52 if (limited > 0) return { ok: true, ms, degraded: `Rate limited ${limited} times in ${report.minutes} minutes` };
Repositories shard across git store namespaces, move between them, and can keep to the EU; a namespace can be served read-only from the self-hosted git store, rebuilt from the nightly backups (#20, #25)53 // A namespace served from the fallback store (`<namespace>@fallback`,
54 // repos src/fallback.rs): reads work from the last backup, writes wait.
55 const fallback = report.stores.filter((row) => row.store.endsWith("@fallback") && Number(row.calls) > 0);
56 if (fallback.length > 0) {
57 return { ok: true, ms, degraded: "Served from the backup store: reads work, pushes and merges wait" };
58 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily59 return { ok: true, ms };
60}
61
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas62/** No request waits longer than this. */
63export const TIMEOUT_MS = 5000;
64
65export const USER_AGENT = "g1t-status (+https://status.g1t.sh)";
66
67type Fetch = (url: string, init: RequestInit) => Promise<Response>;
68
69class Timeout extends Error {}
70
71/** Runs `work`, timing it, and failing it after `timeoutMs`. */
72export async function timed(
73 work: (signal: AbortSignal) => Promise<true | string>,
74 timeoutMs = TIMEOUT_MS,
75 now: () => number = Date.now,
76): Promise<ProbeResult> {
77 const started = now();
78 const controller = new AbortController();
79 let timer: ReturnType<typeof setTimeout> | undefined;
80 const deadline = new Promise<never>((_, reject) => {
81 timer = setTimeout(() => {
82 controller.abort();
83 reject(new Timeout());
84 }, timeoutMs);
85 });
86 try {
87 const outcome = await Promise.race([work(controller.signal), deadline]);
88 const ms = now() - started;
89 return outcome === true ? { ok: true, ms } : { ok: false, ms, error: outcome };
90 } catch (error) {
91 const ms = now() - started;
92 return { ok: false, ms, error: error instanceof Timeout ? "timed out" : "could not connect" };
93 } finally {
94 clearTimeout(timer);
95 }
96}
97
98/** One request, answered with the status that means it works. */
99export function step(fetcher: Fetch, { url, headers = {}, expect }: Step, timeoutMs = TIMEOUT_MS): Promise<ProbeResult> {
100 return timed(async (signal) => {
101 const response = await fetcher(url, {
102 signal,
103 redirect: "manual",
104 headers: { "user-agent": USER_AGENT, "cache-control": "no-cache", ...headers },
105 });
106 await response.body?.cancel().catch(() => undefined);
107 const good = expect == null ? response.ok : response.status === expect;
108 return good || `HTTP ${response.status}`;
109 }, timeoutMs);
110}
111
112/** A check's requests together: it works when every one does, and takes as long as the slowest. */
113export function combine(results: ProbeResult[]): ProbeResult {
114 const ms = Math.max(0, ...results.map((r) => r.ms));
115 const failed = results.find((r) => !r.ok);
116 return failed ? { ok: false, ms, error: failed.error ?? "no answer" } : { ok: true, ms };
117}
118
119/** What runs a check. `billing` is null when there is no binding to it. */
120export type Probers = {
121 fetch: Fetch;
122 billing: (() => Promise<unknown>) | null;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily123 /** Git storage's recent health, through the repos service; null when not bound. */
124 storage?: (() => Promise<StorageReport>) | null;
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas125 timeoutMs?: number;
126};
127
128/** Runs one part's check. Null for a part with no check. */
129export async function runCheck(check: Check, probers: Probers): Promise<ProbeResult | null> {
130 const timeoutMs = probers.timeoutMs ?? TIMEOUT_MS;
131 switch (check.kind) {
132 case "http":
133 return combine(await Promise.all(check.steps.map((s) => step(probers.fetch, s, timeoutMs))));
134 case "billing": {
135 const billing = probers.billing;
136 if (!billing) return null;
137 return timed(async () => ((await billing()) ? true : "no price book"), timeoutMs);
138 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily139 case "storage": {
140 const storage = probers.storage;
141 if (!storage) return null;
142 let report: StorageReport | null = null;
143 const asked = await timed(async () => {
144 report = await storage();
145 return true;
146 }, timeoutMs);
147 return report ? judgeStorage(report) : asked;
148 }
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas149 case "none":
150 return null;
151 }
152}