Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
214 linesCodeBlameRaw
1/**
2 * Running a part's check: each request it makes, all at once, with a short
3 * timeout. The answer's body is never read.
4 */
5import type { Check, Step } from "./components.ts";
6
7/** A check's raw outcome. */
8export type ProbeResult = {
9 ok: boolean;
10 /** How long it took, in milliseconds: the slowest of its requests. */
11 ms: number;
12 /** Why it failed, in a few words: "timed out", "HTTP 502". */
13 error?: string;
14 /** Why it worked but not well, when that is not just slowness. */
15 degraded?: string;
16 /**
17 * The Cloudflare data centre that answered, from the `cf-ray` header's
18 * suffix (`8c1f…-IAD`): where the check ran from, as far as g1t saw it.
19 */
20 colo?: string;
21 /** Slow at first and checked again at once (`confirmSlow`): the first try's time. */
22 first_ms?: number;
23};
24
25/** How one git store namespace answered lately: repos `store_health`. */
26export type StoreHealthRow = {
27 store: string;
28 calls: number;
29 errors: number;
30 rate_limited: number;
31 rejected: number;
32 ms_total: number;
33};
34
35export type StorageReport = { minutes: number; stores: StoreHealthRow[] };
36
37/** At least this many failed calls, and this share of them, before git storage is down. */
38const STORAGE_MIN_ERRORS = 5;
39const STORAGE_DOWN_SHARE = 0.25;
40
41/**
42 * What the git store's recent answers mean: down when a quarter or more of
43 * its calls failed (at least five), or calls were refused after repeated
44 * failures; degraded when it rate limited g1t; otherwise as fast as its
45 * mean call. Quiet is up.
46 */
47export function judgeStorage(report: StorageReport): ProbeResult {
48 const sum = (key: keyof Omit<StoreHealthRow, "store">) => report.stores.reduce((total, row) => total + (Number(row[key]) || 0), 0);
49 const calls = sum("calls");
50 const errors = sum("errors");
51 const limited = sum("rate_limited");
52 const rejected = sum("rejected");
53 const ms = calls > 0 ? sum("ms_total") / calls : 0;
54 if (rejected > 0) return { ok: false, ms, error: `calls refused after repeated failures (${rejected})` };
55 if (errors >= STORAGE_MIN_ERRORS && errors / Math.max(calls, 1) >= STORAGE_DOWN_SHARE) {
56 return { ok: false, ms, error: `${Math.round((100 * errors) / calls)}% of calls failed` };
57 }
58 if (limited > 0) return { ok: true, ms, degraded: `Rate limited ${limited} times in ${report.minutes} minutes` };
59 // A namespace served from the fallback store (`<namespace>@fallback`,
60 // repos src/fallback.rs): reads work from the last backup, writes wait.
61 const fallback = report.stores.filter((row) => row.store.endsWith("@fallback") && Number(row.calls) > 0);
62 if (fallback.length > 0) {
63 return { ok: true, ms, degraded: "Served from the backup store: reads work, pushes and merges wait" };
64 }
65 return { ok: true, ms };
66}
67
68/** No request waits longer than this. */
69export const TIMEOUT_MS = 5000;
70
71/** What every check but a page load says it is. */
72export const USER_AGENT = "g1t-status (+https://status.g1t.sh)";
73
74/**
75 * What a page load (`Step.browser`) says it is: a browser's user agent with
76 * `g1t-status/1.0 (+status.g1t.sh)` on the end, so it still says who it is.
77 *
78 * The site renders a page for a crawler in full before sending a byte (an
79 * `isbot` match makes apps/web's entry.server.tsx wait for `allReady`),
80 * and streams the shell first for a browser. USER_AGENT matches isbot (on
81 * "http", and "status/"), so with it Page speed timed a crawler's full
82 * render, while its budget is to the first byte. probe.test.ts checks this
83 * one against the isbot the site uses. Keep the name after "Safari/537.36",
84 * and keep "http" and "compatible;" out of it: isbot matches a URL, and
85 * "status/" inside a "compatible" comment.
86 */
87export const BROWSER_USER_AGENT =
88 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 g1t-status/1.0 (+status.g1t.sh)";
89
90type Fetch = (url: string, init: RequestInit) => Promise<Response>;
91
92class Timeout extends Error {}
93
94/** Runs `work`, timing it, and failing it after `timeoutMs`. */
95export async function timed(work: (signal: AbortSignal) => Promise<true | string>, timeoutMs = TIMEOUT_MS, now: () => number = Date.now): Promise<ProbeResult> {
96 const started = now();
97 const controller = new AbortController();
98 let timer: ReturnType<typeof setTimeout> | undefined;
99 const deadline = new Promise<never>((_, reject) => {
100 timer = setTimeout(() => {
101 controller.abort();
102 reject(new Timeout());
103 }, timeoutMs);
104 });
105 try {
106 const outcome = await Promise.race([work(controller.signal), deadline]);
107 const ms = now() - started;
108 return outcome === true ? { ok: true, ms } : { ok: false, ms, error: outcome };
109 } catch (error) {
110 const ms = now() - started;
111 return { ok: false, ms, error: error instanceof Timeout ? "timed out" : "could not connect" };
112 } finally {
113 clearTimeout(timer);
114 }
115}
116
117/** One request, answered with the status that means it works. */
118export async function step(fetcher: Fetch, { url, headers = {}, expect, browser = false }: Step, timeoutMs = TIMEOUT_MS): Promise<ProbeResult> {
119 let colo: string | null = null;
120 const result = await timed(async (signal) => {
121 const response = await fetcher(url, {
122 signal,
123 redirect: "manual",
124 headers: { "user-agent": browser ? BROWSER_USER_AGENT : USER_AGENT, "cache-control": "no-cache", ...headers },
125 });
126 // Timed to the answer's headers: the body is never read.
127 colo = coloOf(response.headers.get("cf-ray"));
128 await response.body?.cancel().catch(() => undefined);
129 const good = expect == null ? response.ok : response.status === expect;
130 return good || `HTTP ${response.status}`;
131 }, timeoutMs);
132 return colo ? { ...result, colo } : result;
133}
134
135/** The data centre in a `cf-ray` header: `8c1f2e3d4a5b6c7d-IAD` is `IAD`. */
136export function coloOf(ray: string | null | undefined): string | null {
137 const m = /-([A-Za-z]{3,4})$/.exec((ray ?? "").trim());
138 return m ? m[1]!.toUpperCase() : null;
139}
140
141/** A check's requests together: it works when every one does, and takes as long as the slowest. */
142export function combine(results: ProbeResult[]): ProbeResult {
143 const ms = Math.max(0, ...results.map((r) => r.ms));
144 const failed = results.find((r) => !r.ok);
145 const colo = results.find((r) => r.colo)?.colo;
146 const out: ProbeResult = failed ? { ok: false, ms, error: failed.error ?? "no answer" } : { ok: true, ms };
147 return colo ? { ...out, colo } : out;
148}
149
150/** Whether a result is only slow: it worked, with nothing else wrong, but over `slowMs`. */
151export function onlySlow(result: ProbeResult | null, slowMs: number): boolean {
152 return result != null && result.ok && !result.degraded && Math.round(result.ms) > slowMs;
153}
154
155/**
156 * A slow check, and the same check run again at once: the better of the
157 * two. One slow answer (a cold isolate, a cache refill, a busy moment on
158 * the path) does not count when the next answers in time; slow twice is
159 * slow, at the faster of the two times. A second try that failed does not
160 * make a slow check worse. `first_ms` keeps the first try's time.
161 */
162export function confirmSlow(first: ProbeResult, again: ProbeResult | null): ProbeResult {
163 if (!again || !again.ok || again.degraded) return { ...first, first_ms: first.ms };
164 const better = again.ms < first.ms ? again : first;
165 const colo = better.colo ?? first.colo ?? again.colo;
166 return { ...better, ...(colo ? { colo } : {}), first_ms: first.ms };
167}
168
169/** What runs a check. `billing` is null when there is no binding to it. */
170export type Probers = {
171 fetch: Fetch;
172 billing: (() => Promise<unknown>) | null;
173 /** Git storage's recent health, through the repos service; null when not bound. */
174 storage?: (() => Promise<StorageReport>) | null;
175 timeoutMs?: number;
176};
177
178/** Runs one part's check. Null for a part with no check. */
179export async function runCheck(check: Check, probers: Probers): Promise<ProbeResult | null> {
180 const timeoutMs = probers.timeoutMs ?? TIMEOUT_MS;
181 switch (check.kind) {
182 case "http":
183 return combine(await Promise.all(check.steps.map((s) => step(probers.fetch, s, timeoutMs))));
184 case "billing": {
185 const billing = probers.billing;
186 if (!billing) return null;
187 return timed(async () => ((await billing()) ? true : "no price book"), timeoutMs);
188 }
189 case "storage": {
190 const storage = probers.storage;
191 if (!storage) return null;
192 let report: StorageReport | null = null;
193 const asked = await timed(async () => {
194 report = await storage();
195 return true;
196 }, timeoutMs);
197 return report ? judgeStorage(report) : asked;
198 }
199 case "none":
200 return null;
201 }
202}
203
204/**
205 * Runs one part's check as the cron does: an address that answered, but
206 * slowly, is asked once more straight away (`confirmSlow`) before the slow
207 * answer counts. Checks through a binding are not repeated: git storage
208 * reports the minutes gone by, and asking twice says the same.
209 */
210export async function probe(check: Check, probers: Probers, slowMs: number): Promise<ProbeResult | null> {
211 const first = await runCheck(check, probers);
212 if (check.kind !== "http" || !first || !onlySlow(first, slowMs)) return first;
213 return confirmSlow(first, await runCheck(check, probers));
214}