Skip to content
Sign in

flagon-io/g1t

public
0
355 linesCodeBlameRaw
1/**
2 * Noticing trouble before anyone reports it: when a part fails (or is
3 * slow) on DETECT_BAD of its last DETECT_WINDOW checks, a draft incident
4 * is made for staff in sudo, not shown on the status page until someone
5 * publishes it. A run of trouble ends after RECOVER_AFTER good checks in a
6 * row, however long it lasted; when a run that had crossed the line ends,
7 * open incidents on its part get a note. A detected draft nobody has
8 * picked up is dismissed on its own once its parts have stayed healthy for
9 * RECOVERED_FOR_MS, and one left unacknowledged is raised again after
10 * STALE_AFTER_MS, then every STALE_EVERY_MS. While a deploy is running
11 * (and for DEPLOY_GRACE_MS after) no new draft is made unless the trouble
12 * outlasts it. No Workers imports, so it is tested under Node.
13 *
14 * A slow answer only counts once the check, asked again at once, is slow
15 * too (probe.ts `probe`), so one cold start is not a slow check.
16 */
17import type { ComponentImpact, StatusComponentState } from "@g1t/contracts";
18
19/** Failed or slow checks, of the last DETECT_WINDOW, before a draft is made. */
20export const DETECT_BAD = 4;
21/** How many of a part's latest checks DETECT_BAD counts over (a minute apart). */
22export const DETECT_WINDOW = 5;
23/** Good checks in a row that end a run of trouble, whether or not it was raised. */
24export const RECOVER_AFTER = 3;
25/** How long a detected draft's parts stay healthy before it is dismissed on its own. */
26export const RECOVERED_FOR_MS = 10 * 60_000;
27/** How long a detected draft waits, unacknowledged, before the alert goes out again. */
28export const STALE_AFTER_MS = 45 * 60_000;
29/** After that, how often it goes out again while the draft still waits. */
30export const STALE_EVERY_MS = 6 * 60 * 60_000;
31/** How long after a deploy finishes its restarts are still forgiven. */
32export const DEPLOY_GRACE_MS = 3 * 60_000;
33/** A deploy that said it started and never said it finished stops counting after this. */
34export const DEPLOY_MAX_MS = 30 * 60_000;
35
36/** One check in a run's `recent`: good, slow, or not answering. */
37export type Mark = "." | "s" | "x";
38
39/**
40 * One part's current run of trouble: from its first failed or slow check
41 * until RECOVER_AFTER good checks in a row. Good checks inside the run
42 * (flapping) do not end it; they are counted in `checks` and `recent`.
43 */
44export type Streak = {
45 component: string;
46 /** The worst seen in this run. */
47 state: "degraded" | "down";
48 /** Failed or slow checks in this run. */
49 count: number;
50 /** Every check in this run, from its first failed or slow one. */
51 checks: number;
52 /** The run's latest checks, oldest first, at most DETECT_WINDOW: "." good, "s" slow, "x" not answering. */
53 recent: string;
54 /** The first failed or slow check of this run. */
55 since: string;
56 /** Whether it has crossed the line, and been raised. */
57 alerted: boolean;
58};
59
60/** An open incident (draft or public), with the parts it affects. */
61export type OpenRef = { id: string; components: string[] };
62
63/** A run that crossed the line: `checks` failed or slow of the `of` checks since `since`. */
64export type Trouble = { key: string; state: "degraded" | "down"; since: string; checks: number; of: number };
65
66export type Detection = {
67 /** Every part's run still going after this round; parts not here have none. */
68 streaks: Streak[];
69 /** Parts that crossed the line with no open incident on them: one draft for all. */
70 draft: Trouble[];
71 /** Parts that crossed the line while an incident on them was open. */
72 failing: (Trouble & { incident: string })[];
73 /** Parts whose raised run ended: answering again, in time. */
74 recovered: (Trouble & { incident: string })[];
75 /** Parts that crossed the line during a deploy: kept counting, not raised yet. */
76 held: string[];
77};
78
79export type DetectOptions = {
80 /** Failed or slow checks of the last `window` that cross the line. */
81 bad?: number;
82 window?: number;
83 /** Good checks in a row that end a run. */
84 recoverAfter?: number;
85 /** A deploy is running, or just finished: no new drafts, only counting. */
86 quiet?: boolean;
87};
88
89const isBad = (m: string) => m === "s" || m === "x";
90
91function mark(state: StatusComponentState): Mark {
92 return state === "down" ? "x" : state === "degraded" ? "s" : ".";
93}
94
95/** How many of `recent` were failed or slow. */
96export function badIn(recent: string): number {
97 return [...recent].filter(isBad).length;
98}
99
100/** Good checks at the end of `recent`. */
101function goodTail(recent: string): number {
102 let n = 0;
103 for (let i = recent.length - 1; i >= 0 && recent[i] === "."; i--) n++;
104 return n;
105}
106
107/** Whether a run's latest check was failed or slow: what keeps a draft from counting as healthy. */
108export function troubledNow(streak: Streak): boolean {
109 return isBad(streak.recent.at(-1) ?? "x");
110}
111
112/**
113 * A run as kept before `checks` and `recent` (migration 0003): every one
114 * of its checks failed, in a row.
115 */
116export function upgradeStreak(s: Omit<Streak, "checks" | "recent"> & { checks?: number | null; recent?: string | null }, window = DETECT_WINDOW): Streak {
117 const count = Math.max(1, Number(s.count) || 1);
118 const recent = s.recent || (s.state === "down" ? "x" : "s").repeat(Math.min(count, window));
119 return { ...s, count, checks: Math.max(Number(s.checks) || 0, count), recent };
120}
121
122export function detect(
123 previous: Map<string, Streak>,
124 observations: { component: string; state: StatusComponentState }[],
125 open: OpenRef[],
126 maintenance: Set<string>,
127 at: Date,
128 { bad: need = DETECT_BAD, window = DETECT_WINDOW, recoverAfter = RECOVER_AFTER, quiet = false }: DetectOptions = {},
129): Detection {
130 const out: Detection = { streaks: [], draft: [], failing: [], recovered: [], held: [] };
131 const covering = (key: string) => open.filter((i) => i.components.includes(key));
132 for (const { component: key, state } of observations) {
133 const prev = previous.get(key);
134 if (state === "unmonitored" || maintenance.has(key)) continue;
135 const m = mark(state);
136 if (!prev && !isBad(m)) continue;
137 const recent = `${prev?.recent ?? ""}${m}`.slice(-window);
138 const streak: Streak = prev
139 ? {
140 ...prev,
141 recent,
142 checks: prev.checks + 1,
143 count: prev.count + (isBad(m) ? 1 : 0),
144 state: prev.state === "down" || state === "down" ? "down" : "degraded",
145 }
146 : { component: key, state: state as "degraded" | "down", count: 1, checks: 1, recent, since: at.toISOString(), alerted: false };
147 const trouble = (): Trouble => ({ key, state: streak.state, since: streak.since, checks: streak.count, of: streak.checks });
148 if (!isBad(m)) {
149 // Enough good checks in a row end the run, however long it was. Leaving it out of `streaks` ends it.
150 if (goodTail(recent) >= Math.min(recoverAfter, window)) {
151 if (streak.alerted) {
152 // The good checks that ended it are not part of the trouble.
153 const t = { ...trouble(), of: streak.checks - goodTail(recent) };
154 for (const i of covering(key)) out.recovered.push({ incident: i.id, ...t });
155 }
156 continue;
157 }
158 } else if (!streak.alerted && badIn(recent) >= need) {
159 const t = trouble();
160 const incidents = covering(key);
161 if (incidents.length) {
162 streak.alerted = true;
163 for (const i of incidents) out.failing.push({ incident: i.id, ...t });
164 } else if (quiet) {
165 out.held.push(key);
166 } else {
167 streak.alerted = true;
168 out.draft.push(t);
169 }
170 }
171 out.streaks.push(streak);
172 }
173 return out;
174}
175
176// --- Deploys --------------------------------------------------------------------------
177
178/**
179 * A deploy as the deploy tool reported it. Deploys that overlap (the jobs
180 * of one stage, in parallel) are one window: `running` counts the starts
181 * not yet finished, and it is finished when the last one is.
182 */
183export type DeployWindow = {
184 id: string | null;
185 started_at: string;
186 finished_at: string | null;
187 /** Starts not yet finished. */
188 running: number;
189 /** The latest start: a window not finished stops counting DEPLOY_MAX_MS after it. */
190 last_started_at: string;
191};
192
193/** Whether detection holds off at `at`: during a deploy, and for a grace period after. */
194export function deployQuiet(window: DeployWindow | null, at: Date): boolean {
195 if (!window) return false;
196 const start = Date.parse(window.started_at);
197 const t = at.getTime();
198 if (Number.isNaN(start) || t < start - 60_000) return false;
199 if (window.finished_at) return t < Date.parse(window.finished_at) + DEPLOY_GRACE_MS;
200 const last = Date.parse(window.last_started_at);
201 return t < (Number.isNaN(last) ? start : Math.max(start, last)) + DEPLOY_MAX_MS;
202}
203
204/** When detection stops holding off for this window. */
205export function quietUntil(window: DeployWindow): string {
206 const end = window.finished_at
207 ? Date.parse(window.finished_at) + DEPLOY_GRACE_MS
208 : Math.max(Date.parse(window.started_at), Date.parse(window.last_started_at)) + DEPLOY_MAX_MS;
209 return new Date(end).toISOString();
210}
211
212/** The deploy tool's start or finish, folded into what is kept. */
213export function deployChange(window: DeployWindow | null, phase: "started" | "finished", id: string | null, at: Date): DeployWindow {
214 const now = at.toISOString();
215 // Still quiet: running, or in its grace period. A start then joins the window, keeping its start.
216 const live = window && deployQuiet(window, at) ? window : null;
217 const running = live && !live.finished_at ? Math.max(1, live.running) : 0;
218 if (phase === "started") {
219 return { id: id ?? live?.id ?? null, started_at: live ? live.started_at : now, finished_at: null, running: running + 1, last_started_at: now };
220 }
221 if (running > 1) return { ...live!, id: live!.id ?? id, running: running - 1 };
222 // The last one running finished; a finish with no start is a window of its own.
223 return {
224 id: id ?? window?.id ?? null,
225 started_at: running ? live!.started_at : now,
226 finished_at: now,
227 running: 0,
228 last_started_at: running ? live!.last_started_at : now,
229 };
230}
231
232// --- Detected drafts that recover, and drafts left waiting ---------------------------------
233
234/** A detected draft no one has picked up yet, since when its parts have been healthy, and when it was last raised again. */
235export type WatchedDraft = {
236 id: string;
237 title: string;
238 components: string[];
239 started_at: string;
240 /** When the draft was made: STALE_AFTER_MS counts from here. */
241 declared_at: string;
242 healthy_since: string | null;
243 /** When the alert last went out again for it; null before the first time. */
244 reminded_at: string | null;
245};
246
247export type Settled = {
248 /** Every watched draft's healthy-since after this round: null while a part is in trouble. */
249 healthy: { id: string; since: string | null }[];
250 /** Drafts healthy long enough to dismiss. */
251 dismiss: { id: string; title: string; recovered_at: string; lasted_ms: number }[];
252};
253
254/**
255 * Which detected drafts have recovered for good. `troubled` is every part
256 * whose latest check was failed or slow, in a run still going (detect.ts
257 * `troubledNow`).
258 */
259export function settleDrafts(drafts: WatchedDraft[], troubled: Set<string>, at: Date, after = RECOVERED_FOR_MS): Settled {
260 const out: Settled = { healthy: [], dismiss: [] };
261 for (const d of drafts) {
262 if (d.components.some((k) => troubled.has(k))) {
263 out.healthy.push({ id: d.id, since: null });
264 continue;
265 }
266 const since = d.healthy_since ?? at.toISOString();
267 if (at.getTime() - Date.parse(since) >= after) {
268 out.dismiss.push({ id: d.id, title: d.title, recovered_at: since, lasted_ms: Math.max(0, Date.parse(since) - Date.parse(d.started_at)) });
269 } else {
270 out.healthy.push({ id: d.id, since });
271 }
272 }
273 return out;
274}
275
276/**
277 * Detected drafts to raise again: unacknowledged for STALE_AFTER_MS since
278 * they were made, the first time; then every STALE_EVERY_MS after the last
279 * time. `drafts` are the ones still waiting after this round's dismissals.
280 */
281export function staleDrafts(
282 drafts: Pick<WatchedDraft, "id" | "title" | "declared_at" | "reminded_at">[],
283 at: Date,
284 { after = STALE_AFTER_MS, every = STALE_EVERY_MS } = {},
285): { id: string; title: string; waiting_ms: number }[] {
286 const t = at.getTime();
287 return drafts
288 .filter((d) => (d.reminded_at ? t - Date.parse(d.reminded_at) >= every : t - Date.parse(d.declared_at) >= after))
289 .map((d) => ({ id: d.id, title: d.title, waiting_ms: Math.max(0, t - Date.parse(d.declared_at)) }));
290}
291
292// --- Wording --------------------------------------------------------------------------------
293
294/** What a detected failure does to its part, for the draft. */
295export function detectedImpact(state: "degraded" | "down"): ComponentImpact {
296 return state === "down" ? "major_outage" : "degraded";
297}
298
299/** The draft's title: "Detected: API and Git not answering". */
300export function draftTitle(parts: { name: string; state: "degraded" | "down" }[]): string {
301 const list = (names: string[]) => (names.length <= 2 ? names.join(" and ") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`);
302 const down = parts.filter((p) => p.state === "down").map((p) => p.name);
303 const slow = parts.filter((p) => p.state === "degraded").map((p) => p.name);
304 const said = [down.length ? `${list(down)} not answering` : "", slow.length ? `${list(slow)} slow` : ""].filter(Boolean).join("; ");
305 return `Detected: ${said}`.slice(0, 120);
306}
307
308/** A part's slow line in words: "1.5 s", "800 ms". */
309export function limitWords(ms: number): string {
310 return ms >= 1000 ? `${Number((ms / 1000).toFixed(1))} s` : `${ms} ms`;
311}
312
313/** "3 minutes", "1 minute", "2h 05m". */
314export function minutesWords(ms: number): string {
315 const minutes = Math.max(1, Math.round(ms / 60_000));
316 if (minutes < 60) return `${minutes} minute${minutes === 1 ? "" : "s"}`;
317 return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, "0")}m`;
318}
319
320/** "4 checks in a row", or "4 of 5 checks" when good ones came between. */
321function checksWords(t: { checks: number; of?: number }): string {
322 const of = t.of ?? t.checks;
323 return of > t.checks ? `${t.checks} of ${of} checks` : `${t.checks} check${t.checks === 1 ? "" : "s"} in a row`;
324}
325
326/**
327 * One part's trouble in a sentence, slow and down said apart:
328 * "Git has been slow — over 1.5 s — on 4 checks in a row since 6 Oct 07:25 UTC."
329 * "API has not answered on 4 of 5 checks since 6 Oct 07:25 UTC."
330 * `since` is already written out, in whatever zone the reader needs.
331 */
332export function troubleSentence(name: string, t: { state: "degraded" | "down"; checks: number; of?: number }, since: string, slowMs: number): string {
333 const checks = checksWords(t);
334 return t.state === "down"
335 ? `${name} has not answered on ${checks} since ${since}.`
336 : `${name} has been slow — over ${limitWords(slowMs)} — on ${checks} since ${since}.`;
337}
338
339/** A part answering again, after a run that crossed the line. */
340export function recoverySentence(name: string, t: { state: "degraded" | "down"; checks: number; of?: number }, since: string): string {
341 const checks = checksWords(t);
342 return t.state === "down"
343 ? `${name} is answering again, after not answering on ${checks} since ${since}.`
344 : `${name} is back to normal speed, after being slow on ${checks} since ${since}.`;
345}
346
347/** The timeline line, and the follow-up email's words, when a recovered draft is dismissed on its own. */
348export function autoDismissText(lastedMs: number, recoveredAt: string, healthyFor = RECOVERED_FOR_MS): string {
349 return `Recovered after ${minutesWords(lastedMs)}, at ${recoveredAt}, and stayed healthy for ${minutesWords(healthyFor)}; dismissed automatically. It never appeared on the status page.`;
350}
351
352/** The timeline line when a draft left waiting is raised again. */
353export function staleText(waitingMs: number, emailed: boolean): string {
354 return `Unacknowledged for ${minutesWords(waitingMs)}.${emailed ? " The alert address was emailed again." : ""}`;
355}