Sign in

flagon-io/g1t

public

g1t/apps/status/src/detect.ts

214 lines10,190 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1/**
2 * Noticing trouble before anyone reports it: when a part fails (or is
3 * slow) for DETECT_AFTER checks in a row, a draft incident is made for
4 * staff in sudo, not shown on the status page until someone publishes it.
5 * When a part that had crossed that line answers again, open incidents on
Fast pages, required checks on the branch, self-hosted runners, honest incidents6 * it get a note. A detected draft nobody has picked up is dismissed on its
7 * own once its parts have stayed healthy for RECOVERED_FOR_MS, and while a
8 * deploy is running (and for DEPLOY_GRACE_MS after) no new draft is made
9 * unless the trouble outlasts it. No Workers imports, so it is tested
10 * under Node.
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas11 */
12import type { ComponentImpact, StatusComponentState } from "@g1t/contracts";
13
14/** Checks in a row, a minute apart, before a draft is made. */
15export const DETECT_AFTER = 3;
Fast pages, required checks on the branch, self-hosted runners, honest incidents16/** How long a detected draft's parts stay healthy before it is dismissed on its own. */
17export const RECOVERED_FOR_MS = 10 * 60_000;
18/** How long after a deploy finishes its restarts are still forgiven. */
19export const DEPLOY_GRACE_MS = 3 * 60_000;
20/** A deploy that said it started and never said it finished stops counting after this. */
21export const DEPLOY_MAX_MS = 30 * 60_000;
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas22
Fast pages, required checks on the branch, self-hosted runners, honest incidents23/**
24 * One part's current run of failed or slow checks. Only parts that were
25 * failing or slow on the latest check have one: any good check ends it,
26 * and the next trouble starts a new run, with its own start.
27 */
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas28export type Streak = {
29 component: string;
30 /** The worst seen in this run of failures. */
31 state: "degraded" | "down";
32 count: number;
Fast pages, required checks on the branch, self-hosted runners, honest incidents33 /** The first failed or slow check of this run. */
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas34 since: string;
35 /** Whether it has crossed the line, and been raised. */
36 alerted: boolean;
37};
38
39/** An open incident (draft or public), with the parts it affects. */
40export type OpenRef = { id: string; components: string[] };
41
Fast pages, required checks on the branch, self-hosted runners, honest incidents42export type Trouble = { key: string; state: "degraded" | "down"; since: string; checks: number };
43
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas44export type Detection = {
45 /** Every failing part's run after this round; parts not here have none. */
46 streaks: Streak[];
47 /** Parts that crossed the line with no open incident on them: one draft for all. */
Fast pages, required checks on the branch, self-hosted runners, honest incidents48 draft: Trouble[];
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas49 /** Parts that crossed the line while an incident on them was open. */
Fast pages, required checks on the branch, self-hosted runners, honest incidents50 failing: (Trouble & { incident: string })[];
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas51 /** Parts answering again after crossing the line. */
Fast pages, required checks on the branch, self-hosted runners, honest incidents52 recovered: { incident: string; key: string; state: "degraded" | "down"; since: string; checks: number }[];
53 /** Parts that crossed the line during a deploy: kept counting, not raised yet. */
54 held: string[];
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas55};
56
Fast pages, required checks on the branch, self-hosted runners, honest incidents57export type DetectOptions = {
58 threshold?: number;
59 /** A deploy is running, or just finished: no new drafts, only counting. */
60 quiet?: boolean;
61};
62
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas63export function detect(
64 previous: Map<string, Streak>,
65 observations: { component: string; state: StatusComponentState }[],
66 open: OpenRef[],
67 maintenance: Set<string>,
68 at: Date,
Fast pages, required checks on the branch, self-hosted runners, honest incidents69 { threshold = DETECT_AFTER, quiet = false }: DetectOptions = {},
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas70): Detection {
Fast pages, required checks on the branch, self-hosted runners, honest incidents71 const out: Detection = { streaks: [], draft: [], failing: [], recovered: [], held: [] };
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas72 const covering = (key: string) => open.filter((i) => i.components.includes(key));
73 for (const { component: key, state } of observations) {
74 const prev = previous.get(key);
75 if (state === "unmonitored" || maintenance.has(key)) continue;
76 if (state !== "degraded" && state !== "down") {
Fast pages, required checks on the branch, self-hosted runners, honest incidents77 // A good check: the run, if any, is over. Leaving it out of `streaks` ends it.
78 if (prev?.alerted) for (const i of covering(key)) out.recovered.push({ incident: i.id, key, state: prev.state, since: prev.since, checks: prev.count });
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas79 continue;
80 }
81 const streak: Streak = prev
82 ? { ...prev, count: prev.count + 1, state: prev.state === "down" || state === "down" ? "down" : "degraded" }
83 : { component: key, state, count: 1, since: at.toISOString(), alerted: false };
84 if (!streak.alerted && streak.count >= threshold) {
Fast pages, required checks on the branch, self-hosted runners, honest incidents85 const trouble = { key, state: streak.state, since: streak.since, checks: streak.count };
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas86 const incidents = covering(key);
Fast pages, required checks on the branch, self-hosted runners, honest incidents87 if (incidents.length) {
88 streak.alerted = true;
89 for (const i of incidents) out.failing.push({ incident: i.id, ...trouble });
90 } else if (quiet) {
91 out.held.push(key);
92 } else {
93 streak.alerted = true;
94 out.draft.push(trouble);
95 }
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas96 }
97 out.streaks.push(streak);
98 }
99 return out;
100}
101
Fast pages, required checks on the branch, self-hosted runners, honest incidents102// --- Deploys --------------------------------------------------------------------------
103
104/** A deploy as the deploy tool reported it. */
105export type DeployWindow = { id: string | null; started_at: string; finished_at: string | null };
106
107/** Whether detection holds off at `at`: during a deploy, and for a grace period after. */
108export function deployQuiet(window: DeployWindow | null, at: Date): boolean {
109 if (!window) return false;
110 const start = Date.parse(window.started_at);
111 const t = at.getTime();
112 if (Number.isNaN(start) || t < start - 60_000) return false;
113 if (window.finished_at) return t < Date.parse(window.finished_at) + DEPLOY_GRACE_MS;
114 return t < start + DEPLOY_MAX_MS;
115}
116
117/** The deploy tool's start or finish, folded into what is kept. */
118export function deployChange(window: DeployWindow | null, phase: "started" | "finished", id: string | null, at: Date): DeployWindow {
119 const now = at.toISOString();
120 if (phase === "started") {
121 // A second start while one is running keeps the earlier start.
122 const running = window && !window.finished_at && deployQuiet(window, at);
123 return { id, started_at: running ? window.started_at : now, finished_at: null };
124 }
125 const running = window && !window.finished_at && deployQuiet(window, at);
126 return { id: id ?? window?.id ?? null, started_at: running ? window.started_at : now, finished_at: now };
127}
128
129// --- Detected drafts that recover ---------------------------------------------------------
130
131/** A detected draft no one has picked up yet, and since when its parts have been healthy. */
132export type WatchedDraft = { id: string; title: string; components: string[]; started_at: string; healthy_since: string | null };
133
134export type Settled = {
135 /** Every watched draft's healthy-since after this round: null while a part is in trouble. */
136 healthy: { id: string; since: string | null }[];
137 /** Drafts healthy long enough to dismiss. */
138 dismiss: { id: string; title: string; recovered_at: string; lasted_ms: number }[];
139};
140
141/**
142 * Which detected drafts have recovered for good. `troubled` is every part
143 * with a run of failed or slow checks after this round.
144 */
145export function settleDrafts(drafts: WatchedDraft[], troubled: Set<string>, at: Date, after = RECOVERED_FOR_MS): Settled {
146 const out: Settled = { healthy: [], dismiss: [] };
147 for (const d of drafts) {
148 if (d.components.some((k) => troubled.has(k))) {
149 out.healthy.push({ id: d.id, since: null });
150 continue;
151 }
152 const since = d.healthy_since ?? at.toISOString();
153 if (at.getTime() - Date.parse(since) >= after) {
154 out.dismiss.push({ id: d.id, title: d.title, recovered_at: since, lasted_ms: Math.max(0, Date.parse(since) - Date.parse(d.started_at)) });
155 } else {
156 out.healthy.push({ id: d.id, since });
157 }
158 }
159 return out;
160}
161
162// --- Wording --------------------------------------------------------------------------------
163
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas164/** What a detected failure does to its part, for the draft. */
165export function detectedImpact(state: "degraded" | "down"): ComponentImpact {
166 return state === "down" ? "major_outage" : "degraded";
167}
168
169/** The draft's title: "Detected: API and Git not answering". */
170export function draftTitle(parts: { name: string; state: "degraded" | "down" }[]): string {
171 const list = (names: string[]) => (names.length <= 2 ? names.join(" and ") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`);
172 const down = parts.filter((p) => p.state === "down").map((p) => p.name);
173 const slow = parts.filter((p) => p.state === "degraded").map((p) => p.name);
174 const said = [down.length ? `${list(down)} not answering` : "", slow.length ? `${list(slow)} slow` : ""].filter(Boolean).join("; ");
175 return `Detected: ${said}`.slice(0, 120);
176}
Fast pages, required checks on the branch, self-hosted runners, honest incidents177
178/** A part's slow line in words: "1.5 s", "800 ms". */
179export function limitWords(ms: number): string {
180 return ms >= 1000 ? `${Number((ms / 1000).toFixed(1))} s` : `${ms} ms`;
181}
182
183/** "3 minutes", "1 minute", "2h 05m". */
184export function minutesWords(ms: number): string {
185 const minutes = Math.max(1, Math.round(ms / 60_000));
186 if (minutes < 60) return `${minutes} minute${minutes === 1 ? "" : "s"}`;
187 return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, "0")}m`;
188}
189
190/**
191 * One part's trouble in a sentence, slow and down said apart:
192 * "Git has been slow — over 1.5 s — on 3 checks in a row since 6 Oct 07:25 UTC."
193 * "API has not answered on 3 checks in a row since 6 Oct 07:25 UTC."
194 * `since` is already written out, in whatever zone the reader needs.
195 */
196export function troubleSentence(name: string, t: { state: "degraded" | "down"; checks: number }, since: string, slowMs: number): string {
197 const checks = `${t.checks} check${t.checks === 1 ? "" : "s"} in a row`;
198 return t.state === "down"
199 ? `${name} has not answered on ${checks} since ${since}.`
200 : `${name} has been slow — over ${limitWords(slowMs)} — on ${checks} since ${since}.`;
201}
202
203/** A part answering again, after a run that crossed the line. */
204export function recoverySentence(name: string, t: { state: "degraded" | "down"; checks: number }, since: string): string {
205 const checks = `${t.checks} check${t.checks === 1 ? "" : "s"} in a row`;
206 return t.state === "down"
207 ? `${name} is answering again, after not answering on ${checks} since ${since}.`
208 : `${name} is back to normal speed, after being slow on ${checks} since ${since}.`;
209}
210
211/** The timeline line, and the follow-up email's words, when a recovered draft is dismissed on its own. */
212export function autoDismissText(lastedMs: number, recoveredAt: string, healthyFor = RECOVERED_FOR_MS): string {
213 return `Recovered after ${minutesWords(lastedMs)}, at ${recoveredAt}, and stayed healthy for ${minutesWords(healthyFor)}; dismissed automatically. It never appeared on the status page.`;
214}