Skip to content
Sign in

flagon-io/g1t

public
0
262 linesCodeBlameRaw
1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import {
5 DEPLOY_GRACE_MS,
6 DEPLOY_MAX_MS,
7 DETECT_BAD,
8 DETECT_WINDOW,
9 RECOVER_AFTER,
10 RECOVERED_FOR_MS,
11 STALE_AFTER_MS,
12 STALE_EVERY_MS,
13 type DeployWindow,
14 type Streak,
15 autoDismissText,
16 deployChange,
17 deployQuiet,
18 detect,
19 detectedImpact,
20 draftTitle,
21 limitWords,
22 minutesWords,
23 quietUntil,
24 recoverySentence,
25 settleDrafts,
26 staleDrafts,
27 staleText,
28 troubleSentence,
29 troubledNow,
30 upgradeStreak,
31} from "./detect.ts";
32
33const at = (minute: number) => new Date(Date.UTC(2026, 9, 5, 12, minute));
34
35type State = "up" | "degraded" | "down";
36
37/** Runs rounds of checks through detection, carrying the streaks along. */
38function run(
39 rounds: Record<string, State>[],
40 open: { id: string; components: string[] }[] = [],
41 maintenance = new Set<string>(),
42 quiet: (minute: number) => boolean = () => false,
43 start = new Map<string, Streak>(),
44) {
45 let streaks = start;
46 const results = rounds.map((round, minute) => {
47 const found = detect(streaks, Object.entries(round).map(([component, state]) => ({ component, state })), open, maintenance, at(minute), { quiet: quiet(minute) });
48 streaks = new Map(found.streaks.map((s) => [s.component, s]));
49 return found;
50 });
51 return results;
52}
53
54/** One part's states as rounds: "x" down, "s" slow, "." up. */
55const rounds = (key: string, pattern: string): Record<string, State>[] =>
56 [...pattern].map((c) => ({ [key]: c === "x" ? "down" : c === "s" ? "degraded" : "up" }));
57
58test(`a draft at ${DETECT_BAD} bad of the last ${DETECT_WINDOW} checks, once`, () => {
59 assert.deepEqual([DETECT_BAD, DETECT_WINDOW, RECOVER_AFTER], [4, 5, 3]);
60 const found = run(rounds("api", "xxxxx"));
61 assert.deepEqual(found.map((r) => r.draft.length), [0, 0, 0, 1, 0]);
62 assert.deepEqual(found[3]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 });
63});
64
65test("N of M: a good check between does not hide trouble that keeps coming", () => {
66 // Three in a row used to be the line, and one good check reset it: down, down, up, down, down was never drafted.
67 const found = run(rounds("api", "xx.xx"));
68 assert.deepEqual(found.map((r) => r.draft.length), [0, 0, 0, 0, 1]);
69 assert.deepEqual(found[4]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 5 });
70});
71
72test("a blip is not an incident: three bad of five is under the line, and three good checks end the run", () => {
73 const flapping = run(rounds("api", "x.x.x.x"));
74 assert.ok(flapping.every((r) => r.draft.length === 0), "half the checks failing, alternately, is not four of five");
75 const blip = run(rounds("api", "xx...xx"));
76 assert.ok(blip.every((r) => r.draft.length === 0));
77 assert.equal(blip[4]!.streaks.length, 0, "three good checks in a row end the run");
78 assert.deepEqual(blip[6]!.streaks[0], { component: "api", state: "down", count: 2, checks: 2, recent: "xx", since: at(5).toISOString(), alerted: false });
79});
80
81test("slow then failing is one run, at its worst", () => {
82 const found = run(rounds("docs", "sxss"));
83 assert.deepEqual(found[3]!.draft, [{ key: "docs", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
84});
85
86test("parts failing together share one draft", () => {
87 const found = run(Array.from({ length: 4 }, () => ({ api: "down" as const, git: "down" as const })));
88 assert.deepEqual(found[3]!.draft.map((d) => d.key), ["api", "git"]);
89});
90
91test("with an incident already open on the part: a line on it, no draft; recovery is noted once the run ends", () => {
92 const open = [{ id: "inc1", components: ["api"] }];
93 const found = run(rounds("api", "xxxx..."), open);
94 assert.equal(found[3]!.draft.length, 0);
95 assert.deepEqual(found[3]!.failing, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
96 assert.deepEqual(found.map((r) => r.recovered.length), [0, 0, 0, 0, 0, 0, 1], "answering again is noted after three good checks, not the first");
97 assert.deepEqual(found[6]!.recovered, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
98 assert.equal(found[6]!.streaks.length, 0);
99});
100
101test("a run that never crossed the line is not noted as a recovery", () => {
102 const found = run(rounds("api", "xx..."), [{ id: "inc1", components: ["api"] }]);
103 assert.ok(found.every((r) => r.recovered.length === 0));
104});
105
106test("parts under maintenance and unmonitored parts are left alone", () => {
107 const found = run(rounds("api", "xxxxx"), [], new Set(["api"]));
108 assert.ok(found.every((r) => r.draft.length === 0 && r.streaks.length === 0));
109 const none = detect(new Map(), [{ component: "sandboxes", state: "unmonitored" }], [], new Set(), at(0));
110 assert.equal(none.streaks.length, 0);
111});
112
113test("the draft says what failed", () => {
114 assert.equal(detectedImpact("down"), "major_outage");
115 assert.equal(detectedImpact("degraded"), "degraded");
116 assert.equal(draftTitle([{ name: "API", state: "down" }, { name: "Git", state: "down" }]), "Detected: API and Git not answering");
117 assert.equal(
118 draftTitle([{ name: "API", state: "down" }, { name: "Git", state: "down" }, { name: "MCP", state: "down" }, { name: "Docs", state: "degraded" }]),
119 "Detected: API, Git and MCP not answering; Docs slow",
120 );
121});
122
123test("a run that recovered is over: the next one starts fresh, with its own start", () => {
124 // 12:00 and 12:01 slow, then fine; slow again from 12:10.
125 const found = run(rounds("git", "ss........ssss"));
126 assert.equal(found[4]!.streaks.length, 0);
127 assert.equal(found[10]!.streaks[0]!.count, 1);
128 assert.equal(found[10]!.streaks[0]!.since, at(10).toISOString());
129 assert.equal(found[12]!.draft.length, 0, "three slow checks after a recovery are not four");
130 assert.deepEqual(found[13]!.draft, [{ key: "git", state: "degraded", since: at(10).toISOString(), checks: 4, of: 4 }]);
131});
132
133test("a run slow for hours, raised long ago, still ends when the checks go healthy", () => {
134 // As kept before N of M: 300 slow checks in a row, raised.
135 const old = upgradeStreak({ component: "speed", state: "degraded", count: 300, since: at(0).toISOString(), alerted: true });
136 assert.deepEqual([old.checks, old.recent], [300, "sssss"]);
137 const open = [{ id: "d1", components: ["speed"] }];
138 const found = run(rounds("speed", "s.s...."), open, new Set(), () => false, new Map([["speed", old]]));
139 assert.ok(found.every((r) => r.draft.length === 0 && r.failing.length === 0), "already raised: nothing new");
140 assert.deepEqual(found.map((r) => r.recovered.length), [0, 0, 0, 0, 0, 1, 0]);
141 assert.deepEqual(found[5]!.recovered[0], { incident: "d1", key: "speed", state: "degraded", since: at(0).toISOString(), checks: 302, of: 303 });
142 assert.equal(found[5]!.streaks.length, 0);
143 // While the run lasts, only a bad latest check holds a draft up (settleDrafts).
144 assert.deepEqual(found.slice(0, 5).map((r) => r.streaks.filter(troubledNow).length), [1, 0, 1, 0, 0]);
145});
146
147test("during a deploy, trouble is counted but not drafted; it is drafted once it outlasts the window", () => {
148 // Quiet for minutes 0-4 (a deploy and its grace).
149 const quiet = (m: number) => m <= 4;
150 const blip = run(rounds("api", "xxxxx..."), [], new Set(), quiet);
151 assert.ok(blip.every((r) => r.draft.length === 0), "a restart that recovers inside the window is never drafted");
152 assert.deepEqual(blip[3]!.held, ["api"]);
153 const lasting = run(rounds("api", "xxxxxx"), [], new Set(), quiet);
154 assert.deepEqual(lasting.map((r) => r.draft.length), [0, 0, 0, 0, 0, 1]);
155 assert.deepEqual(lasting[5]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 6, of: 6 }, "the draft keeps the run's true start");
156});
157
158test("during a deploy, an incident already open still hears about its parts", () => {
159 const found = run(rounds("api", "xxxx"), [{ id: "inc1", components: ["api"] }], new Set(), () => true);
160 assert.equal(found[3]!.failing.length, 1);
161});
162
163test("the deploy window: running, finished plus grace, and a start that never finished", () => {
164 const t = (minute: number) => new Date(Date.UTC(2026, 9, 6, 7, minute));
165 assert.equal(deployQuiet(null, t(0)), false);
166 const running: DeployWindow = deployChange(null, "started", "abc", t(20));
167 assert.deepEqual(running, { id: "abc", started_at: t(20).toISOString(), finished_at: null, running: 1, last_started_at: t(20).toISOString() });
168 assert.equal(deployQuiet(running, t(25)), true);
169 assert.equal(deployQuiet(running, t(10)), false, "trouble before the deploy is not forgiven");
170 assert.equal(deployQuiet(running, new Date(t(20).getTime() + DEPLOY_MAX_MS)), false, "a deploy that never said it finished stops counting");
171 const finished = deployChange(running, "finished", null, t(26));
172 assert.deepEqual(finished, { id: "abc", started_at: t(20).toISOString(), finished_at: t(26).toISOString(), running: 0, last_started_at: t(20).toISOString() });
173 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS - 1)), true);
174 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS)), false);
175 assert.equal(quietUntil(finished), new Date(t(26).getTime() + DEPLOY_GRACE_MS).toISOString());
176 // A new start after the window closed begins a new window.
177 assert.equal(deployChange(finished, "started", "def", t(40)).started_at, t(40).toISOString());
178});
179
180test("deploys that overlap are one window, finished when the last one is", () => {
181 const t = (minute: number) => new Date(Date.UTC(2026, 9, 6, 7, minute));
182 // Two jobs of a stage start; the first finishes early.
183 let w = deployChange(null, "started", "abc", t(0));
184 w = deployChange(w, "started", "abc", t(1));
185 assert.equal(w.running, 2);
186 w = deployChange(w, "finished", "abc", t(4));
187 assert.deepEqual([w.finished_at, w.running], [null, 1], "one still running: not finished");
188 assert.equal(deployQuiet(w, t(12)), true, "the other job's restarts at 12 minutes are still forgiven");
189 w = deployChange(w, "finished", "abc", t(14));
190 assert.deepEqual([w.started_at, w.finished_at, w.running], [t(0).toISOString(), t(14).toISOString(), 0]);
191 // The next stage starts inside the grace period: the same window, from its first start.
192 w = deployChange(w, "started", "abc", t(15));
193 assert.deepEqual([w.started_at, w.finished_at, w.running], [t(0).toISOString(), null, 1]);
194 // The 30-minute cap counts from the latest start, not the first.
195 assert.equal(deployQuiet(w, t(40)), true);
196 assert.equal(deployQuiet(w, new Date(t(15).getTime() + DEPLOY_MAX_MS)), false);
197});
198
199test("a detected draft that recovers and stays healthy is dismissed; trouble in between starts the wait again", () => {
200 const draft = {
201 id: "d1",
202 title: "Detected: Git slow",
203 components: ["git"],
204 started_at: at(0).toISOString(),
205 declared_at: at(3).toISOString(),
206 healthy_since: null as string | null,
207 reminded_at: null,
208 };
209 const minute = 60_000;
210 // Still slow: not healthy.
211 assert.deepEqual(settleDrafts([draft], new Set(["git"]), at(3)), { healthy: [{ id: "d1", since: null }], dismiss: [] });
212 // Healthy from 12:04.
213 const first = settleDrafts([draft], new Set(), at(4));
214 assert.deepEqual(first.healthy, [{ id: "d1", since: at(4).toISOString() }]);
215 const waiting = { ...draft, healthy_since: at(4).toISOString() };
216 assert.equal(settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS - minute)).dismiss.length, 0);
217 // A slow check in between: the wait starts again.
218 assert.deepEqual(settleDrafts([waiting], new Set(["git"]), at(8)).healthy, [{ id: "d1", since: null }]);
219 // Healthy long enough: dismissed, resolved when it recovered, lasting 4 minutes.
220 const done = settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS));
221 assert.deepEqual(done.dismiss, [{ id: "d1", title: "Detected: Git slow", recovered_at: at(4).toISOString(), lasted_ms: 4 * minute }]);
222 assert.equal(done.healthy.length, 0);
223 // Other parts in trouble do not hold it up.
224 assert.equal(settleDrafts([waiting], new Set(["api"]), new Date(at(4).getTime() + RECOVERED_FOR_MS)).dismiss.length, 1);
225});
226
227test("a draft nobody acknowledges is raised again after 45 minutes, once, then every 6 hours", () => {
228 assert.deepEqual([STALE_AFTER_MS, STALE_EVERY_MS], [45 * 60_000, 6 * 3_600_000]);
229 const made = at(0);
230 const later = (ms: number) => new Date(made.getTime() + ms);
231 const draft = { id: "d1", title: "Detected: Page speed slow", declared_at: made.toISOString(), reminded_at: null as string | null };
232 assert.deepEqual(staleDrafts([draft], later(STALE_AFTER_MS - 1)), []);
233 assert.deepEqual(staleDrafts([draft], later(STALE_AFTER_MS)), [{ id: "d1", title: draft.title, waiting_ms: STALE_AFTER_MS }]);
234 // Raised at 45 minutes: not again the minute after, nor at 90 minutes.
235 const reminded = { ...draft, reminded_at: later(STALE_AFTER_MS).toISOString() };
236 assert.deepEqual(staleDrafts([reminded], later(STALE_AFTER_MS + 60_000)), []);
237 assert.deepEqual(staleDrafts([reminded], later(2 * STALE_AFTER_MS)), []);
238 assert.deepEqual(staleDrafts([reminded], later(STALE_AFTER_MS + STALE_EVERY_MS - 1)), []);
239 assert.equal(staleDrafts([reminded], later(STALE_AFTER_MS + STALE_EVERY_MS)).length, 1);
240 assert.equal(staleText(STALE_AFTER_MS, true), "Unacknowledged for 45 minutes. The alert address was emailed again.");
241 assert.equal(staleText(STALE_AFTER_MS + STALE_EVERY_MS, false), "Unacknowledged for 6h 45m.");
242});
243
244test("slow and down are said apart, with the run's own start", () => {
245 assert.equal(limitWords(1500), "1.5 s");
246 assert.equal(limitWords(800), "800 ms");
247 assert.equal(
248 troubleSentence("Git and repositories", { state: "degraded", checks: 4, of: 4 }, "6 Oct 07:25 UTC", 1500),
249 "Git and repositories has been slow — over 1.5 s — on 4 checks in a row since 6 Oct 07:25 UTC.",
250 );
251 assert.equal(troubleSentence("API", { state: "down", checks: 4, of: 5 }, "6 Oct 07:25 UTC", 1500), "API has not answered on 4 of 5 checks since 6 Oct 07:25 UTC.");
252 assert.equal(troubleSentence("API", { state: "down", checks: 3 }, "6 Oct 07:25 UTC", 1500), "API has not answered on 3 checks in a row since 6 Oct 07:25 UTC.");
253 assert.match(recoverySentence("Git", { state: "degraded", checks: 4, of: 4 }, "6 Oct 07:25 UTC"), /^Git is back to normal speed, after being slow on 4 checks in a row/);
254 assert.match(recoverySentence("API", { state: "down", checks: 40, of: 42 }, "6 Oct 07:25 UTC"), /^API is answering again, after not answering on 40 of 42 checks/);
255 assert.equal(minutesWords(30_000), "1 minute");
256 assert.equal(minutesWords(4 * 60_000), "4 minutes");
257 assert.equal(minutesWords(125 * 60_000), "2h 05m");
258 assert.equal(
259 autoDismissText(4 * 60_000, "6 Oct 07:29 UTC"),
260 "Recovered after 4 minutes, at 6 Oct 07:29 UTC, and stayed healthy for 10 minutes; dismissed automatically. It never appeared on the status page.",
261 );
262});