Sign in

flagon-io/g1t

public

g1t/apps/status/src/detect.test.ts

185 lines10,105 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1import assert from "node:assert/strict";
2import { test } from "node:test";
3
Fast pages, required checks on the branch, self-hosted runners, honest incidents4import {
5 DEPLOY_GRACE_MS,
6 DEPLOY_MAX_MS,
7 DETECT_AFTER,
8 RECOVERED_FOR_MS,
9 type DeployWindow,
10 type Streak,
11 autoDismissText,
12 deployChange,
13 deployQuiet,
14 detect,
15 detectedImpact,
16 draftTitle,
17 limitWords,
18 minutesWords,
19 recoverySentence,
20 settleDrafts,
21 troubleSentence,
22} from "./detect.ts";
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas23
24const at = (minute: number) => new Date(Date.UTC(2026, 9, 5, 12, minute));
25
26/** Runs rounds of checks through detection, carrying the streaks along. */
Fast pages, required checks on the branch, self-hosted runners, honest incidents27function run(
28 rounds: Record<string, "up" | "degraded" | "down">[],
29 open: { id: string; components: string[] }[] = [],
30 maintenance = new Set<string>(),
31 quiet: (minute: number) => boolean = () => false,
32) {
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas33 let streaks = new Map<string, Streak>();
34 const results = rounds.map((round, minute) => {
Fast pages, required checks on the branch, self-hosted runners, honest incidents35 const found = detect(streaks, Object.entries(round).map(([component, state]) => ({ component, state })), open, maintenance, at(minute), { quiet: quiet(minute) });
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas36 streaks = new Map(found.streaks.map((s) => [s.component, s]));
37 return found;
38 });
39 return results;
40}
41
42test(`a draft after ${DETECT_AFTER} failed checks in a row, once`, () => {
43 assert.equal(DETECT_AFTER, 3);
44 const rounds = run([{ api: "down" }, { api: "down" }, { api: "down" }, { api: "down" }]);
45 assert.deepEqual(rounds.map((r) => r.draft.length), [0, 0, 1, 0]);
Fast pages, required checks on the branch, self-hosted runners, honest incidents46 assert.deepEqual(rounds[2]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 3 });
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas47});
48
49test("a blip is not an incident: a good check resets the run", () => {
50 const rounds = run([{ api: "down" }, { api: "down" }, { api: "up" }, { api: "down" }, { api: "down" }]);
51 assert.ok(rounds.every((r) => r.draft.length === 0));
52 assert.equal(rounds[4]!.streaks[0]!.count, 2);
53});
54
55test("slow then failing is one run, at its worst", () => {
56 const rounds = run([{ docs: "degraded" }, { docs: "down" }, { docs: "degraded" }]);
Fast pages, required checks on the branch, self-hosted runners, honest incidents57 assert.deepEqual(rounds[2]!.draft, [{ key: "docs", state: "down", since: at(0).toISOString(), checks: 3 }]);
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas58});
59
60test("parts failing together share one draft", () => {
61 const rounds = run([{ api: "down", git: "down" }, { api: "down", git: "down" }, { api: "down", git: "down" }]);
62 assert.deepEqual(rounds[2]!.draft.map((d) => d.key), ["api", "git"]);
63});
64
65test("with an incident already open on the part: a line on it, no draft; recovery is noted", () => {
66 const open = [{ id: "inc1", components: ["api"] }];
67 const rounds = run([{ api: "down" }, { api: "down" }, { api: "down" }, { api: "up" }], open);
68 assert.equal(rounds[2]!.draft.length, 0);
Fast pages, required checks on the branch, self-hosted runners, honest incidents69 assert.deepEqual(rounds[2]!.failing, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 3 }]);
70 assert.deepEqual(rounds[3]!.recovered, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 3 }]);
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas71 assert.equal(rounds[3]!.streaks.length, 0);
72});
73
74test("a short blip that never crossed the line is not noted as a recovery", () => {
75 const rounds = run([{ api: "down" }, { api: "up" }], [{ id: "inc1", components: ["api"] }]);
76 assert.equal(rounds[1]!.recovered.length, 0);
77});
78
79test("parts under maintenance and unmonitored parts are left alone", () => {
80 const rounds = run([{ api: "down" }, { api: "down" }, { api: "down" }], [], new Set(["api"]));
81 assert.ok(rounds.every((r) => r.draft.length === 0 && r.streaks.length === 0));
82 const none = detect(new Map(), [{ component: "sandboxes", state: "unmonitored" }], [], new Set(), at(0));
83 assert.equal(none.streaks.length, 0);
84});
85
86test("the draft says what failed", () => {
87 assert.equal(detectedImpact("down"), "major_outage");
88 assert.equal(detectedImpact("degraded"), "degraded");
89 assert.equal(draftTitle([{ name: "API", state: "down" }, { name: "Git", state: "down" }]), "Detected: API and Git not answering");
90 assert.equal(
91 draftTitle([{ name: "API", state: "down" }, { name: "Git", state: "down" }, { name: "MCP", state: "down" }, { name: "Docs", state: "degraded" }]),
92 "Detected: API, Git and MCP not answering; Docs slow",
93 );
94});
Fast pages, required checks on the branch, self-hosted runners, honest incidents95
96test("a run that recovered is over: the next one starts fresh, with its own start", () => {
97 // 12:00 and 12:01 slow, 12:02 fine; then slow again from 12:10.
98 const rounds = run([
99 { git: "degraded" },
100 { git: "degraded" },
101 { git: "up" },
102 ...Array.from({ length: 7 }, () => ({ git: "up" as const })),
103 { git: "degraded" },
104 { git: "degraded" },
105 { git: "degraded" },
106 ]);
107 assert.equal(rounds[2]!.streaks.length, 0);
108 assert.equal(rounds[10]!.streaks[0]!.count, 1);
109 assert.equal(rounds[10]!.streaks[0]!.since, at(10).toISOString());
110 assert.equal(rounds[11]!.draft.length, 0, "two slow checks after a recovery are not three");
111 assert.deepEqual(rounds[12]!.draft, [{ key: "git", state: "degraded", since: at(10).toISOString(), checks: 3 }]);
112});
113
114test("during a deploy, trouble is counted but not drafted; it is drafted once it outlasts the window", () => {
115 // Quiet for minutes 0-3 (a deploy and its grace).
116 const quiet = (m: number) => m <= 3;
117 const blip = run([{ api: "down" }, { api: "down" }, { api: "down" }, { api: "down" }, { api: "up" }, { api: "up" }], [], new Set(), quiet);
118 assert.ok(blip.every((r) => r.draft.length === 0), "a restart that recovers inside the window is never drafted");
119 assert.deepEqual(blip[2]!.held, ["api"]);
120 const lasting = run([{ api: "down" }, { api: "down" }, { api: "down" }, { api: "down" }, { api: "down" }], [], new Set(), quiet);
121 assert.deepEqual(lasting.map((r) => r.draft.length), [0, 0, 0, 0, 1]);
122 assert.deepEqual(lasting[4]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 5 }, "the draft keeps the run's true start");
123});
124
125test("during a deploy, an incident already open still hears about its parts", () => {
126 const rounds = run([{ api: "down" }, { api: "down" }, { api: "down" }], [{ id: "inc1", components: ["api"] }], new Set(), () => true);
127 assert.equal(rounds[2]!.failing.length, 1);
128});
129
130test("the deploy window: running, finished plus grace, and a start that never finished", () => {
131 const t = (minute: number) => new Date(Date.UTC(2026, 9, 6, 7, minute));
132 assert.equal(deployQuiet(null, t(0)), false);
133 const running: DeployWindow = deployChange(null, "started", "abc", t(20));
134 assert.deepEqual(running, { id: "abc", started_at: t(20).toISOString(), finished_at: null });
135 assert.equal(deployQuiet(running, t(25)), true);
136 assert.equal(deployQuiet(running, t(10)), false, "trouble before the deploy is not forgiven");
137 assert.equal(deployQuiet(running, new Date(t(20).getTime() + DEPLOY_MAX_MS)), false, "a deploy that never said it finished stops counting");
138 const finished = deployChange(running, "finished", null, t(26));
139 assert.deepEqual(finished, { id: "abc", started_at: t(20).toISOString(), finished_at: t(26).toISOString() });
140 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS - 1)), true);
141 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS)), false);
142 // A second start while one runs keeps the first start.
143 assert.equal(deployChange(running, "started", "def", t(22)).started_at, t(20).toISOString());
144 // A new start after one finished begins a new window.
145 assert.equal(deployChange(finished, "started", "def", t(40)).started_at, t(40).toISOString());
146});
147
148test("a detected draft that recovers and stays healthy is dismissed; trouble in between starts the wait again", () => {
149 const draft = { id: "d1", title: "Detected: Git slow", components: ["git"], started_at: at(0).toISOString(), healthy_since: null as string | null };
150 const minute = 60_000;
151 // Still slow: not healthy.
152 assert.deepEqual(settleDrafts([draft], new Set(["git"]), at(3)), { healthy: [{ id: "d1", since: null }], dismiss: [] });
153 // Healthy from 12:04.
154 const first = settleDrafts([draft], new Set(), at(4));
155 assert.deepEqual(first.healthy, [{ id: "d1", since: at(4).toISOString() }]);
156 const waiting = { ...draft, healthy_since: at(4).toISOString() };
157 assert.equal(settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS - minute)).dismiss.length, 0);
158 // A slow check in between: the wait starts again.
159 assert.deepEqual(settleDrafts([waiting], new Set(["git"]), at(8)).healthy, [{ id: "d1", since: null }]);
160 // Healthy long enough: dismissed, resolved when it recovered, lasting 4 minutes.
161 const done = settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS));
162 assert.deepEqual(done.dismiss, [{ id: "d1", title: "Detected: Git slow", recovered_at: at(4).toISOString(), lasted_ms: 4 * minute }]);
163 assert.equal(done.healthy.length, 0);
164 // Other parts in trouble do not hold it up.
165 assert.equal(settleDrafts([waiting], new Set(["api"]), new Date(at(4).getTime() + RECOVERED_FOR_MS)).dismiss.length, 1);
166});
167
168test("slow and down are said apart, with the run's own start", () => {
169 assert.equal(limitWords(1500), "1.5 s");
170 assert.equal(limitWords(800), "800 ms");
171 assert.equal(
172 troubleSentence("Git and repositories", { state: "degraded", checks: 3 }, "6 Oct 07:25 UTC", 1500),
173 "Git and repositories has been slow — over 1.5 s — on 3 checks in a row since 6 Oct 07:25 UTC.",
174 );
175 assert.equal(troubleSentence("API", { state: "down", checks: 3 }, "6 Oct 07:25 UTC", 1500), "API has not answered on 3 checks in a row since 6 Oct 07:25 UTC.");
176 assert.match(recoverySentence("Git", { state: "degraded", checks: 4 }, "6 Oct 07:25 UTC"), /^Git is back to normal speed, after being slow on 4 checks/);
177 assert.match(recoverySentence("API", { state: "down", checks: 4 }, "6 Oct 07:25 UTC"), /^API is answering again, after not answering on 4 checks/);
178 assert.equal(minutesWords(30_000), "1 minute");
179 assert.equal(minutesWords(4 * 60_000), "4 minutes");
180 assert.equal(minutesWords(125 * 60_000), "2h 05m");
181 assert.equal(
182 autoDismissText(4 * 60_000, "6 Oct 07:29 UTC"),
183 "Recovered after 4 minutes, at 6 Oct 07:29 UTC, and stayed healthy for 10 minutes; dismissed automatically. It never appeared on the status page.",
184 );
185});