Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
329 linesCodeBlameRaw
1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import {
5 DEPLOY_GRACE_MS,
6 DEPLOY_MAX_MS,
7 DETECT_BAD,
8 DETECT_WINDOW,
9 RECOVER_AFTER,
10 RECOVERED_FOR_MS,
11 STALE_AFTER_MS,
12 STALE_EVERY_MS,
13 type DeployWindow,
14 type Streak,
15 autoDismissText,
16 deployChange,
17 deployQuiet,
18 detect,
19 detectedImpact,
20 draftTitle,
21 limitWords,
22 minutesWords,
23 quietUntil,
24 recoverySentence,
25 settleDrafts,
26 staleDrafts,
27 staleText,
28 troubleSentence,
29 troubledNow,
30 upgradeStreak,
31} from "./detect.ts";
32
33const at = (minute: number) => new Date(Date.UTC(2026, 9, 5, 12, minute));
34
35type State = "up" | "degraded" | "down";
36
37/** Runs rounds of checks through detection, carrying the streaks along. */
38function run(
39 rounds: Record<string, State>[],
40 open: { id: string; components: string[] }[] = [],
41 maintenance = new Set<string>(),
42 quiet: (minute: number) => boolean = () => false,
43 start = new Map<string, Streak>(),
44) {
45 let streaks = start;
46 const results = rounds.map((round, minute) => {
47 const found = detect(
48 streaks,
49 Object.entries(round).map(([component, state]) => ({ component, state })),
50 open,
51 maintenance,
52 at(minute),
53 { quiet: quiet(minute) },
54 );
55 streaks = new Map(found.streaks.map((s) => [s.component, s]));
56 return found;
57 });
58 return results;
59}
60
61/** One part's states as rounds: "x" down, "s" slow, "." up. */
62const rounds = (key: string, pattern: string): Record<string, State>[] =>
63 [...pattern].map((c) => ({ [key]: c === "x" ? "down" : c === "s" ? "degraded" : "up" }));
64
65test(`a draft at ${DETECT_BAD} bad of the last ${DETECT_WINDOW} checks, once`, () => {
66 assert.deepEqual([DETECT_BAD, DETECT_WINDOW, RECOVER_AFTER], [4, 5, 3]);
67 const found = run(rounds("api", "xxxxx"));
68 assert.deepEqual(
69 found.map((r) => r.draft.length),
70 [0, 0, 0, 1, 0],
71 );
72 assert.deepEqual(found[3]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 });
73});
74
75test("N of M: a good check between does not hide trouble that keeps coming", () => {
76 // Three in a row used to be the line, and one good check reset it: down, down, up, down, down was never drafted.
77 const found = run(rounds("api", "xx.xx"));
78 assert.deepEqual(
79 found.map((r) => r.draft.length),
80 [0, 0, 0, 0, 1],
81 );
82 assert.deepEqual(found[4]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 5 });
83});
84
85test("a blip is not an incident: three bad of five is under the line, and three good checks end the run", () => {
86 const flapping = run(rounds("api", "x.x.x.x"));
87 assert.ok(
88 flapping.every((r) => r.draft.length === 0),
89 "half the checks failing, alternately, is not four of five",
90 );
91 const blip = run(rounds("api", "xx...xx"));
92 assert.ok(blip.every((r) => r.draft.length === 0));
93 assert.equal(blip[4]!.streaks.length, 0, "three good checks in a row end the run");
94 assert.deepEqual(blip[6]!.streaks[0], { component: "api", state: "down", count: 2, checks: 2, recent: "xx", since: at(5).toISOString(), alerted: false });
95});
96
97test("slow then failing is one run, at its worst", () => {
98 const found = run(rounds("docs", "sxss"));
99 assert.deepEqual(found[3]!.draft, [{ key: "docs", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
100});
101
102test("parts failing together share one draft", () => {
103 const found = run(Array.from({ length: 4 }, () => ({ api: "down" as const, git: "down" as const })));
104 assert.deepEqual(
105 found[3]!.draft.map((d) => d.key),
106 ["api", "git"],
107 );
108});
109
110test("with an incident already open on the part: a line on it, no draft; recovery is noted once the run ends", () => {
111 const open = [{ id: "inc1", components: ["api"] }];
112 const found = run(rounds("api", "xxxx..."), open);
113 assert.equal(found[3]!.draft.length, 0);
114 assert.deepEqual(found[3]!.failing, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
115 assert.deepEqual(
116 found.map((r) => r.recovered.length),
117 [0, 0, 0, 0, 0, 0, 1],
118 "answering again is noted after three good checks, not the first",
119 );
120 assert.deepEqual(found[6]!.recovered, [{ incident: "inc1", key: "api", state: "down", since: at(0).toISOString(), checks: 4, of: 4 }]);
121 assert.equal(found[6]!.streaks.length, 0);
122});
123
124test("a run that never crossed the line is not noted as a recovery", () => {
125 const found = run(rounds("api", "xx..."), [{ id: "inc1", components: ["api"] }]);
126 assert.ok(found.every((r) => r.recovered.length === 0));
127});
128
129test("parts under maintenance and unmonitored parts are left alone", () => {
130 const found = run(rounds("api", "xxxxx"), [], new Set(["api"]));
131 assert.ok(found.every((r) => r.draft.length === 0 && r.streaks.length === 0));
132 const none = detect(new Map(), [{ component: "sandboxes", state: "unmonitored" }], [], new Set(), at(0));
133 assert.equal(none.streaks.length, 0);
134});
135
136test("the draft says what failed", () => {
137 assert.equal(detectedImpact("down"), "major_outage");
138 assert.equal(detectedImpact("degraded"), "degraded");
139 assert.equal(
140 draftTitle([
141 { name: "API", state: "down" },
142 { name: "Git", state: "down" },
143 ]),
144 "Detected: API and Git not answering",
145 );
146 assert.equal(
147 draftTitle([
148 { name: "API", state: "down" },
149 { name: "Git", state: "down" },
150 { name: "MCP", state: "down" },
151 { name: "Docs", state: "degraded" },
152 ]),
153 "Detected: API, Git and MCP not answering; Docs slow",
154 );
155});
156
157test("a run that recovered is over: the next one starts fresh, with its own start", () => {
158 // 12:00 and 12:01 slow, then fine; slow again from 12:10.
159 const found = run(rounds("git", "ss........ssss"));
160 assert.equal(found[4]!.streaks.length, 0);
161 assert.equal(found[10]!.streaks[0]!.count, 1);
162 assert.equal(found[10]!.streaks[0]!.since, at(10).toISOString());
163 assert.equal(found[12]!.draft.length, 0, "three slow checks after a recovery are not four");
164 assert.deepEqual(found[13]!.draft, [{ key: "git", state: "degraded", since: at(10).toISOString(), checks: 4, of: 4 }]);
165});
166
167test("a run slow for hours, raised long ago, still ends when the checks go healthy", () => {
168 // As kept before N of M: 300 slow checks in a row, raised.
169 const old = upgradeStreak({ component: "speed", state: "degraded", count: 300, since: at(0).toISOString(), alerted: true });
170 assert.deepEqual([old.checks, old.recent], [300, "sssss"]);
171 const open = [{ id: "d1", components: ["speed"] }];
172 const found = run(rounds("speed", "s.s...."), open, new Set(), () => false, new Map([["speed", old]]));
173 assert.ok(
174 found.every((r) => r.draft.length === 0 && r.failing.length === 0),
175 "already raised: nothing new",
176 );
177 assert.deepEqual(
178 found.map((r) => r.recovered.length),
179 [0, 0, 0, 0, 0, 1, 0],
180 );
181 assert.deepEqual(found[5]!.recovered[0], { incident: "d1", key: "speed", state: "degraded", since: at(0).toISOString(), checks: 302, of: 303 });
182 assert.equal(found[5]!.streaks.length, 0);
183 // While the run lasts, only a bad latest check holds a draft up (settleDrafts).
184 assert.deepEqual(
185 found.slice(0, 5).map((r) => r.streaks.filter(troubledNow).length),
186 [1, 0, 1, 0, 0],
187 );
188});
189
190test("during a deploy, trouble is counted but not drafted; it is drafted once it outlasts the window", () => {
191 // Quiet for minutes 0-4 (a deploy and its grace).
192 const quiet = (m: number) => m <= 4;
193 const blip = run(rounds("api", "xxxxx..."), [], new Set(), quiet);
194 assert.ok(
195 blip.every((r) => r.draft.length === 0),
196 "a restart that recovers inside the window is never drafted",
197 );
198 assert.deepEqual(blip[3]!.held, ["api"]);
199 const lasting = run(rounds("api", "xxxxxx"), [], new Set(), quiet);
200 assert.deepEqual(
201 lasting.map((r) => r.draft.length),
202 [0, 0, 0, 0, 0, 1],
203 );
204 assert.deepEqual(lasting[5]!.draft[0], { key: "api", state: "down", since: at(0).toISOString(), checks: 6, of: 6 }, "the draft keeps the run's true start");
205});
206
207test("during a deploy, an incident already open still hears about its parts", () => {
208 const found = run(rounds("api", "xxxx"), [{ id: "inc1", components: ["api"] }], new Set(), () => true);
209 assert.equal(found[3]!.failing.length, 1);
210});
211
212test("the deploy window: running, finished plus grace, and a start that never finished", () => {
213 const t = (minute: number) => new Date(Date.UTC(2026, 9, 6, 7, minute));
214 assert.equal(deployQuiet(null, t(0)), false);
215 const running: DeployWindow = deployChange(null, "started", "abc", t(20));
216 assert.deepEqual(running, { id: "abc", started_at: t(20).toISOString(), finished_at: null, running: 1, last_started_at: t(20).toISOString() });
217 assert.equal(deployQuiet(running, t(25)), true);
218 assert.equal(deployQuiet(running, t(10)), false, "trouble before the deploy is not forgiven");
219 assert.equal(deployQuiet(running, new Date(t(20).getTime() + DEPLOY_MAX_MS)), false, "a deploy that never said it finished stops counting");
220 const finished = deployChange(running, "finished", null, t(26));
221 assert.deepEqual(finished, {
222 id: "abc",
223 started_at: t(20).toISOString(),
224 finished_at: t(26).toISOString(),
225 running: 0,
226 last_started_at: t(20).toISOString(),
227 });
228 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS - 1)), true);
229 assert.equal(deployQuiet(finished, new Date(t(26).getTime() + DEPLOY_GRACE_MS)), false);
230 assert.equal(quietUntil(finished), new Date(t(26).getTime() + DEPLOY_GRACE_MS).toISOString());
231 // A new start after the window closed begins a new window.
232 assert.equal(deployChange(finished, "started", "def", t(40)).started_at, t(40).toISOString());
233});
234
235test("deploys that overlap are one window, finished when the last one is", () => {
236 const t = (minute: number) => new Date(Date.UTC(2026, 9, 6, 7, minute));
237 // Two jobs of a stage start; the first finishes early.
238 let w = deployChange(null, "started", "abc", t(0));
239 w = deployChange(w, "started", "abc", t(1));
240 assert.equal(w.running, 2);
241 w = deployChange(w, "finished", "abc", t(4));
242 assert.deepEqual([w.finished_at, w.running], [null, 1], "one still running: not finished");
243 assert.equal(deployQuiet(w, t(12)), true, "the other job's restarts at 12 minutes are still forgiven");
244 w = deployChange(w, "finished", "abc", t(14));
245 assert.deepEqual([w.started_at, w.finished_at, w.running], [t(0).toISOString(), t(14).toISOString(), 0]);
246 // The next stage starts inside the grace period: the same window, from its first start.
247 w = deployChange(w, "started", "abc", t(15));
248 assert.deepEqual([w.started_at, w.finished_at, w.running], [t(0).toISOString(), null, 1]);
249 // The 30-minute cap counts from the latest start, not the first.
250 assert.equal(deployQuiet(w, t(40)), true);
251 assert.equal(deployQuiet(w, new Date(t(15).getTime() + DEPLOY_MAX_MS)), false);
252});
253
254test("a detected draft that recovers and stays healthy is dismissed; trouble in between starts the wait again", () => {
255 const draft = {
256 id: "d1",
257 title: "Detected: Git slow",
258 components: ["git"],
259 started_at: at(0).toISOString(),
260 declared_at: at(3).toISOString(),
261 healthy_since: null as string | null,
262 reminded_at: null,
263 };
264 const minute = 60_000;
265 // Still slow: not healthy.
266 assert.deepEqual(settleDrafts([draft], new Set(["git"]), at(3)), { healthy: [{ id: "d1", since: null }], dismiss: [] });
267 // Healthy from 12:04.
268 const first = settleDrafts([draft], new Set(), at(4));
269 assert.deepEqual(first.healthy, [{ id: "d1", since: at(4).toISOString() }]);
270 const waiting = { ...draft, healthy_since: at(4).toISOString() };
271 assert.equal(settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS - minute)).dismiss.length, 0);
272 // A slow check in between: the wait starts again.
273 assert.deepEqual(settleDrafts([waiting], new Set(["git"]), at(8)).healthy, [{ id: "d1", since: null }]);
274 // Healthy long enough: dismissed, resolved when it recovered, lasting 4 minutes.
275 const done = settleDrafts([waiting], new Set(), new Date(at(4).getTime() + RECOVERED_FOR_MS));
276 assert.deepEqual(done.dismiss, [{ id: "d1", title: "Detected: Git slow", recovered_at: at(4).toISOString(), lasted_ms: 4 * minute }]);
277 assert.equal(done.healthy.length, 0);
278 // Other parts in trouble do not hold it up.
279 assert.equal(settleDrafts([waiting], new Set(["api"]), new Date(at(4).getTime() + RECOVERED_FOR_MS)).dismiss.length, 1);
280});
281
282test("a draft nobody acknowledges is raised again after 45 minutes, once, then every 6 hours", () => {
283 assert.deepEqual([STALE_AFTER_MS, STALE_EVERY_MS], [45 * 60_000, 6 * 3_600_000]);
284 const made = at(0);
285 const later = (ms: number) => new Date(made.getTime() + ms);
286 const draft = { id: "d1", title: "Detected: Page speed slow", declared_at: made.toISOString(), reminded_at: null as string | null };
287 assert.deepEqual(staleDrafts([draft], later(STALE_AFTER_MS - 1)), []);
288 assert.deepEqual(staleDrafts([draft], later(STALE_AFTER_MS)), [{ id: "d1", title: draft.title, waiting_ms: STALE_AFTER_MS }]);
289 // Raised at 45 minutes: not again the minute after, nor at 90 minutes.
290 const reminded = { ...draft, reminded_at: later(STALE_AFTER_MS).toISOString() };
291 assert.deepEqual(staleDrafts([reminded], later(STALE_AFTER_MS + 60_000)), []);
292 assert.deepEqual(staleDrafts([reminded], later(2 * STALE_AFTER_MS)), []);
293 assert.deepEqual(staleDrafts([reminded], later(STALE_AFTER_MS + STALE_EVERY_MS - 1)), []);
294 assert.equal(staleDrafts([reminded], later(STALE_AFTER_MS + STALE_EVERY_MS)).length, 1);
295 assert.equal(staleText(STALE_AFTER_MS, true), "Unacknowledged for 45 minutes. The alert address was emailed again.");
296 assert.equal(staleText(STALE_AFTER_MS + STALE_EVERY_MS, false), "Unacknowledged for 6h 45m.");
297});
298
299test("slow and down are said apart, with the run's own start", () => {
300 assert.equal(limitWords(1500), "1.5 s");
301 assert.equal(limitWords(800), "800 ms");
302 assert.equal(
303 troubleSentence("Git and repositories", { state: "degraded", checks: 4, of: 4 }, "6 Oct 07:25 UTC", 1500),
304 "Git and repositories has been slow — over 1.5 s — on 4 checks in a row since 6 Oct 07:25 UTC.",
305 );
306 assert.equal(
307 troubleSentence("API", { state: "down", checks: 4, of: 5 }, "6 Oct 07:25 UTC", 1500),
308 "API has not answered on 4 of 5 checks since 6 Oct 07:25 UTC.",
309 );
310 assert.equal(
311 troubleSentence("API", { state: "down", checks: 3 }, "6 Oct 07:25 UTC", 1500),
312 "API has not answered on 3 checks in a row since 6 Oct 07:25 UTC.",
313 );
314 assert.match(
315 recoverySentence("Git", { state: "degraded", checks: 4, of: 4 }, "6 Oct 07:25 UTC"),
316 /^Git is back to normal speed, after being slow on 4 checks in a row/,
317 );
318 assert.match(
319 recoverySentence("API", { state: "down", checks: 40, of: 42 }, "6 Oct 07:25 UTC"),
320 /^API is answering again, after not answering on 40 of 42 checks/,
321 );
322 assert.equal(minutesWords(30_000), "1 minute");
323 assert.equal(minutesWords(4 * 60_000), "4 minutes");
324 assert.equal(minutesWords(125 * 60_000), "2h 05m");
325 assert.equal(
326 autoDismissText(4 * 60_000, "6 Oct 07:29 UTC"),
327 "Recovered after 4 minutes, at 6 Oct 07:29 UTC, and stayed healthy for 10 minutes; dismissed automatically. It never appeared on the status page.",
328 );
329});