Skip to content
Sign in

flagon-io/g1t

public
0
287 linesCodeBlameRaw
1/**
2 * Detection through the database, as the cron runs it: streaks saved and
3 * read back each round, deploy windows, and recovered drafts dismissed.
4 * D1 here is SQLite (node:sqlite) behind the few D1 calls the store makes,
5 * with the real migrations applied.
6 */
7import assert from "node:assert/strict";
8import { readFileSync, readdirSync } from "node:fs";
9import { DatabaseSync } from "node:sqlite";
10import { test } from "node:test";
11
12import { type Streak, autoDismissText, deployChange, detect, settleDrafts, staleDrafts, staleText } from "./detect.ts";
13import { stamp } from "./postmortem.ts";
14import {
15 CHECK_HISTORY_DAYS,
16 autoDismiss,
17 checkHistory,
18 createIncident,
19 historySpan,
20 incidentDetail,
21 loadDeploy,
22 loadStreaks,
23 openRefs,
24 record,
25 saveDeploy,
26 saveHealthy,
27 saveReminders,
28 saveStreaks,
29 watchedDrafts,
30} from "./store.ts";
31
32type Params = (string | number | null)[];
33
34/** Just enough of D1 over SQLite for the store. */
35function d1(): D1Database {
36 const db = new DatabaseSync(":memory:");
37 const dir = new URL("../migrations/", import.meta.url);
38 for (const file of readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) db.exec(readFileSync(new URL(file, dir), "utf8"));
39 class Statement {
40 constructor(
41 readonly sql: string,
42 readonly params: Params = [],
43 ) {}
44 bind(...params: Params) {
45 return new Statement(this.sql, params);
46 }
47 async all() {
48 return { results: db.prepare(this.sql).all(...this.params), success: true, meta: {} };
49 }
50 async first() {
51 return db.prepare(this.sql).get(...this.params) ?? null;
52 }
53 async run() {
54 const r = db.prepare(this.sql).run(...this.params);
55 return { results: [], success: true, meta: { changes: Number(r.changes) } };
56 }
57 }
58 return {
59 prepare: (sql: string) => new Statement(sql),
60 batch: async (list: Statement[]) => {
61 db.exec("BEGIN");
62 try {
63 const out = [];
64 for (const s of list) out.push(/^\s*SELECT/i.test(s.sql) ? await s.all() : await s.run());
65 db.exec("COMMIT");
66 return out;
67 } catch (error) {
68 db.exec("ROLLBACK");
69 throw error;
70 }
71 },
72 } as unknown as D1Database;
73}
74
75const at = (h: number, m: number) => new Date(Date.UTC(2026, 9, 6, h, m, 32));
76
77/** One cron round's detection against the database; returns what it found. */
78async function round(db: D1Database, now: Date, git: "up" | "degraded" | "down", quiet = false) {
79 const found = detect(await loadStreaks(db), [{ component: "git", state: git }, { component: "api", state: "up" }], await openRefs(db), new Set(), now, { quiet });
80 await saveStreaks(db, found.streaks);
81 return found;
82}
83
84test("the 6 Oct draft: a slow run at 03:13 that recovered does not leak into the one at 07:25", async () => {
85 const db = d1();
86 // 03:13 and 03:14 slow, then fine: never four of five.
87 assert.equal((await round(db, at(3, 13), "degraded")).draft.length, 0);
88 assert.equal((await round(db, at(3, 14), "degraded")).draft.length, 0);
89 await round(db, at(3, 15), "up");
90 assert.equal((await loadStreaks(db)).size, 1, "one good check does not end a run");
91 await round(db, at(3, 16), "up");
92 await round(db, at(3, 17), "up");
93 assert.equal((await loadStreaks(db)).size, 0, "three good checks in a row, with nothing else failing, leave no run behind");
94 // 07:25 slow: the first check of a new run, not the third of the old one.
95 const first = await round(db, at(7, 25), "degraded");
96 assert.equal(first.draft.length, 0);
97 assert.deepEqual([...(await loadStreaks(db)).values()], [{ component: "git", state: "degraded", count: 1, checks: 1, recent: "s", since: at(7, 25).toISOString(), alerted: false }]);
98 await round(db, at(7, 26), "degraded");
99 await round(db, at(7, 27), "degraded");
100 const fourth = await round(db, at(7, 28), "degraded");
101 assert.deepEqual(fourth.draft, [{ key: "git", state: "degraded", since: at(7, 25).toISOString(), checks: 4, of: 4 }]);
102});
103
104test("a run kept before N of M (no checks, no recent) is read as all bad, and still ends", async () => {
105 const db = d1();
106 await db.prepare(`INSERT INTO streak (component, state, count, since, alerted) VALUES ('git', 'degraded', 180, ?1, 1)`).bind(at(4, 0).toISOString()).run();
107 assert.deepEqual((await loadStreaks(db)).get("git"), { component: "git", state: "degraded", count: 180, checks: 180, recent: "sssss", since: at(4, 0).toISOString(), alerted: true });
108 await round(db, at(7, 0), "up");
109 await round(db, at(7, 1), "up");
110 assert.equal((await loadStreaks(db)).size, 1);
111 await round(db, at(7, 2), "up");
112 assert.equal((await loadStreaks(db)).size, 0);
113});
114
115test("a run is kept while it lasts, and only the parts still failing keep one", async () => {
116 const db = d1();
117 const streak = (component: string): Streak => ({ component, state: "down", count: 2, checks: 3, recent: "x.x", since: at(1, 0).toISOString(), alerted: false });
118 await saveStreaks(db, [streak("git"), streak("api")]);
119 await saveStreaks(db, [streak("api")]);
120 assert.deepEqual([...(await loadStreaks(db)).keys()], ["api"]);
121 assert.deepEqual((await loadStreaks(db)).get("api"), streak("api"));
122 await saveStreaks(db, []);
123 assert.equal((await loadStreaks(db)).size, 0);
124});
125
126test("the deploy window is kept between rounds", async () => {
127 const db = d1();
128 assert.equal(await loadDeploy(db), null);
129 await saveDeploy(db, deployChange(null, "started", "run-1", at(7, 20)));
130 await saveDeploy(db, deployChange(await loadDeploy(db), "finished", null, at(7, 24)));
131 assert.deepEqual(await loadDeploy(db), { id: "run-1", started_at: at(7, 20).toISOString(), finished_at: at(7, 24).toISOString(), running: 0, last_started_at: at(7, 20).toISOString() });
132 // A window kept before deploys were counted reads as one deploy.
133 await db.prepare(`UPDATE meta SET value = ?1 WHERE key = 'deploy'`).bind(JSON.stringify({ id: "old", started_at: at(8, 0).toISOString(), finished_at: null })).run();
134 assert.deepEqual(await loadDeploy(db), { id: "old", started_at: at(8, 0).toISOString(), finished_at: null, running: 1, last_started_at: at(8, 0).toISOString() });
135});
136
137test("every check is kept for 7 days, with where it ran from, and an incident's page reads its parts' checks", async () => {
138 const db = d1();
139 const minute = (m: number) => new Date(at(7, 0).getTime() + m * 60_000);
140 for (let m = 0; m < 5; m++) {
141 await record(
142 db,
143 [
144 { component: "speed", state: m < 3 ? "degraded" : "up", detail: "", latency_ms: m < 3 ? 1900 : 300, colo: "IAD", first_ms: m < 3 ? 2400 : null },
145 { component: "api", state: "up", detail: "", latency_ms: 90, colo: "IAD" },
146 { component: "sandboxes", state: "unmonitored", detail: "", latency_ms: null },
147 ],
148 minute(m),
149 );
150 }
151 const kept = await checkHistory(db, ["speed"], minute(0), minute(4));
152 assert.equal(kept.length, 5);
153 assert.deepEqual(kept[0], { component: "speed", at: minute(0).toISOString(), ms: 1900, outcome: "degraded", colo: "IAD", first_ms: 2400 });
154 assert.deepEqual(kept[4], { component: "speed", at: minute(4).toISOString(), ms: 300, outcome: "up", colo: "IAD", first_ms: null });
155 assert.equal((await checkHistory(db, ["sandboxes"], minute(0), minute(4))).length, 0, "a part with no check keeps no history");
156 // A round 7 days later prunes everything older.
157 await record(db, [{ component: "api", state: "up", detail: "", latency_ms: 80, colo: "SJC" }], new Date(minute(2).getTime() + CHECK_HISTORY_DAYS * 86_400_000));
158 const left = await checkHistory(db, ["speed", "api"], minute(0), new Date(minute(0).getTime() + 8 * 86_400_000));
159 assert.deepEqual(left.map((c) => `${c.component}@${c.colo}`), ["api@IAD", "speed@IAD", "api@IAD", "speed@IAD", "api@IAD", "speed@IAD", "api@SJC"], "the two oldest rounds are gone");
160
161 // The incident page: its parts' checks around it, with the slow line.
162 const id = await createIncident(
163 db,
164 {
165 title: "Detected: Page speed slow",
166 severity: "sev3",
167 status: "investigating",
168 visibility: "draft",
169 source: "detected",
170 components: [{ key: "speed", impact: "degraded" }],
171 started_at: minute(0).toISOString(),
172 acknowledged_at: null,
173 commander: null,
174 communications: null,
175 by: "status",
176 },
177 [{ kind: "detected", public: false, status: null, text: "Detected." }],
178 minute(4),
179 null,
180 );
181 const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map(), { limits: new Map([["speed", 800]]), now: minute(5) }))!;
182 assert.equal(detail.checks.length, 1);
183 assert.deepEqual([detail.checks[0]!.key, detail.checks[0]!.slow_ms, detail.checks[0]!.samples.length], ["speed", 800, 3]);
184 assert.deepEqual(historySpan(minute(0).toISOString(), null, minute(5)), { from: minute(-30), to: minute(5) });
185 assert.deepEqual(historySpan(minute(0).toISOString(), minute(10).toISOString(), minute(500)), { from: minute(-30), to: minute(40) });
186 // A long one shows its last day.
187 assert.deepEqual(historySpan(minute(0).toISOString(), minute(3000).toISOString(), minute(4000)), { from: minute(3030 - 1440), to: minute(3030) });
188});
189
190test("a draft left waiting is raised again, with a line on its timeline, and forgotten once it is not waiting", async () => {
191 const db = d1();
192 const id = await createIncident(
193 db,
194 {
195 title: "Detected: Page speed slow",
196 severity: "sev3",
197 status: "investigating",
198 visibility: "draft",
199 source: "detected",
200 components: [{ key: "speed", impact: "degraded" }],
201 started_at: at(7, 0).toISOString(),
202 acknowledged_at: null,
203 commander: null,
204 communications: null,
205 by: "status",
206 },
207 [{ kind: "detected", public: false, status: null, text: "Detected." }],
208 at(7, 4),
209 null,
210 );
211 const tick = async (now: Date) => {
212 const waiting = await watchedDrafts(db);
213 const due = staleDrafts(waiting, now);
214 await saveReminders(db, waiting.map((d) => d.id), due.map((d) => ({ id: d.id, text: staleText(d.waiting_ms, true) })), now);
215 return due.map((d) => d.id);
216 };
217 assert.deepEqual(await tick(at(7, 48)), []);
218 assert.deepEqual(await tick(at(7, 49)), [id], "45 minutes after the draft was made");
219 assert.equal((await watchedDrafts(db))[0]!.reminded_at, at(7, 49).toISOString());
220 assert.deepEqual(await tick(at(7, 50)), [], "once");
221 assert.deepEqual(await tick(at(13, 48)), []);
222 assert.deepEqual(await tick(at(13, 49)), [id], "then every 6 hours");
223 const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map()))!;
224 assert.deepEqual(detail.timeline.filter((e) => e.kind === "note").map((e) => [e.by, e.text]), [
225 ["status", "Unacknowledged for 45 minutes. The alert address was emailed again."],
226 ["status", "Unacknowledged for 6h 45m. The alert address was emailed again."],
227 ]);
228 assert.equal(detail.acknowledged_at, null, "a reminder is not an acknowledgement");
229 // Picked up: no longer watched, and its bookkeeping goes.
230 await db.prepare(`UPDATE incident SET acknowledged_at = ?1 WHERE id = ?2`).bind(at(14, 0).toISOString(), id).run();
231 assert.deepEqual(await tick(at(20, 0)), []);
232 assert.equal((await db.prepare(`SELECT COUNT(*) AS n FROM meta WHERE key LIKE 'draft_reminded:%'`).first<{ n: number }>())!.n, 0);
233});
234
235test("a detected draft that recovered is dismissed after ten healthy minutes; one staff picked up is left alone", async () => {
236 const db = d1();
237 const draft = (title: string, acknowledged: string | null) =>
238 createIncident(
239 db,
240 {
241 title,
242 severity: "sev3",
243 status: "investigating",
244 visibility: "draft",
245 source: "detected",
246 components: [{ key: "git", impact: "degraded" }],
247 started_at: at(7, 25).toISOString(),
248 acknowledged_at: acknowledged,
249 commander: null,
250 communications: null,
251 by: "status",
252 },
253 [{ kind: "detected", public: false, status: null, text: "Detected." }],
254 at(7, 27),
255 null,
256 );
257 const id = await draft("Detected: Git and repositories slow", null);
258 const picked = await draft("Detected: Git and repositories slow (picked up)", at(7, 28).toISOString());
259
260 const tick = async (now: Date, troubled: string[]) => {
261 const settled = settleDrafts(await watchedDrafts(db), new Set(troubled), now);
262 await saveHealthy(db, settled.healthy);
263 const dismissed: string[] = [];
264 for (const d of settled.dismiss) if (await autoDismiss(db, d.id, d.recovered_at, autoDismissText(d.lasted_ms, stamp(d.recovered_at)), now)) dismissed.push(d.id);
265 return dismissed;
266 };
267 assert.deepEqual((await watchedDrafts(db)).map((d) => d.id), [id], "only untouched drafts are watched");
268 await tick(at(7, 28), ["git"]);
269 assert.deepEqual(await tick(at(7, 29), []), []);
270 assert.deepEqual(await tick(at(7, 38), []), []);
271 assert.deepEqual(await tick(at(7, 39), []), [id]);
272
273 const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map([["git", "Git and repositories"]])))!;
274 assert.equal(detail.visibility, "dismissed");
275 assert.equal(detail.status, "resolved");
276 assert.equal(detail.resolved_at, at(7, 29).toISOString(), "resolved when it recovered, not when it was dismissed");
277 assert.equal(detail.durations.to_resolve, 4 * 60);
278 assert.equal(
279 detail.timeline.at(-1)!.text,
280 "Recovered after 4 minutes, at 6 Oct 07:29 UTC, and stayed healthy for 10 minutes; dismissed automatically. It never appeared on the status page.",
281 );
282 assert.equal(detail.timeline.at(-1)!.by, "status");
283 assert.equal((await incidentDetail(db, picked, "https://status.g1t.sh", new Map()))!.visibility, "draft");
284 // Nothing is left to watch, and the bookkeeping is gone with it.
285 assert.deepEqual(await watchedDrafts(db), []);
286 assert.equal(await autoDismiss(db, id, at(7, 29).toISOString(), "again", at(7, 40)), false, "dismissing twice changes nothing");
287});