| 1 | /** |
| 2 | * Detection through the database, as the cron runs it: streaks saved and |
| 3 | * read back each round, deploy windows, and recovered drafts dismissed. |
| 4 | * D1 here is SQLite (node:sqlite) behind the few D1 calls the store makes, |
| 5 | * with the real migrations applied. |
| 6 | */ |
| 7 | import assert from "node:assert/strict"; |
| 8 | import { readFileSync, readdirSync } from "node:fs"; |
| 9 | import { DatabaseSync } from "node:sqlite"; |
| 10 | import { test } from "node:test"; |
| 11 | |
| 12 | import { type Streak, autoDismissText, deployChange, detect, settleDrafts, staleDrafts, staleText } from "./detect.ts"; |
| 13 | import { stamp } from "./postmortem.ts"; |
| 14 | import { |
| 15 | CHECK_HISTORY_DAYS, |
| 16 | autoDismiss, |
| 17 | checkHistory, |
| 18 | createIncident, |
| 19 | historySpan, |
| 20 | incidentDetail, |
| 21 | loadDeploy, |
| 22 | loadStreaks, |
| 23 | openRefs, |
| 24 | record, |
| 25 | saveDeploy, |
| 26 | saveHealthy, |
| 27 | saveReminders, |
| 28 | saveStreaks, |
| 29 | watchedDrafts, |
| 30 | } from "./store.ts"; |
| 31 | |
| 32 | type Params = (string | number | null)[]; |
| 33 | |
| 34 | /** Just enough of D1 over SQLite for the store. */ |
| 35 | function d1(): D1Database { |
| 36 | const db = new DatabaseSync(":memory:"); |
| 37 | const dir = new URL("../migrations/", import.meta.url); |
| 38 | for (const file of readdirSync(dir) |
| 39 | .filter((f) => f.endsWith(".sql")) |
| 40 | .sort()) |
| 41 | db.exec(readFileSync(new URL(file, dir), "utf8")); |
| 42 | class Statement { |
| 43 | constructor( |
| 44 | readonly sql: string, |
| 45 | readonly params: Params = [], |
| 46 | ) {} |
| 47 | bind(...params: Params) { |
| 48 | return new Statement(this.sql, params); |
| 49 | } |
| 50 | async all() { |
| 51 | return { results: db.prepare(this.sql).all(...this.params), success: true, meta: {} }; |
| 52 | } |
| 53 | async first() { |
| 54 | return db.prepare(this.sql).get(...this.params) ?? null; |
| 55 | } |
| 56 | async run() { |
| 57 | const r = db.prepare(this.sql).run(...this.params); |
| 58 | return { results: [], success: true, meta: { changes: Number(r.changes) } }; |
| 59 | } |
| 60 | } |
| 61 | return { |
| 62 | prepare: (sql: string) => new Statement(sql), |
| 63 | batch: async (list: Statement[]) => { |
| 64 | db.exec("BEGIN"); |
| 65 | try { |
| 66 | const out = []; |
| 67 | for (const s of list) out.push(/^\s*SELECT/i.test(s.sql) ? await s.all() : await s.run()); |
| 68 | db.exec("COMMIT"); |
| 69 | return out; |
| 70 | } catch (error) { |
| 71 | db.exec("ROLLBACK"); |
| 72 | throw error; |
| 73 | } |
| 74 | }, |
| 75 | } as unknown as D1Database; |
| 76 | } |
| 77 | |
| 78 | const at = (h: number, m: number) => new Date(Date.UTC(2026, 9, 6, h, m, 32)); |
| 79 | |
| 80 | /** One cron round's detection against the database; returns what it found. */ |
| 81 | async function round(db: D1Database, now: Date, git: "up" | "degraded" | "down", quiet = false) { |
| 82 | const found = detect( |
| 83 | await loadStreaks(db), |
| 84 | [ |
| 85 | { component: "git", state: git }, |
| 86 | { component: "api", state: "up" }, |
| 87 | ], |
| 88 | await openRefs(db), |
| 89 | new Set(), |
| 90 | now, |
| 91 | { quiet }, |
| 92 | ); |
| 93 | await saveStreaks(db, found.streaks); |
| 94 | return found; |
| 95 | } |
| 96 | |
| 97 | test("the 6 Oct draft: a slow run at 03:13 that recovered does not leak into the one at 07:25", async () => { |
| 98 | const db = d1(); |
| 99 | // 03:13 and 03:14 slow, then fine: never four of five. |
| 100 | assert.equal((await round(db, at(3, 13), "degraded")).draft.length, 0); |
| 101 | assert.equal((await round(db, at(3, 14), "degraded")).draft.length, 0); |
| 102 | await round(db, at(3, 15), "up"); |
| 103 | assert.equal((await loadStreaks(db)).size, 1, "one good check does not end a run"); |
| 104 | await round(db, at(3, 16), "up"); |
| 105 | await round(db, at(3, 17), "up"); |
| 106 | assert.equal((await loadStreaks(db)).size, 0, "three good checks in a row, with nothing else failing, leave no run behind"); |
| 107 | // 07:25 slow: the first check of a new run, not the third of the old one. |
| 108 | const first = await round(db, at(7, 25), "degraded"); |
| 109 | assert.equal(first.draft.length, 0); |
| 110 | assert.deepEqual( |
| 111 | [...(await loadStreaks(db)).values()], |
| 112 | [{ component: "git", state: "degraded", count: 1, checks: 1, recent: "s", since: at(7, 25).toISOString(), alerted: false }], |
| 113 | ); |
| 114 | await round(db, at(7, 26), "degraded"); |
| 115 | await round(db, at(7, 27), "degraded"); |
| 116 | const fourth = await round(db, at(7, 28), "degraded"); |
| 117 | assert.deepEqual(fourth.draft, [{ key: "git", state: "degraded", since: at(7, 25).toISOString(), checks: 4, of: 4 }]); |
| 118 | }); |
| 119 | |
| 120 | test("a run kept before N of M (no checks, no recent) is read as all bad, and still ends", async () => { |
| 121 | const db = d1(); |
| 122 | await db.prepare(`INSERT INTO streak (component, state, count, since, alerted) VALUES ('git', 'degraded', 180, ?1, 1)`).bind(at(4, 0).toISOString()).run(); |
| 123 | assert.deepEqual((await loadStreaks(db)).get("git"), { |
| 124 | component: "git", |
| 125 | state: "degraded", |
| 126 | count: 180, |
| 127 | checks: 180, |
| 128 | recent: "sssss", |
| 129 | since: at(4, 0).toISOString(), |
| 130 | alerted: true, |
| 131 | }); |
| 132 | await round(db, at(7, 0), "up"); |
| 133 | await round(db, at(7, 1), "up"); |
| 134 | assert.equal((await loadStreaks(db)).size, 1); |
| 135 | await round(db, at(7, 2), "up"); |
| 136 | assert.equal((await loadStreaks(db)).size, 0); |
| 137 | }); |
| 138 | |
| 139 | test("a run is kept while it lasts, and only the parts still failing keep one", async () => { |
| 140 | const db = d1(); |
| 141 | const streak = (component: string): Streak => ({ |
| 142 | component, |
| 143 | state: "down", |
| 144 | count: 2, |
| 145 | checks: 3, |
| 146 | recent: "x.x", |
| 147 | since: at(1, 0).toISOString(), |
| 148 | alerted: false, |
| 149 | }); |
| 150 | await saveStreaks(db, [streak("git"), streak("api")]); |
| 151 | await saveStreaks(db, [streak("api")]); |
| 152 | assert.deepEqual([...(await loadStreaks(db)).keys()], ["api"]); |
| 153 | assert.deepEqual((await loadStreaks(db)).get("api"), streak("api")); |
| 154 | await saveStreaks(db, []); |
| 155 | assert.equal((await loadStreaks(db)).size, 0); |
| 156 | }); |
| 157 | |
| 158 | test("the deploy window is kept between rounds", async () => { |
| 159 | const db = d1(); |
| 160 | assert.equal(await loadDeploy(db), null); |
| 161 | await saveDeploy(db, deployChange(null, "started", "run-1", at(7, 20))); |
| 162 | await saveDeploy(db, deployChange(await loadDeploy(db), "finished", null, at(7, 24))); |
| 163 | assert.deepEqual(await loadDeploy(db), { |
| 164 | id: "run-1", |
| 165 | started_at: at(7, 20).toISOString(), |
| 166 | finished_at: at(7, 24).toISOString(), |
| 167 | running: 0, |
| 168 | last_started_at: at(7, 20).toISOString(), |
| 169 | }); |
| 170 | // A window kept before deploys were counted reads as one deploy. |
| 171 | await db |
| 172 | .prepare(`UPDATE meta SET value = ?1 WHERE key = 'deploy'`) |
| 173 | .bind(JSON.stringify({ id: "old", started_at: at(8, 0).toISOString(), finished_at: null })) |
| 174 | .run(); |
| 175 | assert.deepEqual(await loadDeploy(db), { |
| 176 | id: "old", |
| 177 | started_at: at(8, 0).toISOString(), |
| 178 | finished_at: null, |
| 179 | running: 1, |
| 180 | last_started_at: at(8, 0).toISOString(), |
| 181 | }); |
| 182 | }); |
| 183 | |
| 184 | test("every check is kept for 7 days, with where it ran from, and an incident's page reads its parts' checks", async () => { |
| 185 | const db = d1(); |
| 186 | const minute = (m: number) => new Date(at(7, 0).getTime() + m * 60_000); |
| 187 | for (let m = 0; m < 5; m++) { |
| 188 | await record( |
| 189 | db, |
| 190 | [ |
| 191 | { component: "speed", state: m < 3 ? "degraded" : "up", detail: "", latency_ms: m < 3 ? 1900 : 300, colo: "IAD", first_ms: m < 3 ? 2400 : null }, |
| 192 | { component: "api", state: "up", detail: "", latency_ms: 90, colo: "IAD" }, |
| 193 | { component: "sandboxes", state: "unmonitored", detail: "", latency_ms: null }, |
| 194 | ], |
| 195 | minute(m), |
| 196 | ); |
| 197 | } |
| 198 | const kept = await checkHistory(db, ["speed"], minute(0), minute(4)); |
| 199 | assert.equal(kept.length, 5); |
| 200 | assert.deepEqual(kept[0], { component: "speed", at: minute(0).toISOString(), ms: 1900, outcome: "degraded", colo: "IAD", first_ms: 2400 }); |
| 201 | assert.deepEqual(kept[4], { component: "speed", at: minute(4).toISOString(), ms: 300, outcome: "up", colo: "IAD", first_ms: null }); |
| 202 | assert.equal((await checkHistory(db, ["sandboxes"], minute(0), minute(4))).length, 0, "a part with no check keeps no history"); |
| 203 | // A round 7 days later prunes everything older. |
| 204 | await record( |
| 205 | db, |
| 206 | [{ component: "api", state: "up", detail: "", latency_ms: 80, colo: "SJC" }], |
| 207 | new Date(minute(2).getTime() + CHECK_HISTORY_DAYS * 86_400_000), |
| 208 | ); |
| 209 | const left = await checkHistory(db, ["speed", "api"], minute(0), new Date(minute(0).getTime() + 8 * 86_400_000)); |
| 210 | assert.deepEqual( |
| 211 | left.map((c) => `${c.component}@${c.colo}`), |
| 212 | ["api@IAD", "speed@IAD", "api@IAD", "speed@IAD", "api@IAD", "speed@IAD", "api@SJC"], |
| 213 | "the two oldest rounds are gone", |
| 214 | ); |
| 215 | |
| 216 | // The incident page: its parts' checks around it, with the slow line. |
| 217 | const id = await createIncident( |
| 218 | db, |
| 219 | { |
| 220 | title: "Detected: Page speed slow", |
| 221 | severity: "sev3", |
| 222 | status: "investigating", |
| 223 | visibility: "draft", |
| 224 | source: "detected", |
| 225 | components: [{ key: "speed", impact: "degraded" }], |
| 226 | started_at: minute(0).toISOString(), |
| 227 | acknowledged_at: null, |
| 228 | commander: null, |
| 229 | communications: null, |
| 230 | by: "status", |
| 231 | }, |
| 232 | [{ kind: "detected", public: false, status: null, text: "Detected." }], |
| 233 | minute(4), |
| 234 | null, |
| 235 | ); |
| 236 | const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map(), { limits: new Map([["speed", 800]]), now: minute(5) }))!; |
| 237 | assert.equal(detail.checks.length, 1); |
| 238 | assert.deepEqual([detail.checks[0]!.key, detail.checks[0]!.slow_ms, detail.checks[0]!.samples.length], ["speed", 800, 3]); |
| 239 | assert.deepEqual(historySpan(minute(0).toISOString(), null, minute(5)), { from: minute(-30), to: minute(5) }); |
| 240 | assert.deepEqual(historySpan(minute(0).toISOString(), minute(10).toISOString(), minute(500)), { from: minute(-30), to: minute(40) }); |
| 241 | // A long one shows its last day. |
| 242 | assert.deepEqual(historySpan(minute(0).toISOString(), minute(3000).toISOString(), minute(4000)), { from: minute(3030 - 1440), to: minute(3030) }); |
| 243 | }); |
| 244 | |
| 245 | test("a draft left waiting is raised again, with a line on its timeline, and forgotten once it is not waiting", async () => { |
| 246 | const db = d1(); |
| 247 | const id = await createIncident( |
| 248 | db, |
| 249 | { |
| 250 | title: "Detected: Page speed slow", |
| 251 | severity: "sev3", |
| 252 | status: "investigating", |
| 253 | visibility: "draft", |
| 254 | source: "detected", |
| 255 | components: [{ key: "speed", impact: "degraded" }], |
| 256 | started_at: at(7, 0).toISOString(), |
| 257 | acknowledged_at: null, |
| 258 | commander: null, |
| 259 | communications: null, |
| 260 | by: "status", |
| 261 | }, |
| 262 | [{ kind: "detected", public: false, status: null, text: "Detected." }], |
| 263 | at(7, 4), |
| 264 | null, |
| 265 | ); |
| 266 | const tick = async (now: Date) => { |
| 267 | const waiting = await watchedDrafts(db); |
| 268 | const due = staleDrafts(waiting, now); |
| 269 | await saveReminders( |
| 270 | db, |
| 271 | waiting.map((d) => d.id), |
| 272 | due.map((d) => ({ id: d.id, text: staleText(d.waiting_ms, true) })), |
| 273 | now, |
| 274 | ); |
| 275 | return due.map((d) => d.id); |
| 276 | }; |
| 277 | assert.deepEqual(await tick(at(7, 48)), []); |
| 278 | assert.deepEqual(await tick(at(7, 49)), [id], "45 minutes after the draft was made"); |
| 279 | assert.equal((await watchedDrafts(db))[0]!.reminded_at, at(7, 49).toISOString()); |
| 280 | assert.deepEqual(await tick(at(7, 50)), [], "once"); |
| 281 | assert.deepEqual(await tick(at(13, 48)), []); |
| 282 | assert.deepEqual(await tick(at(13, 49)), [id], "then every 6 hours"); |
| 283 | const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map()))!; |
| 284 | assert.deepEqual( |
| 285 | detail.timeline.filter((e) => e.kind === "note").map((e) => [e.by, e.text]), |
| 286 | [ |
| 287 | ["status", "Unacknowledged for 45 minutes. The alert address was emailed again."], |
| 288 | ["status", "Unacknowledged for 6h 45m. The alert address was emailed again."], |
| 289 | ], |
| 290 | ); |
| 291 | assert.equal(detail.acknowledged_at, null, "a reminder is not an acknowledgement"); |
| 292 | // Picked up: no longer watched, and its bookkeeping goes. |
| 293 | await db.prepare(`UPDATE incident SET acknowledged_at = ?1 WHERE id = ?2`).bind(at(14, 0).toISOString(), id).run(); |
| 294 | assert.deepEqual(await tick(at(20, 0)), []); |
| 295 | assert.equal((await db.prepare(`SELECT COUNT(*) AS n FROM meta WHERE key LIKE 'draft_reminded:%'`).first<{ n: number }>())!.n, 0); |
| 296 | }); |
| 297 | |
| 298 | test("a detected draft that recovered is dismissed after ten healthy minutes; one staff picked up is left alone", async () => { |
| 299 | const db = d1(); |
| 300 | const draft = (title: string, acknowledged: string | null) => |
| 301 | createIncident( |
| 302 | db, |
| 303 | { |
| 304 | title, |
| 305 | severity: "sev3", |
| 306 | status: "investigating", |
| 307 | visibility: "draft", |
| 308 | source: "detected", |
| 309 | components: [{ key: "git", impact: "degraded" }], |
| 310 | started_at: at(7, 25).toISOString(), |
| 311 | acknowledged_at: acknowledged, |
| 312 | commander: null, |
| 313 | communications: null, |
| 314 | by: "status", |
| 315 | }, |
| 316 | [{ kind: "detected", public: false, status: null, text: "Detected." }], |
| 317 | at(7, 27), |
| 318 | null, |
| 319 | ); |
| 320 | const id = await draft("Detected: Git and repositories slow", null); |
| 321 | const picked = await draft("Detected: Git and repositories slow (picked up)", at(7, 28).toISOString()); |
| 322 | |
| 323 | const tick = async (now: Date, troubled: string[]) => { |
| 324 | const settled = settleDrafts(await watchedDrafts(db), new Set(troubled), now); |
| 325 | await saveHealthy(db, settled.healthy); |
| 326 | const dismissed: string[] = []; |
| 327 | for (const d of settled.dismiss) |
| 328 | if (await autoDismiss(db, d.id, d.recovered_at, autoDismissText(d.lasted_ms, stamp(d.recovered_at)), now)) dismissed.push(d.id); |
| 329 | return dismissed; |
| 330 | }; |
| 331 | assert.deepEqual( |
| 332 | (await watchedDrafts(db)).map((d) => d.id), |
| 333 | [id], |
| 334 | "only untouched drafts are watched", |
| 335 | ); |
| 336 | await tick(at(7, 28), ["git"]); |
| 337 | assert.deepEqual(await tick(at(7, 29), []), []); |
| 338 | assert.deepEqual(await tick(at(7, 38), []), []); |
| 339 | assert.deepEqual(await tick(at(7, 39), []), [id]); |
| 340 | |
| 341 | const detail = (await incidentDetail(db, id, "https://status.g1t.sh", new Map([["git", "Git and repositories"]])))!; |
| 342 | assert.equal(detail.visibility, "dismissed"); |
| 343 | assert.equal(detail.status, "resolved"); |
| 344 | assert.equal(detail.resolved_at, at(7, 29).toISOString(), "resolved when it recovered, not when it was dismissed"); |
| 345 | assert.equal(detail.durations.to_resolve, 4 * 60); |
| 346 | assert.equal( |
| 347 | detail.timeline.at(-1)!.text, |
| 348 | "Recovered after 4 minutes, at 6 Oct 07:29 UTC, and stayed healthy for 10 minutes; dismissed automatically. It never appeared on the status page.", |
| 349 | ); |
| 350 | assert.equal(detail.timeline.at(-1)!.by, "status"); |
| 351 | assert.equal((await incidentDetail(db, picked, "https://status.g1t.sh", new Map()))!.visibility, "draft"); |
| 352 | // Nothing is left to watch, and the bookkeeping is gone with it. |
| 353 | assert.deepEqual(await watchedDrafts(db), []); |
| 354 | assert.equal(await autoDismiss(db, id, at(7, 29).toISOString(), "again", at(7, 40)), false, "dismissing twice changes nothing"); |
| 355 | }); |