g1t/apps/status/src/incidents.test.ts
| 1 | import assert from "node:assert/strict"; |
| 2 | import { test } from "node:test"; |
| 3 | |
| 4 | import { |
| 5 | type IncidentFacts, |
| 6 | applyChange, |
| 7 | applyRoles, |
| 8 | checkChange, |
| 9 | checkDeclare, |
| 10 | checkMaintenance, |
| 11 | checkMaintenanceChange, |
| 12 | checkPostmortem, |
| 13 | duration, |
| 14 | durations, |
| 15 | notifyByDefault, |
| 16 | postmortemReady, |
| 17 | shortId, |
| 18 | } from "./incidents.ts"; |
| 19 | import { postmortemDraft, timelineText } from "./postmortem.ts"; |
| 20 | |
| 21 | const NOW = new Date("2026-10-05T12:00:00Z"); |
| 22 | const KNOWN = ["git", "api", "docs"]; |
| 23 | const base = { |
| 24 | title: " Pushes failing ", |
| 25 | severity: "sev2", |
| 26 | message: "We are looking into it.\r\n\r\n\r\nMore soon.", |
| 27 | components: [ |
| 28 | { key: "git", impact: "major_outage" }, |
| 29 | { key: "api", impact: "operational" }, |
| 30 | ], |
| 31 | commander: " Ana@G1T.sh ", |
| 32 | notify: true, |
| 33 | by: "staff@g1t.sh", |
| 34 | }; |
| 35 | |
| 36 | test("declaring: cleaned and checked; operational parts are left out", () => { |
| 37 | const checked = checkDeclare(base, KNOWN, NOW); |
| 38 | assert.ok(checked.ok); |
| 39 | assert.equal(checked.value.title, "Pushes failing"); |
| 40 | assert.equal(checked.value.status, "investigating"); |
| 41 | assert.equal(checked.value.message, "We are looking into it.\n\nMore soon."); |
| 42 | assert.deepEqual(checked.value.components, [{ key: "git", impact: "major_outage" }]); |
| 43 | assert.equal(checked.value.commander, "ana@g1t.sh"); |
| 44 | assert.equal(checked.value.started_at, null); |
| 45 | assert.equal(checked.value.notify, true); |
| 46 | }); |
| 47 | |
| 48 | test("declaring is refused without what it needs", () => { |
| 49 | assert.equal(checkDeclare({ ...base, title: "" }, KNOWN, NOW).ok, false); |
| 50 | assert.equal(checkDeclare({ ...base, severity: "sev9" }, KNOWN, NOW).ok, false); |
| 51 | assert.equal(checkDeclare({ ...base, status: "resolved" }, KNOWN, NOW).ok, false); |
| 52 | assert.equal(checkDeclare({ ...base, components: [{ key: "git", impact: "operational" }] }, KNOWN, NOW).ok, false); |
| 53 | assert.equal(checkDeclare({ ...base, components: [{ key: "nope", impact: "degraded" }] }, KNOWN, NOW).ok, false); |
| 54 | assert.equal(checkDeclare({ ...base, components: [{ key: "git", impact: "meh" }] }, KNOWN, NOW).ok, false); |
| 55 | assert.equal(checkDeclare({ ...base, commander: "not an email" }, KNOWN, NOW).ok, false); |
| 56 | assert.equal(checkDeclare({ ...base, message: " " }, KNOWN, NOW).ok, false); |
| 57 | assert.equal(checkDeclare({ ...base, started_at: "2026-10-06T00:00:00Z" }, KNOWN, NOW).ok, false); |
| 58 | const backdated = checkDeclare({ ...base, started_at: "2026-10-05T11:00:00Z" }, KNOWN, NOW); |
| 59 | assert.ok(backdated.ok && backdated.value.started_at === "2026-10-05T11:00:00.000Z"); |
| 60 | }); |
| 61 | |
| 62 | test("severities: SEV1 and SEV2 email subscribers by default", () => { |
| 63 | assert.deepEqual( |
| 64 | (["sev1", "sev2", "sev3", "sev4"] as const).map(notifyByDefault), |
| 65 | [true, true, false, false], |
| 66 | ); |
| 67 | }); |
| 68 | |
| 69 | test("an update needs words when it is public; a note can be just a change", () => { |
| 70 | assert.equal(checkChange({ public: true, message: " ", by: "x" }, KNOWN).ok, false); |
| 71 | const note = checkChange({ public: false, message: "", severity: "sev1", by: "x" }, KNOWN); |
| 72 | assert.ok(note.ok && note.value.severity === "sev1" && note.value.notify === false); |
| 73 | assert.equal(checkChange({ public: true, message: "ok", status: "panicking", by: "x" }, KNOWN).ok, false); |
| 74 | assert.equal(checkChange({ public: true, message: "ok", impacts: [{ key: "zzz", impact: "degraded" }], by: "x" }, KNOWN).ok, false); |
| 75 | const notify = checkChange({ public: false, message: "internal", notify: true, by: "x" }, KNOWN); |
| 76 | assert.ok(notify.ok && notify.value.notify === false, "a note is never emailed"); |
| 77 | }); |
| 78 | |
| 79 | const open: IncidentFacts = { |
| 80 | status: "investigating", |
| 81 | severity: "sev2", |
| 82 | visibility: "public", |
| 83 | components: [{ key: "git", impact: "major_outage" }], |
| 84 | started_at: "2026-10-05T11:30:00.000Z", |
| 85 | acknowledged_at: "2026-10-05T11:35:00.000Z", |
| 86 | mitigated_at: null, |
| 87 | resolved_at: null, |
| 88 | commander: null, |
| 89 | communications: null, |
| 90 | }; |
| 91 | const names = new Map([["git", "Git and repositories"]]); |
| 92 | |
| 93 | function change(input: Record<string, unknown>) { |
| 94 | const checked = checkChange({ by: "staff@g1t.sh", ...input }, KNOWN); |
| 95 | assert.ok(checked.ok, checked.ok ? "" : checked.error); |
| 96 | return checked.value; |
| 97 | } |
| 98 | |
| 99 | test("lifecycle: identified, then monitoring marks it mitigated, then resolved", () => { |
| 100 | const a = applyChange(open, change({ public: true, status: "identified", message: "A bad deploy." }), NOW, names); |
| 101 | assert.ok(a.ok); |
| 102 | assert.equal(a.value.next.status, "identified"); |
| 103 | assert.equal(a.value.next.mitigated_at, null); |
| 104 | assert.deepEqual(a.value.entries.map((e) => [e.kind, e.public]), [["status", false], ["update", true]]); |
| 105 | assert.equal(a.value.entries[1]!.status, "identified"); |
| 106 | assert.equal(a.value.entries[0]!.text, "Investigating → Identified."); |
| 107 | |
| 108 | const b = applyChange(a.value.next, change({ public: true, status: "monitoring", message: "Rolled back." }), new Date("2026-10-05T12:10:00Z")); |
| 109 | assert.ok(b.ok && b.value.next.mitigated_at === "2026-10-05T12:10:00.000Z"); |
| 110 | |
| 111 | const c = applyChange(b.value.next, change({ public: true, status: "resolved", message: "Fixed." }), new Date("2026-10-05T12:40:00Z")); |
| 112 | assert.ok(c.ok); |
| 113 | assert.equal(c.value.next.resolved_at, "2026-10-05T12:40:00.000Z"); |
| 114 | assert.equal(c.value.next.mitigated_at, "2026-10-05T12:10:00.000Z", "mitigated keeps its first time"); |
| 115 | assert.deepEqual(durations(c.value.next), { to_acknowledge: 300, to_mitigate: 2400, to_resolve: 4200 }); |
| 116 | }); |
| 117 | |
| 118 | test("lifecycle: resolving straight away marks it mitigated too; reopening clears resolved", () => { |
| 119 | const done = applyChange(open, change({ public: true, status: "resolved", message: "It was a blip." }), NOW); |
| 120 | assert.ok(done.ok && done.value.next.mitigated_at === NOW.toISOString() && done.value.next.resolved_at === NOW.toISOString()); |
| 121 | const again = applyChange(done.value.next, change({ public: true, status: "investigating", message: "It is back." }), NOW); |
| 122 | assert.ok(again.ok); |
| 123 | assert.equal(again.value.next.resolved_at, null); |
| 124 | assert.match(again.value.entries[0]!.text, /^Reopened/); |
| 125 | }); |
| 126 | |
| 127 | test("lifecycle: a published incident's status changes only with public words", () => { |
| 128 | const quiet = applyChange(open, change({ public: false, status: "identified", message: "Found it." }), NOW); |
| 129 | assert.equal(quiet.ok, false); |
| 130 | const note = applyChange(open, change({ public: false, message: "Paging the storage team." }), NOW); |
| 131 | assert.ok(note.ok && note.value.entries.length === 1 && note.value.entries[0]!.kind === "note" && !note.value.entries[0]!.public); |
| 132 | assert.equal(applyChange(open, change({ public: false, message: "" }), NOW).ok, false, "nothing to post"); |
| 133 | }); |
| 134 | |
| 135 | test("lifecycle: severity and impact changes are written on the timeline", () => { |
| 136 | const r = applyChange(open, change({ public: false, message: "", severity: "sev1", impacts: [{ key: "git", impact: "partial_outage" }, { key: "api", impact: "degraded" }] }), NOW, names); |
| 137 | assert.ok(r.ok); |
| 138 | assert.equal(r.value.next.severity, "sev1"); |
| 139 | assert.deepEqual(r.value.next.components, [{ key: "git", impact: "partial_outage" }, { key: "api", impact: "degraded" }]); |
| 140 | assert.deepEqual(r.value.entries.map((e) => e.text), ["Severity SEV2 → SEV1.", "Git and repositories: Major outage → Partial outage.", "api: Operational → Degraded performance."]); |
| 141 | }); |
| 142 | |
| 143 | test("lifecycle: drafts take notes but nothing public; dismissed takes nothing; the first touch acknowledges", () => { |
| 144 | const draft: IncidentFacts = { ...open, visibility: "draft", acknowledged_at: null }; |
| 145 | assert.equal(applyChange(draft, change({ public: true, message: "Hi" }), NOW).ok, false); |
| 146 | const note = applyChange(draft, change({ public: false, message: "Looking.", status: "identified" }), NOW); |
| 147 | assert.ok(note.ok); |
| 148 | assert.equal(note.value.next.acknowledged_at, NOW.toISOString()); |
| 149 | assert.equal(note.value.entries[0]!.kind, "acknowledged"); |
| 150 | assert.equal(note.value.next.status, "identified", "a draft's status can move without words"); |
| 151 | assert.equal(applyChange({ ...open, visibility: "dismissed" }, change({ public: false, message: "x" }), NOW).ok, false); |
| 152 | }); |
| 153 | |
| 154 | test("roles: each change is a line, and nobody is said plainly", () => { |
| 155 | const r = applyRoles({ ...open, commander: "ana@g1t.sh" }, { commander: null, communications: "bo@g1t.sh", by: "x" }, NOW); |
| 156 | assert.deepEqual(r.entries.map((e) => e.text), ["Incident commander: nobody.", "Communications: bo@g1t.sh."]); |
| 157 | assert.equal(applyRoles(open, { commander: null, communications: null, by: "x" }, NOW).entries.length, 0); |
| 158 | }); |
| 159 | |
| 160 | test("durations read plainly", () => { |
| 161 | assert.equal(duration(null), "—"); |
| 162 | assert.equal(duration(42), "42s"); |
| 163 | assert.equal(duration(720), "12m"); |
| 164 | assert.equal(duration(7500), "2h 05m"); |
| 165 | assert.equal(duration(3 * 86400 + 4 * 3600), "3d 4h"); |
| 166 | }); |
| 167 | |
| 168 | test("maintenance: a window in the future, under 72 hours, on known parts", () => { |
| 169 | const m = { title: "Database upgrade", message: "Pushes pause for a few minutes.", components: ["git"], starts_at: "2026-10-06T02:00Z", ends_at: "2026-10-06T03:00Z", by: "x" }; |
| 170 | const checked = checkMaintenance(m, KNOWN, NOW); |
| 171 | assert.ok(checked.ok && checked.value.ends_at === "2026-10-06T03:00:00.000Z"); |
| 172 | assert.equal(checkMaintenance({ ...m, ends_at: "2026-10-06T01:00Z" }, KNOWN, NOW).ok, false); |
| 173 | assert.equal(checkMaintenance({ ...m, starts_at: "2026-10-04T01:00Z", ends_at: "2026-10-04T02:00Z" }, KNOWN, NOW).ok, false); |
| 174 | assert.equal(checkMaintenance({ ...m, ends_at: "2026-10-10T03:00Z" }, KNOWN, NOW).ok, false); |
| 175 | assert.equal(checkMaintenance({ ...m, components: [] }, KNOWN, NOW).ok, false); |
| 176 | assert.equal(checkMaintenanceChange({ action: "update", message: "", by: "x" }).ok, false); |
| 177 | assert.ok(checkMaintenanceChange({ action: "cancel", message: "", by: "x" }).ok); |
| 178 | }); |
| 179 | |
| 180 | test("postmortems: published only with a summary and a cause; the draft is filled from the incident", () => { |
| 181 | const pm = checkPostmortem({ summary: "A deploy broke pushes.", impact: "", timeline: "", root_cause: "", went_well: "", went_badly: "", action_items: "", by: "x" }); |
| 182 | assert.ok(pm.ok); |
| 183 | assert.match(postmortemReady(pm.value) ?? "", /caused/); |
| 184 | assert.equal(postmortemReady({ ...pm.value, root_cause: "A missing index." }), null); |
| 185 | |
| 186 | const timeline = [ |
| 187 | { id: "1", at: "2026-10-05T11:35:00Z", by: "x", kind: "declared" as const, public: false, status: null, text: "Declared SEV2.", notified: null }, |
| 188 | { id: "2", at: "2026-10-05T11:36:00Z", by: "x", kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged.", notified: null }, |
| 189 | { id: "3", at: "2026-10-05T11:40:00Z", by: "x", kind: "update" as const, public: true, status: "investigating" as const, text: "Pushes fail.\nLooking.", notified: 3 }, |
| 190 | ]; |
| 191 | assert.equal(timelineText(timeline), "- 5 Oct 11:35 UTC: Declared SEV2.\n- 5 Oct 11:40 UTC: Status page: Pushes fail. Looking."); |
| 192 | const draft = postmortemDraft( |
| 193 | { started_at: "2026-10-05T11:30:00Z", resolved_at: "2026-10-05T12:40:00Z", components: [{ key: "git", impact: "major_outage" }], durations: { to_acknowledge: 300, to_mitigate: null, to_resolve: 4200 } }, |
| 194 | timeline, |
| 195 | [{ id: "f", title: "Alert on push errors", owner: "ana@g1t.sh", done_at: null, created_at: "", created_by: "" }], |
| 196 | names, |
| 197 | ); |
| 198 | assert.equal(draft.impact, "Git and repositories (major outage) affected for 1h 10m, from 5 Oct 11:30 UTC to 5 Oct 12:40 UTC."); |
| 199 | assert.equal(draft.action_items, "- Alert on push errors"); |
| 200 | }); |
| 201 | |
| 202 | test("short ids are ten unambiguous characters", () => { |
| 203 | const id = shortId(); |
| 204 | assert.match(id, /^[a-km-np-z2-9]{10}$/); |
| 205 | assert.notEqual(shortId(), id); |
| 206 | }); |