g1t/apps/status/src/incidents.test.ts

206 lines11,429 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import {
5 type IncidentFacts,
6 applyChange,
7 applyRoles,
8 checkChange,
9 checkDeclare,
10 checkMaintenance,
11 checkMaintenanceChange,
12 checkPostmortem,
13 duration,
14 durations,
15 notifyByDefault,
16 postmortemReady,
17 shortId,
18} from "./incidents.ts";
19import { postmortemDraft, timelineText } from "./postmortem.ts";
20
21const NOW = new Date("2026-10-05T12:00:00Z");
22const KNOWN = ["git", "api", "docs"];
23const base = {
24 title: " Pushes failing ",
25 severity: "sev2",
26 message: "We are looking into it.\r\n\r\n\r\nMore soon.",
27 components: [
28 { key: "git", impact: "major_outage" },
29 { key: "api", impact: "operational" },
30 ],
31 commander: " Ana@G1T.sh ",
32 notify: true,
33 by: "staff@g1t.sh",
34};
35
36test("declaring: cleaned and checked; operational parts are left out", () => {
37 const checked = checkDeclare(base, KNOWN, NOW);
38 assert.ok(checked.ok);
39 assert.equal(checked.value.title, "Pushes failing");
40 assert.equal(checked.value.status, "investigating");
41 assert.equal(checked.value.message, "We are looking into it.\n\nMore soon.");
42 assert.deepEqual(checked.value.components, [{ key: "git", impact: "major_outage" }]);
43 assert.equal(checked.value.commander, "ana@g1t.sh");
44 assert.equal(checked.value.started_at, null);
45 assert.equal(checked.value.notify, true);
46});
47
48test("declaring is refused without what it needs", () => {
49 assert.equal(checkDeclare({ ...base, title: "" }, KNOWN, NOW).ok, false);
50 assert.equal(checkDeclare({ ...base, severity: "sev9" }, KNOWN, NOW).ok, false);
51 assert.equal(checkDeclare({ ...base, status: "resolved" }, KNOWN, NOW).ok, false);
52 assert.equal(checkDeclare({ ...base, components: [{ key: "git", impact: "operational" }] }, KNOWN, NOW).ok, false);
53 assert.equal(checkDeclare({ ...base, components: [{ key: "nope", impact: "degraded" }] }, KNOWN, NOW).ok, false);
54 assert.equal(checkDeclare({ ...base, components: [{ key: "git", impact: "meh" }] }, KNOWN, NOW).ok, false);
55 assert.equal(checkDeclare({ ...base, commander: "not an email" }, KNOWN, NOW).ok, false);
56 assert.equal(checkDeclare({ ...base, message: " " }, KNOWN, NOW).ok, false);
57 assert.equal(checkDeclare({ ...base, started_at: "2026-10-06T00:00:00Z" }, KNOWN, NOW).ok, false);
58 const backdated = checkDeclare({ ...base, started_at: "2026-10-05T11:00:00Z" }, KNOWN, NOW);
59 assert.ok(backdated.ok && backdated.value.started_at === "2026-10-05T11:00:00.000Z");
60});
61
62test("severities: SEV1 and SEV2 email subscribers by default", () => {
63 assert.deepEqual(
64 (["sev1", "sev2", "sev3", "sev4"] as const).map(notifyByDefault),
65 [true, true, false, false],
66 );
67});
68
69test("an update needs words when it is public; a note can be just a change", () => {
70 assert.equal(checkChange({ public: true, message: " ", by: "x" }, KNOWN).ok, false);
71 const note = checkChange({ public: false, message: "", severity: "sev1", by: "x" }, KNOWN);
72 assert.ok(note.ok && note.value.severity === "sev1" && note.value.notify === false);
73 assert.equal(checkChange({ public: true, message: "ok", status: "panicking", by: "x" }, KNOWN).ok, false);
74 assert.equal(checkChange({ public: true, message: "ok", impacts: [{ key: "zzz", impact: "degraded" }], by: "x" }, KNOWN).ok, false);
75 const notify = checkChange({ public: false, message: "internal", notify: true, by: "x" }, KNOWN);
76 assert.ok(notify.ok && notify.value.notify === false, "a note is never emailed");
77});
78
79const open: IncidentFacts = {
80 status: "investigating",
81 severity: "sev2",
82 visibility: "public",
83 components: [{ key: "git", impact: "major_outage" }],
84 started_at: "2026-10-05T11:30:00.000Z",
85 acknowledged_at: "2026-10-05T11:35:00.000Z",
86 mitigated_at: null,
87 resolved_at: null,
88 commander: null,
89 communications: null,
90};
91const names = new Map([["git", "Git and repositories"]]);
92
93function change(input: Record<string, unknown>) {
94 const checked = checkChange({ by: "staff@g1t.sh", ...input }, KNOWN);
95 assert.ok(checked.ok, checked.ok ? "" : checked.error);
96 return checked.value;
97}
98
99test("lifecycle: identified, then monitoring marks it mitigated, then resolved", () => {
100 const a = applyChange(open, change({ public: true, status: "identified", message: "A bad deploy." }), NOW, names);
101 assert.ok(a.ok);
102 assert.equal(a.value.next.status, "identified");
103 assert.equal(a.value.next.mitigated_at, null);
104 assert.deepEqual(a.value.entries.map((e) => [e.kind, e.public]), [["status", false], ["update", true]]);
105 assert.equal(a.value.entries[1]!.status, "identified");
106 assert.equal(a.value.entries[0]!.text, "Investigating → Identified.");
107
108 const b = applyChange(a.value.next, change({ public: true, status: "monitoring", message: "Rolled back." }), new Date("2026-10-05T12:10:00Z"));
109 assert.ok(b.ok && b.value.next.mitigated_at === "2026-10-05T12:10:00.000Z");
110
111 const c = applyChange(b.value.next, change({ public: true, status: "resolved", message: "Fixed." }), new Date("2026-10-05T12:40:00Z"));
112 assert.ok(c.ok);
113 assert.equal(c.value.next.resolved_at, "2026-10-05T12:40:00.000Z");
114 assert.equal(c.value.next.mitigated_at, "2026-10-05T12:10:00.000Z", "mitigated keeps its first time");
115 assert.deepEqual(durations(c.value.next), { to_acknowledge: 300, to_mitigate: 2400, to_resolve: 4200 });
116});
117
118test("lifecycle: resolving straight away marks it mitigated too; reopening clears resolved", () => {
119 const done = applyChange(open, change({ public: true, status: "resolved", message: "It was a blip." }), NOW);
120 assert.ok(done.ok && done.value.next.mitigated_at === NOW.toISOString() && done.value.next.resolved_at === NOW.toISOString());
121 const again = applyChange(done.value.next, change({ public: true, status: "investigating", message: "It is back." }), NOW);
122 assert.ok(again.ok);
123 assert.equal(again.value.next.resolved_at, null);
124 assert.match(again.value.entries[0]!.text, /^Reopened/);
125});
126
127test("lifecycle: a published incident's status changes only with public words", () => {
128 const quiet = applyChange(open, change({ public: false, status: "identified", message: "Found it." }), NOW);
129 assert.equal(quiet.ok, false);
130 const note = applyChange(open, change({ public: false, message: "Paging the storage team." }), NOW);
131 assert.ok(note.ok && note.value.entries.length === 1 && note.value.entries[0]!.kind === "note" && !note.value.entries[0]!.public);
132 assert.equal(applyChange(open, change({ public: false, message: "" }), NOW).ok, false, "nothing to post");
133});
134
135test("lifecycle: severity and impact changes are written on the timeline", () => {
136 const r = applyChange(open, change({ public: false, message: "", severity: "sev1", impacts: [{ key: "git", impact: "partial_outage" }, { key: "api", impact: "degraded" }] }), NOW, names);
137 assert.ok(r.ok);
138 assert.equal(r.value.next.severity, "sev1");
139 assert.deepEqual(r.value.next.components, [{ key: "git", impact: "partial_outage" }, { key: "api", impact: "degraded" }]);
140 assert.deepEqual(r.value.entries.map((e) => e.text), ["Severity SEV2 → SEV1.", "Git and repositories: Major outage → Partial outage.", "api: Operational → Degraded performance."]);
141});
142
143test("lifecycle: drafts take notes but nothing public; dismissed takes nothing; the first touch acknowledges", () => {
144 const draft: IncidentFacts = { ...open, visibility: "draft", acknowledged_at: null };
145 assert.equal(applyChange(draft, change({ public: true, message: "Hi" }), NOW).ok, false);
146 const note = applyChange(draft, change({ public: false, message: "Looking.", status: "identified" }), NOW);
147 assert.ok(note.ok);
148 assert.equal(note.value.next.acknowledged_at, NOW.toISOString());
149 assert.equal(note.value.entries[0]!.kind, "acknowledged");
150 assert.equal(note.value.next.status, "identified", "a draft's status can move without words");
151 assert.equal(applyChange({ ...open, visibility: "dismissed" }, change({ public: false, message: "x" }), NOW).ok, false);
152});
153
154test("roles: each change is a line, and nobody is said plainly", () => {
155 const r = applyRoles({ ...open, commander: "ana@g1t.sh" }, { commander: null, communications: "bo@g1t.sh", by: "x" }, NOW);
156 assert.deepEqual(r.entries.map((e) => e.text), ["Incident commander: nobody.", "Communications: bo@g1t.sh."]);
157 assert.equal(applyRoles(open, { commander: null, communications: null, by: "x" }, NOW).entries.length, 0);
158});
159
160test("durations read plainly", () => {
161 assert.equal(duration(null), "—");
162 assert.equal(duration(42), "42s");
163 assert.equal(duration(720), "12m");
164 assert.equal(duration(7500), "2h 05m");
165 assert.equal(duration(3 * 86400 + 4 * 3600), "3d 4h");
166});
167
168test("maintenance: a window in the future, under 72 hours, on known parts", () => {
169 const m = { title: "Database upgrade", message: "Pushes pause for a few minutes.", components: ["git"], starts_at: "2026-10-06T02:00Z", ends_at: "2026-10-06T03:00Z", by: "x" };
170 const checked = checkMaintenance(m, KNOWN, NOW);
171 assert.ok(checked.ok && checked.value.ends_at === "2026-10-06T03:00:00.000Z");
172 assert.equal(checkMaintenance({ ...m, ends_at: "2026-10-06T01:00Z" }, KNOWN, NOW).ok, false);
173 assert.equal(checkMaintenance({ ...m, starts_at: "2026-10-04T01:00Z", ends_at: "2026-10-04T02:00Z" }, KNOWN, NOW).ok, false);
174 assert.equal(checkMaintenance({ ...m, ends_at: "2026-10-10T03:00Z" }, KNOWN, NOW).ok, false);
175 assert.equal(checkMaintenance({ ...m, components: [] }, KNOWN, NOW).ok, false);
176 assert.equal(checkMaintenanceChange({ action: "update", message: "", by: "x" }).ok, false);
177 assert.ok(checkMaintenanceChange({ action: "cancel", message: "", by: "x" }).ok);
178});
179
180test("postmortems: published only with a summary and a cause; the draft is filled from the incident", () => {
181 const pm = checkPostmortem({ summary: "A deploy broke pushes.", impact: "", timeline: "", root_cause: "", went_well: "", went_badly: "", action_items: "", by: "x" });
182 assert.ok(pm.ok);
183 assert.match(postmortemReady(pm.value) ?? "", /caused/);
184 assert.equal(postmortemReady({ ...pm.value, root_cause: "A missing index." }), null);
185
186 const timeline = [
187 { id: "1", at: "2026-10-05T11:35:00Z", by: "x", kind: "declared" as const, public: false, status: null, text: "Declared SEV2.", notified: null },
188 { id: "2", at: "2026-10-05T11:36:00Z", by: "x", kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged.", notified: null },
189 { id: "3", at: "2026-10-05T11:40:00Z", by: "x", kind: "update" as const, public: true, status: "investigating" as const, text: "Pushes fail.\nLooking.", notified: 3 },
190 ];
191 assert.equal(timelineText(timeline), "- 5 Oct 11:35 UTC: Declared SEV2.\n- 5 Oct 11:40 UTC: Status page: Pushes fail. Looking.");
192 const draft = postmortemDraft(
193 { started_at: "2026-10-05T11:30:00Z", resolved_at: "2026-10-05T12:40:00Z", components: [{ key: "git", impact: "major_outage" }], durations: { to_acknowledge: 300, to_mitigate: null, to_resolve: 4200 } },
194 timeline,
195 [{ id: "f", title: "Alert on push errors", owner: "ana@g1t.sh", done_at: null, created_at: "", created_by: "" }],
196 names,
197 );
198 assert.equal(draft.impact, "Git and repositories (major outage) affected for 1h 10m, from 5 Oct 11:30 UTC to 5 Oct 12:40 UTC.");
199 assert.equal(draft.action_items, "- Alert on push errors");
200});
201
202test("short ids are ten unambiguous characters", () => {
203 const id = shortId();
204 assert.match(id, /^[a-km-np-z2-9]{10}$/);
205 assert.notEqual(shortId(), id);
206});