Sign in

flagon-io/g1t

public

g1t/apps/status/src/incidents.ts

410 lines18,344 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1/**
2 * Incidents and maintenance as staff run them: checking what sudo sends,
3 * and the lifecycle rules (which changes are allowed, which timestamps
4 * they set, and what each writes on the timeline). No Workers imports,
5 * so it is tested under Node.
6 */
7import {
8 INCIDENT_SEVERITIES,
9 type ComponentImpact,
10 type DeclareIncident,
11 type IncidentChange,
12 type IncidentDurations,
13 type IncidentSeverity,
14 type IncidentStatus,
15 type IncidentVisibility,
16 type ImpactInput,
17 type MaintenanceChange,
18 type NewMaintenance,
19 type PostmortemFields,
20 type PublishIncident,
21 type RolesChange,
22 type TimelineKind,
23} from "@g1t/contracts/status";
24
25import { IMPACT_WORD, INCIDENT_STATUS } from "./model.ts";
26
27export const STATUSES: IncidentStatus[] = ["investigating", "identified", "monitoring", "resolved"];
28export const IMPACTS: ComponentImpact[] = ["operational", "degraded", "partial_outage", "major_outage"];
29
30export const SEVERITIES = INCIDENT_SEVERITIES;
31
32export const SEVERITY_LABEL = Object.fromEntries(SEVERITIES.map((s) => [s.value, s.label])) as Record<IncidentSeverity, string>;
33
34/** Whether subscribers are emailed by default at a severity. */
35export function notifyByDefault(severity: IncidentSeverity): boolean {
36 return SEVERITIES.find((s) => s.value === severity)?.notify ?? false;
37}
38
39export const MAX_TITLE = 120;
40export const MAX_MESSAGE = 4000;
41export const MAX_SECTION = 20000;
42/** How long maintenance may be scheduled ahead, and how long a window may be. */
43export const MAX_AHEAD_DAYS = 180;
44export const MAX_WINDOW_HOURS = 72;
45
46export type Checked<T> = { ok: true; value: T } | { ok: false; error: string };
47
48const fail = (error: string): { ok: false; error: string } => ({ ok: false, error });
49
50function clean(text: unknown): string {
51 return typeof text === "string" ? text.replace(/\s+/g, " ").trim() : "";
52}
53
54/** A message keeps its paragraphs; runs of spaces within a line are folded. */
55export function cleanMessage(text: unknown): string {
56 if (typeof text !== "string") return "";
57 return text
58 .replace(/\r\n?/g, "\n")
59 .split("\n")
60 .map((line) => line.replace(/[ \t]+/g, " ").trimEnd())
61 .join("\n")
62 .replace(/\n{3,}/g, "\n\n")
63 .trim();
64}
65
66const EMAIL = /^[^\s@<>"]{1,64}@[^\s@<>"]{1,190}\.[a-z]{2,}$/i;
67
68/** A role's holder: a staff email, lowercased, or null for nobody. */
69function role(raw: unknown, name: string): Checked<string | null> {
70 const value = clean(raw).toLowerCase();
71 if (!value) return { ok: true, value: null };
72 if (!EMAIL.test(value)) return fail(`The ${name} is an email address.`);
73 return { ok: true, value };
74}
75
76function startTime(raw: unknown, now: Date): Checked<string | null> {
77 if (raw == null || raw === "") return { ok: true, value: null };
78 const at = Date.parse(String(raw));
79 if (Number.isNaN(at)) return fail("The start time is not a date.");
80 if (at > now.getTime() + 60_000) return fail("The start time is in the future.");
81 if (at < now.getTime() - 90 * 24 * 60 * 60 * 1000) return fail("The start time is over 90 days ago.");
82 return { ok: true, value: new Date(at).toISOString() };
83}
84
85function title(raw: unknown): Checked<string> {
86 const value = clean(raw);
87 if (!value) return fail("Give it a title.");
88 if (value.length > MAX_TITLE) return fail(`Keep the title under ${MAX_TITLE} characters.`);
89 return { ok: true, value };
90}
91
92function by(raw: unknown): Checked<string> {
93 const value = clean(raw);
94 return value ? { ok: true, value } : fail("Who is making the change is missing.");
95}
96
97/** Parts and their impact: known parts, each once, the last one given winning. */
98export function checkImpacts(raw: unknown, known: string[]): Checked<ImpactInput[]> {
99 if (!Array.isArray(raw)) return { ok: true, value: [] };
100 const byKey = new Map<string, ComponentImpact>();
101 for (const item of raw) {
102 const key = String((item as ImpactInput)?.key ?? "");
103 const impact = (item as ImpactInput)?.impact;
104 if (!known.includes(key)) return fail(`Unknown part: ${key || "(none)"}.`);
105 if (!IMPACTS.includes(impact)) return fail(`Choose an impact for ${key}.`);
106 byKey.set(key, impact);
107 }
108 return { ok: true, value: [...byKey].map(([key, impact]) => ({ key, impact })) };
109}
110
111export function checkDeclare(input: unknown, known: string[], now = new Date()): Checked<DeclareIncident & { started_at: string | null }> {
112 const raw = (input ?? {}) as Record<string, unknown>;
113 const t = title(raw.title);
114 if (!t.ok) return t;
115 const severity = raw.severity as IncidentSeverity;
116 if (!SEVERITY_LABEL[severity]) return fail("Choose a severity.");
117 const status = (raw.status ?? "investigating") as IncidentStatus;
118 if (!STATUSES.includes(status) || status === "resolved") return fail("A new incident is investigating, identified or monitoring.");
119 const impacts = checkImpacts(raw.components, known);
120 if (!impacts.ok) return impacts;
121 const affected = impacts.value.filter((c) => c.impact !== "operational");
122 if (affected.length === 0) return fail("Choose at least one part it affects, and how.");
123 const message = cleanMessage(raw.message);
124 if (!message) return fail("Write the first update: what people notice, in a sentence or two.");
125 if (message.length > MAX_MESSAGE) return fail(`Keep the update under ${MAX_MESSAGE} characters.`);
126 const commander = role(raw.commander, "incident commander");
127 if (!commander.ok) return commander;
128 const communications = role(raw.communications, "communications lead");
129 if (!communications.ok) return communications;
130 const who = by(raw.by);
131 if (!who.ok) return who;
132 const started = startTime(raw.started_at, now);
133 if (!started.ok) return started;
134 return {
135 ok: true,
136 value: {
137 title: t.value,
138 severity,
139 status,
140 components: affected,
141 message,
142 started_at: started.value,
143 commander: commander.value,
144 communications: communications.value,
145 notify: raw.notify === true,
146 by: who.value,
147 },
148 };
149}
150
151export function checkChange(input: unknown, known: string[]): Checked<IncidentChange> {
152 const raw = (input ?? {}) as Record<string, unknown>;
153 const message = cleanMessage(raw.message);
154 if (message.length > MAX_MESSAGE) return fail(`Keep the update under ${MAX_MESSAGE} characters.`);
155 const status = raw.status == null || raw.status === "" ? null : (raw.status as IncidentStatus);
156 if (status && !STATUSES.includes(status)) return fail("Choose a status.");
157 const severity = raw.severity == null || raw.severity === "" ? null : (raw.severity as IncidentSeverity);
158 if (severity && !SEVERITY_LABEL[severity]) return fail("Choose a severity.");
159 const impacts = raw.impacts == null ? { ok: true as const, value: null } : checkImpacts(raw.impacts, known);
160 if (!impacts.ok) return impacts;
161 const who = by(raw.by);
162 if (!who.ok) return who;
163 const isPublic = raw.public === true;
164 if (isPublic && !message) return fail("Write the update the status page will show.");
165 return {
166 ok: true,
167 value: { public: isPublic, message, status, severity, impacts: impacts.value, notify: isPublic && raw.notify === true, by: who.value },
168 };
169}
170
171export function checkRoles(input: unknown): Checked<RolesChange> {
172 const raw = (input ?? {}) as Record<string, unknown>;
173 const commander = role(raw.commander, "incident commander");
174 if (!commander.ok) return commander;
175 const communications = role(raw.communications, "communications lead");
176 if (!communications.ok) return communications;
177 const who = by(raw.by);
178 if (!who.ok) return who;
179 return { ok: true, value: { commander: commander.value, communications: communications.value, by: who.value } };
180}
181
182export function checkPublish(input: unknown): Checked<PublishIncident> {
183 const raw = (input ?? {}) as Record<string, unknown>;
184 let name: string | null = null;
185 if (raw.title != null && raw.title !== "") {
186 const t = title(raw.title);
187 if (!t.ok) return t;
188 name = t.value;
189 }
190 const message = cleanMessage(raw.message);
191 if (!message) return fail("Write the first update the status page will show.");
192 if (message.length > MAX_MESSAGE) return fail(`Keep the update under ${MAX_MESSAGE} characters.`);
193 const who = by(raw.by);
194 if (!who.ok) return who;
195 return { ok: true, value: { title: name, message, notify: raw.notify === true, by: who.value } };
196}
197
198export function checkFollowUp(input: unknown): Checked<{ title: string; owner: string | null; by: string }> {
199 const raw = (input ?? {}) as Record<string, unknown>;
200 const name = clean(raw.title);
201 if (!name) return fail("Say what needs doing.");
202 if (name.length > 200) return fail("Keep a follow-up under 200 characters.");
203 const owner = role(raw.owner, "owner");
204 if (!owner.ok) return owner;
205 const who = by(raw.by);
206 if (!who.ok) return who;
207 return { ok: true, value: { title: name, owner: owner.value, by: who.value } };
208}
209
210export const POSTMORTEM_FIELDS: (keyof PostmortemFields)[] = ["summary", "impact", "timeline", "root_cause", "went_well", "went_badly", "action_items"];
211
212export function checkPostmortem(input: unknown): Checked<PostmortemFields & { by: string }> {
213 const raw = (input ?? {}) as Record<string, unknown>;
214 const fields = {} as PostmortemFields;
215 for (const key of POSTMORTEM_FIELDS) {
216 const value = cleanMessage(raw[key]);
217 if (value.length > MAX_SECTION) return fail(`Keep each section under ${MAX_SECTION} characters.`);
218 fields[key] = value;
219 }
220 const who = by(raw.by);
221 if (!who.ok) return who;
222 return { ok: true, value: { ...fields, by: who.value } };
223}
224
225/** A postmortem can be published once it says what happened and why. */
226export function postmortemReady(p: PostmortemFields): string | null {
227 if (!p.summary) return "Write a summary before publishing.";
228 if (!p.root_cause) return "Say what caused it before publishing.";
229 return null;
230}
231
232export function checkMaintenance(input: unknown, known: string[], now = new Date()): Checked<NewMaintenance> {
233 const raw = (input ?? {}) as Record<string, unknown>;
234 const t = title(raw.title);
235 if (!t.ok) return t;
236 const message = cleanMessage(raw.message);
237 if (!message) return fail("Say what will happen and what people will notice.");
238 if (message.length > MAX_MESSAGE) return fail(`Keep the message under ${MAX_MESSAGE} characters.`);
239 const components = Array.isArray(raw.components) ? [...new Set(raw.components.map(String))] : [];
240 const unknown = components.filter((c) => !known.includes(c));
241 if (unknown.length) return fail(`Unknown parts: ${unknown.join(", ")}.`);
242 if (components.length === 0) return fail("Choose at least one part the work affects.");
243 const starts = Date.parse(String(raw.starts_at ?? ""));
244 const ends = Date.parse(String(raw.ends_at ?? ""));
245 if (Number.isNaN(starts) || Number.isNaN(ends)) return fail("Give the window's start and end.");
246 if (ends <= starts) return fail("The window ends after it starts.");
247 if (ends <= now.getTime()) return fail("The window is already over.");
248 if (starts > now.getTime() + MAX_AHEAD_DAYS * 86_400_000) return fail(`Schedule maintenance at most ${MAX_AHEAD_DAYS} days ahead.`);
249 if (ends - starts > MAX_WINDOW_HOURS * 3_600_000) return fail(`Keep a window under ${MAX_WINDOW_HOURS} hours.`);
250 const who = by(raw.by);
251 if (!who.ok) return who;
252 return {
253 ok: true,
254 value: {
255 title: t.value,
256 message,
257 components,
258 starts_at: new Date(starts).toISOString(),
259 ends_at: new Date(ends).toISOString(),
260 notify: raw.notify === true,
261 by: who.value,
262 },
263 };
264}
265
266export function checkMaintenanceChange(input: unknown): Checked<MaintenanceChange> {
267 const raw = (input ?? {}) as Record<string, unknown>;
268 const action = raw.action as MaintenanceChange["action"];
269 if (!["update", "start", "complete", "cancel"].includes(action)) return fail("Choose what to do.");
270 const message = cleanMessage(raw.message);
271 if (action === "update" && !message) return fail("Write the update.");
272 if (message.length > MAX_MESSAGE) return fail(`Keep the message under ${MAX_MESSAGE} characters.`);
273 const who = by(raw.by);
274 if (!who.ok) return who;
275 return { ok: true, value: { action, message, notify: raw.notify === true, by: who.value } };
276}
277
278// --- Lifecycle -------------------------------------------------------------------
279
280/** What the lifecycle reads and changes on an incident. */
281export type IncidentFacts = {
282 status: IncidentStatus;
283 severity: IncidentSeverity;
284 visibility: IncidentVisibility;
285 components: ImpactInput[];
286 started_at: string;
287 acknowledged_at: string | null;
288 mitigated_at: string | null;
289 resolved_at: string | null;
290 commander: string | null;
291 communications: string | null;
292};
293
294/** A line the change writes on the timeline. */
295export type Entry = { kind: TimelineKind; public: boolean; status: IncidentStatus | null; text: string };
296
297const ORDER: Record<IncidentStatus, number> = { investigating: 0, identified: 1, monitoring: 2, resolved: 3 };
298
299/**
300 * Applies an update, a note, or changes. The rules:
301 *
302 * - A dismissed incident is closed for good.
303 * - A draft takes notes and changes, but nothing public until it is published.
304 * - On a published incident, a status change is said publicly: the status page
305 * shows each status with words.
306 * - Reaching monitoring (or resolved) marks it mitigated; resolved marks it
307 * resolved; leaving resolved reopens it and clears that.
308 * - The first change by a person acknowledges a detected incident.
309 */
310export function applyChange(
311 facts: IncidentFacts,
312 change: IncidentChange,
313 now: Date,
314 names: Map<string, string> = new Map(),
315): Checked<{ next: IncidentFacts; entries: Entry[] }> {
316 if (facts.visibility === "dismissed") return fail("This draft was dismissed; declare a new incident instead.");
317 if (change.public && facts.visibility === "draft") return fail("Publish the draft before posting public updates.");
318 const at = now.toISOString();
319 const next: IncidentFacts = { ...facts, components: facts.components.map((c) => ({ ...c })) };
320 const entries: Entry[] = [];
321
322 const status = change.status && change.status !== facts.status ? change.status : null;
323 if (status && facts.visibility === "public" && !change.public) {
324 return fail("Changing the status of a published incident is a public update: say what changed for the status page.");
325 }
326 if (!status && !change.severity && !change.impacts?.length && !change.message) return fail("Nothing to post: write an update or change something.");
327
328 if (!next.acknowledged_at) {
329 next.acknowledged_at = at;
330 entries.push({ kind: "acknowledged", public: false, status: null, text: "Acknowledged." });
331 }
332 if (change.severity && change.severity !== facts.severity) {
333 entries.push({
334 kind: "severity",
335 public: false,
336 status: null,
337 text: `Severity ${SEVERITY_LABEL[facts.severity]} → ${SEVERITY_LABEL[change.severity]}.`,
338 });
339 next.severity = change.severity;
340 }
341 for (const { key, impact } of change.impacts ?? []) {
342 const index = next.components.findIndex((c) => c.key === key);
343 const before = index >= 0 ? next.components[index]!.impact : "operational";
344 if (before === impact) continue;
345 if (index >= 0) next.components[index]!.impact = impact;
346 else next.components.push({ key, impact });
347 entries.push({ kind: "impact", public: false, status: null, text: `${names.get(key) ?? key}: ${IMPACT_WORD[before]} → ${IMPACT_WORD[impact]}.` });
348 }
349 if (status) {
350 const reopening = facts.status === "resolved";
351 entries.push({
352 kind: "status",
353 public: false,
354 status,
355 text: reopening ? `Reopened: ${INCIDENT_STATUS[status]}.` : `${INCIDENT_STATUS[facts.status]} → ${INCIDENT_STATUS[status]}.`,
356 });
357 next.status = status;
358 if (ORDER[status] >= ORDER.monitoring && !next.mitigated_at) next.mitigated_at = at;
359 if (status === "resolved") next.resolved_at = at;
360 if (reopening) next.resolved_at = null;
361 }
362 if (change.message) {
363 entries.push({ kind: change.public ? "update" : "note", public: change.public, status: change.public ? next.status : null, text: change.message });
364 }
365 return { ok: true, value: { next, entries } };
366}
367
368/** A change of roles, and the lines it writes. */
369export function applyRoles(facts: IncidentFacts, change: RolesChange, now: Date): { next: IncidentFacts; entries: Entry[] } {
370 const next = { ...facts };
371 const entries: Entry[] = [];
372 if (!next.acknowledged_at) {
373 next.acknowledged_at = now.toISOString();
374 entries.push({ kind: "acknowledged", public: false, status: null, text: "Acknowledged." });
375 }
376 const say = (who: string | null) => who ?? "nobody";
377 if (change.commander !== facts.commander) {
378 entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${say(change.commander)}.` });
379 next.commander = change.commander;
380 }
381 if (change.communications !== facts.communications) {
382 entries.push({ kind: "role", public: false, status: null, text: `Communications: ${say(change.communications)}.` });
383 next.communications = change.communications;
384 }
385 return { next, entries };
386}
387
388/** Seconds from the impact starting to each milestone. */
389export function durations(i: Pick<IncidentFacts, "started_at" | "acknowledged_at" | "mitigated_at" | "resolved_at">): IncidentDurations {
390 const start = Date.parse(i.started_at);
391 const since = (at: string | null) => (at ? Math.max(0, Math.round((Date.parse(at) - start) / 1000)) : null);
392 return { to_acknowledge: since(i.acknowledged_at), to_mitigate: since(i.mitigated_at), to_resolve: since(i.resolved_at) };
393}
394
395/** "45s", "12m", "2h 05m", "3d 4h". */
396export function duration(seconds: number | null): string {
397 if (seconds == null) return "—";
398 if (seconds < 60) return `${seconds}s`;
399 const minutes = Math.floor(seconds / 60);
400 if (minutes < 60) return `${minutes}m`;
401 const hours = Math.floor(minutes / 60);
402 if (hours < 48) return `${hours}h ${String(minutes % 60).padStart(2, "0")}m`;
403 return `${Math.floor(hours / 24)}d ${hours % 24}h`;
404}
405
406/** A short random id for a permalink: ten lowercase letters and digits. */
407export function shortId(random: (n: number) => Uint8Array = (n) => crypto.getRandomValues(new Uint8Array(n))): string {
408 const alphabet = "abcdefghijkmnpqrstuvwxyz23456789";
409 return [...random(10)].map((b) => alphabet[b % alphabet.length]).join("");
410}