Skip to content
503 linesCodeBlameRaw
1import { CheckCircle2, Circle, CircleDot, ExternalLink, FileText, Globe, Lock, Radar } from "lucide-react";
2import { Link, redirect } from "react-router";
3
4import type { AdminIncidentDetail, IncidentSeverity, IncidentStatus, TimelineEntry } from "@g1t/contracts";
5
6import type { Route } from "./+types/incident";
7import { BackLink, ImpactBadge, ImpactPicker, PhaseBadge, SeverityBadge, Timer } from "~/components/incidents";
8import { IncidentChecks } from "~/components/latency";
9import { Badge, Button, Field, Input, Notice, Section, Select, Textarea, When } from "~/components/ui";
10import { text } from "~/lib/forms";
11import {
12 INCIDENT_SEVERITIES,
13 INCIDENT_STATUSES,
14 duration,
15 nextStatus,
16 notifyByDefault,
17 readImpacts,
18 secondsBetween,
19 severityLabel,
20 staffSuggestions,
21 statusLabel,
22} from "~/lib/incidents";
23import { staffEmails, statusAdmin } from "~/lib/services.server";
24import { settle } from "~/lib/settle";
25import { requireStaff } from "~/lib/staff";
26
27export const meta: Route.MetaFunction = ({ loaderData }) => [
28 { title: `${loaderData?.incident?.title ?? "Incident"} · sudo` },
29 { name: "robots", content: "noindex, nofollow" },
30];
31
32/** What the page says after a change, by the code in `?done=`. */
33const DONE: Record<string, string> = {
34 declared: "Declared and published. status.g1t.sh shows it within 30 seconds.",
35 update: "Update posted. The status page shows it within 30 seconds.",
36 note: "Note added. Only staff see it.",
37 resolved: "Resolved. Write the postmortem while it is fresh.",
38 roles: "Roles saved.",
39 published: "Published to the status page.",
40 dismissed: "Dismissed. It never appeared on the status page.",
41 followup: "Follow-up saved.",
42};
43
44export async function loader({ params, request, context }: Route.LoaderArgs) {
45 const staff = requireStaff(context);
46 const [incident, components] = await Promise.all([settle(statusAdmin.incident(params.id)), settle(statusAdmin.components())]);
47 if (incident.ok && !incident.value) throw new Response("No such incident.", { status: 404 });
48 const url = new URL(request.url);
49 const done = url.searchParams.get("declared") ? "declared" : (url.searchParams.get("done") ?? "");
50 const detail = incident.ok ? incident.value : null;
51 return {
52 me: staff.email,
53 incident: detail,
54 components: components.ok ? components.value : [],
55 error: incident.ok ? null : incident.error,
56 done: DONE[done] ?? null,
57 staff: staffSuggestions(staffEmails(), [staff.email, detail?.commander, detail?.communications, ...(detail?.followups.map((f) => f.owner) ?? [])]),
58 now: new Date().toISOString(),
59 };
60}
61
62type ActionData = { error: string; intent: string };
63
64export async function action({ params, request, context }: Route.ActionArgs) {
65 const staff = requireStaff(context);
66 const form = await request.formData();
67 const intent = text(form, "intent");
68 const id = params.id ?? "";
69 const back = (done: string, anchor = "") => redirect(`/incidents/${id}?done=${done}${anchor}`);
70 const failed = (error: string) => ({ error, intent }) satisfies ActionData;
71 const unwrap = async <T,>(call: Promise<{ ok: true; value: T } | { ok: false; error: { message: string } }>) => {
72 const result = await settle(call);
73 if (!result.ok) return { ok: false as const, error: result.error };
74 return result.value.ok ? { ok: true as const, value: result.value.value } : { ok: false as const, error: result.value.error.message };
75 };
76
77 if (intent === "update" || intent === "resolve") {
78 const components = await settle(statusAdmin.components());
79 const keys = components.ok ? components.value.map((c) => c.key) : [];
80 const isPublic = intent === "resolve" || text(form, "visibility") === "public";
81 const status = intent === "resolve" ? "resolved" : (text(form, "status") as IncidentStatus) || null;
82 const r = await unwrap(
83 statusAdmin.update(id, {
84 public: isPublic,
85 message: String(form.get("message") ?? ""),
86 status,
87 severity: (text(form, "severity") as IncidentSeverity) || null,
88 impacts: intent === "resolve" ? [] : readImpacts(form, keys),
89 notify: isPublic && form.get("notify") === "on",
90 by: staff.email,
91 }),
92 );
93 if (!r.ok) return failed(r.error);
94 return back(status === "resolved" ? "resolved" : isPublic ? "update" : "note");
95 }
96 if (intent === "roles") {
97 const r = await unwrap(statusAdmin.roles(id, { commander: text(form, "commander") || null, communications: text(form, "communications") || null, by: staff.email }));
98 return !r.ok ? failed(r.error) : back("roles", "#roles");
99 }
100 if (intent === "publish") {
101 const r = await unwrap(
102 statusAdmin.publish(id, { title: text(form, "title") || null, message: String(form.get("message") ?? ""), notify: form.get("notify") === "on", by: staff.email }),
103 );
104 return !r.ok ? failed(r.error) : back("published");
105 }
106 if (intent === "dismiss") {
107 const r = await unwrap(statusAdmin.dismiss(id, { reason: text(form, "reason"), by: staff.email }));
108 return !r.ok ? failed(r.error) : back("dismissed");
109 }
110 if (intent === "followup_add") {
111 const r = await unwrap(statusAdmin.addFollowUp(id, { title: text(form, "title"), owner: text(form, "owner") || null, by: staff.email }));
112 return !r.ok ? failed(r.error) : back("followup", "#followups");
113 }
114 if (intent === "followup_set") {
115 const r = await unwrap(statusAdmin.setFollowUp(id, text(form, "followup"), { done: text(form, "done") === "1", by: staff.email }));
116 return !r.ok ? failed(r.error) : back("followup", "#followups");
117 }
118 return failed("Unknown action.");
119}
120
121const KIND_ICON = { update: Globe, note: Lock, detected: Radar, followup: CheckCircle2, postmortem: FileText } as const;
122
123function EntryLine({ entry }: { entry: TimelineEntry }) {
124 const who = entry.by === "status" ? "the checks" : entry.by;
125 const meta = (
126 <span className="text-xs text-faint">
127 <When at={entry.at} time /> · <span className="font-mono">{who}</span>
128 </span>
129 );
130 if (entry.kind === "update") {
131 return (
132 <li className="rounded-md border border-accent/30 bg-accent/6 px-3.5 py-3">
133 <div className="flex flex-wrap items-center gap-x-2 gap-y-1">
134 <Globe size={14} aria-hidden="true" className="text-accent" />
135 <span className="text-xs font-semibold text-accent">Public update</span>
136 {entry.status && <Badge tone="lavender">{statusLabel(entry.status)}</Badge>}
137 {meta}
138 {entry.notified != null && <span className="text-xs text-faint">· emailed {entry.notified}</span>}
139 </div>
140 <p className="mt-1.5 text-sm whitespace-pre-line text-fg">{entry.text}</p>
141 </li>
142 );
143 }
144 if (entry.kind === "note") {
145 return (
146 <li className="rounded-md border border-line bg-raised/40 px-3.5 py-3">
147 <div className="flex flex-wrap items-center gap-x-2 gap-y-1">
148 <Lock size={13} aria-hidden="true" className="text-faint" />
149 <span className="text-xs font-medium text-muted">Internal note</span>
150 {meta}
151 </div>
152 <p className="mt-1.5 text-sm whitespace-pre-line text-fg-soft">{entry.text}</p>
153 </li>
154 );
155 }
156 const Icon = KIND_ICON[entry.kind as keyof typeof KIND_ICON] ?? CircleDot;
157 const loud = entry.kind === "detected" || entry.kind === "failing";
158 return (
159 <li className="flex gap-2.5 px-1 py-1">
160 <Icon size={13} aria-hidden="true" className={`mt-1 shrink-0 ${loud ? "text-warn" : "text-faint"}`} />
161 <p className="min-w-0 text-sm text-muted">
162 <span className={loud ? "text-fg-soft" : ""}>{entry.text}</span> {meta}
163 </p>
164 </li>
165 );
166}
167
168function Header({ incident, now }: { incident: AdminIncidentDetail; now: string }) {
169 const open = incident.resolved_at == null;
170 const d = incident.durations;
171 return (
172 <>
173 <div className="mt-3 flex flex-wrap items-center gap-2">
174 <SeverityBadge severity={incident.severity} />
175 <PhaseBadge incident={incident} />
176 {incident.source === "detected" && <Badge tone="warn">Detected by the checks</Badge>}
177 {incident.visibility !== "public" && <Badge>Not on the status page</Badge>}
178 </div>
179 <h1 className="mt-2 text-2xl font-semibold tracking-tight break-words">{incident.title}</h1>
180 <p className="mt-1 text-sm text-muted">
181 Impact began <When at={incident.started_at} time /> · declared by <span className="font-mono text-xs">{incident.created_by === "status" ? "the checks" : incident.created_by}</span>
182 {incident.commander && (
183 <>
184 {" "}· IC <span className="font-mono text-xs">{incident.commander}</span>
185 </>
186 )}
187 {incident.communications && (
188 <>
189 {" "}· comms <span className="font-mono text-xs">{incident.communications}</span>
190 </>
191 )}
192 </p>
193 <div className="mt-4 grid grid-cols-2 divide-line rounded-lg border border-line bg-surface sm:grid-cols-4 sm:divide-x [&>*:nth-child(-n+2)]:border-b [&>*:nth-child(-n+2)]:border-line sm:[&>*:nth-child(-n+2)]:border-b-0">
194 <Timer
195 label={open ? "Open for" : "Lasted"}
196 value={duration(open ? secondsBetween(incident.started_at, now) : d.to_resolve)}
197 tone={open ? "warn" : "mint"}
198 hint={open ? `${severityLabel(incident.severity)} · ${statusLabel(incident.status)}` : <When at={incident.resolved_at} time />}
199 />
200 <Timer label="To acknowledge" value={duration(d.to_acknowledge)} hint={incident.acknowledged_at ? <When at={incident.acknowledged_at} time /> : "Not yet"} />
201 <Timer label="To mitigate" value={duration(d.to_mitigate)} hint={incident.mitigated_at ? <When at={incident.mitigated_at} time /> : "At monitoring"} />
202 <Timer label="To resolve" value={duration(d.to_resolve)} hint={incident.resolved_at ? <When at={incident.resolved_at} time /> : "Still open"} />
203 </div>
204 </>
205 );
206}
207
208function UpdateForm({ incident, components, error }: { incident: AdminIncidentDetail; components: { key: string; name: string }[]; error: string | null }) {
209 const draft = incident.visibility === "draft";
210 const current = new Map(incident.components.map((c) => [c.key, c.impact]));
211 return (
212 <Section id="update" title={draft ? "Add a note" : "Post an update"} description={draft ? "Drafts take notes and changes; publish it to post publicly." : "A public update goes on the status page; a note stays here."}>
213 <form method="post" action="#update" className="space-y-4">
214 <input type="hidden" name="intent" value="update" />
215 {error && <Notice tone="error">{error}</Notice>}
216 {!draft && (
217 <fieldset className="flex flex-wrap gap-2" aria-label="Who sees it">
218 <label className="inline-flex cursor-pointer items-center gap-2 rounded-md border border-line px-3 py-1.5 text-sm has-checked:border-accent/60 has-checked:bg-accent/10">
219 <input type="radio" name="visibility" value="public" defaultChecked className="accent-[var(--g1t-accent)]" />
220 <Globe size={14} aria-hidden="true" /> Public update
221 </label>
222 <label className="inline-flex cursor-pointer items-center gap-2 rounded-md border border-line px-3 py-1.5 text-sm has-checked:border-line-strong has-checked:bg-raised">
223 <input type="radio" name="visibility" value="internal" className="accent-[var(--g1t-accent)]" />
224 <Lock size={14} aria-hidden="true" /> Internal note
225 </label>
226 </fieldset>
227 )}
228 <div className="grid gap-3 sm:grid-cols-2">
229 <Field label="Status" hint={draft ? undefined : "Changing it needs a public update."}>
230 <Select name="status" defaultValue="">
231 <option value="">Keep: {statusLabel(incident.status)}</option>
232 {INCIDENT_STATUSES.filter((s) => s.value !== incident.status).map((s) => (
233 <option key={s.value} value={s.value}>
234 {s.label}
235 {s.value === nextStatus(incident.status) ? " (next)" : ""}
236 </option>
237 ))}
238 </Select>
239 </Field>
240 <Field label="Severity">
241 <Select name="severity" defaultValue="">
242 <option value="">Keep: {severityLabel(incident.severity)}</option>
243 {INCIDENT_SEVERITIES.filter((s) => s.value !== incident.severity).map((s) => (
244 <option key={s.value} value={s.value}>
245 {s.label}
246 </option>
247 ))}
248 </Select>
249 </Field>
250 </div>
251 <details className="group rounded-md">
252 <summary className="cursor-pointer text-sm text-muted select-none hover:text-fg">Change the parts' impact</summary>
253 <div className="mt-2">
254 <ImpactPicker components={components} current={current} blank />
255 </div>
256 </details>
257 <Field label="Message">
258 <Textarea name="message" rows={4} maxLength={4000} placeholder={draft ? "What you found, for the team." : "What changed, in a sentence or two, for the people affected."} />
259 </Field>
260 <div className="flex flex-wrap items-center justify-between gap-3">
261 {!draft ? (
262 <label className="flex items-center gap-2 text-sm">
263 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-[var(--g1t-accent)]" />
264 Email subscribers (public updates only)
265 </label>
266 ) : (
267 <span />
268 )}
269 <Button type="submit" variant="lavender">
270 Post
271 </Button>
272 </div>
273 </form>
274 </Section>
275 );
276}
277
278export default function Incident({ loaderData, actionData }: Route.ComponentProps) {
279 const { incident, components, error, done, staff, now } = loaderData;
280 const failed = actionData as ActionData | undefined;
281 const errorFor = (intent: string) => (failed?.intent === intent ? failed.error : null);
282 if (!incident) {
283 return (
284 <main className="mx-auto max-w-6xl px-4 py-8 sm:py-10">
285 <BackLink to="/incidents">Incidents</BackLink>
286 <div className="mt-4">
287 <Notice tone="warn">The status worker did not answer: {error}</Notice>
288 </div>
289 </main>
290 );
291 }
292 const names = new Map(components.map((c) => [c.key, c.name]));
293 const open = incident.resolved_at == null;
294 const isPublic = incident.visibility === "public";
295 const affected = incident.components.filter((c) => c.impact !== "operational");
296 const timeline = [...incident.timeline].reverse();
297
298 return (
299 <main className="mx-auto max-w-6xl px-4 py-8 sm:py-10">
300 <BackLink to={incident.visibility === "draft" ? "/incidents?tab=drafts" : open ? "/incidents" : "/incidents?tab=resolved"}>Incidents</BackLink>
301 <Header incident={incident} now={now} />
302
303 <div className="mt-4 space-y-3">
304 {done && <Notice tone="ok">{done}</Notice>}
305 {failed && !["update", "roles", "followup_add", "followup_set", "resolve", "publish", "dismiss"].includes(failed.intent) && <Notice tone="error">{failed.error}</Notice>}
306 </div>
307
308 {incident.visibility === "draft" && (
309 <section className="mt-5 rounded-lg border border-warn/35 bg-warn/6 p-4 sm:p-5">
310 <h2 className="font-semibold">This is a draft</h2>
311 <p className="mt-1 text-sm text-muted">
312 {incident.source === "detected"
313 ? "The checks failed or were slow on four of five checks in a row and made this. Nobody outside sees it until you publish it. If it was a blip, dismiss it; if no one picks it up and its parts stay healthy for 10 minutes, it is dismissed on its own. Left unacknowledged, the alert goes out again after 45 minutes, then every 6 hours."
314 : "Nobody outside sees it until you publish it."}
315 </p>
316 <div className="mt-4 grid gap-4 lg:grid-cols-[minmax(0,1fr)_18rem]">
317 <form method="post" className="space-y-3">
318 <input type="hidden" name="intent" value="publish" />
319 {errorFor("publish") && <Notice tone="error">{errorFor("publish")}</Notice>}
320 <Field label="Public title">
321 <Input name="title" defaultValue={incident.title.replace(/^Detected: /, "")} maxLength={120} required />
322 </Field>
323 <Field label="First public update">
324 <Textarea name="message" rows={3} required maxLength={4000} placeholder="We are looking into reports that…" />
325 </Field>
326 <div className="flex flex-wrap items-center justify-between gap-3">
327 <label className="flex items-center gap-2 text-sm">
328 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-[var(--g1t-accent)]" />
329 Email subscribers
330 </label>
331 <Button type="submit" variant="lavender">
332 Publish to the status page
333 </Button>
334 </div>
335 </form>
336 <form method="post" className="space-y-3 lg:border-l lg:border-warn/20 lg:pl-4">
337 <input type="hidden" name="intent" value="dismiss" />
338 {errorFor("dismiss") && <Notice tone="error">{errorFor("dismiss")}</Notice>}
339 <Field label="Not an incident?">
340 <Input name="reason" maxLength={500} placeholder="A blip: answered again within minutes" />
341 </Field>
342 <Button type="submit" variant="quiet">
343 Dismiss draft
344 </Button>
345 </form>
346 </div>
347 </section>
348 )}
349
350 <div className="mt-6 grid gap-6 lg:grid-cols-[minmax(0,1fr)_20rem]">
351 <div className="min-w-0 space-y-6">
352 {incident.visibility !== "dismissed" && <UpdateForm incident={incident} components={components} error={errorFor("update")} />}
353 {(incident.checks?.length ?? 0) > 0 && (
354 <Section
355 id="checks"
356 title="Checks"
357 description="How long each check of its parts took, from 30 minutes before it began to 30 minutes after it ended (at most a day), with the data centre each ran from. Kept for 7 days."
358 >
359 <IncidentChecks checks={incident.checks} names={names} />
360 </Section>
361 )}
362 <Section title="Timeline" description="Newest first. Public updates are what the status page shows; everything else is for staff.">
363 <ol className="space-y-2">
364 {timeline.map((entry) => (
365 <EntryLine key={entry.id} entry={entry} />
366 ))}
367 </ol>
368 </Section>
369 </div>
370
371 <aside className="min-w-0 space-y-6">
372 {open && isPublic && (
373 <Section id="resolve" title="Resolve" description="Closes it, posts a last public update, and stops colouring the parts.">
374 <form method="post" action="#resolve" className="space-y-3">
375 <input type="hidden" name="intent" value="resolve" />
376 {errorFor("resolve") && <Notice tone="error">{errorFor("resolve")}</Notice>}
377 <Textarea name="message" rows={3} required maxLength={4000} aria-label="Resolution update" placeholder="Pushes work again. A bad deploy was rolled back; nothing was lost." />
378 <label className="flex items-center gap-2 text-sm">
379 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-[var(--g1t-accent)]" />
380 Email subscribers
381 </label>
382 <Button type="submit" variant="lavender" className="w-full">
383 <CheckCircle2 size={15} aria-hidden="true" /> Resolve incident
384 </Button>
385 </form>
386 </Section>
387 )}
388
389 {!open && isPublic && (
390 <Section title="Postmortem" description={incident.postmortem_published_at ? "Published on the incident's page." : incident.postmortem ? "Drafted, not published." : "Not started."}>
391 <div className="flex flex-wrap items-center gap-2">
392 <Link to={`/incidents/${incident.id}/postmortem`} className="inline-flex items-center gap-1.5 text-sm font-medium text-accent hover:underline">
393 <FileText size={14} aria-hidden="true" />
394 {incident.postmortem ? "Edit the postmortem" : "Write the postmortem"}
395 </Link>
396 {incident.postmortem_published_at && <Badge tone="mint">Published</Badge>}
397 </div>
398 </Section>
399 )}
400
401 <Section title="Parts" description={open ? "What the status page shows for each while it is open." : "What it did to each."}>
402 {affected.length === 0 ? (
403 <p className="text-sm text-muted">None marked.</p>
404 ) : (
405 <ul className="space-y-2">
406 {affected.map((c) => (
407 <li key={c.key} className="flex items-center justify-between gap-2 text-sm">
408 <span className="min-w-0 truncate">{names.get(c.key) ?? c.key}</span>
409 <ImpactBadge impact={c.impact} />
410 </li>
411 ))}
412 </ul>
413 )}
414 </Section>
415
416 <Section id="roles" title="Roles">
417 <form method="post" action="#roles" className="space-y-3">
418 <input type="hidden" name="intent" value="roles" />
419 {errorFor("roles") && <Notice tone="error">{errorFor("roles")}</Notice>}
420 <Field label="Incident commander">
421 <Input name="commander" type="email" list="staff" defaultValue={incident.commander ?? ""} placeholder="Nobody" />
422 </Field>
423 <Field label="Communications">
424 <Input name="communications" type="email" list="staff" defaultValue={incident.communications ?? ""} placeholder="Nobody" />
425 </Field>
426 <Button type="submit" variant="quiet">
427 Save roles
428 </Button>
429 </form>
430 </Section>
431
432 <Section id="followups" title="Follow-ups" description="What to change so it does not happen again.">
433 {errorFor("followup_set") && (
434 <div className="mb-3">
435 <Notice tone="error">{errorFor("followup_set")}</Notice>
436 </div>
437 )}
438 {incident.followups.length > 0 && (
439 <ul className="mb-4 space-y-1.5">
440 {incident.followups.map((f) => (
441 <li key={f.id}>
442 <form method="post" action="#followups" className="flex items-start gap-2">
443 <input type="hidden" name="intent" value="followup_set" />
444 <input type="hidden" name="followup" value={f.id} />
445 <input type="hidden" name="done" value={f.done_at ? "0" : "1"} />
446 <button
447 type="submit"
448 aria-label={f.done_at ? `Mark “${f.title}” not done` : `Mark “${f.title}” done`}
449 className="mt-0.5 shrink-0 rounded text-faint hover:text-accent"
450 >
451 {f.done_at ? <CheckCircle2 size={16} className="text-success" /> : <Circle size={16} />}
452 </button>
453 <span className="min-w-0 text-sm">
454 <span className={f.done_at ? "text-faint line-through" : ""}>{f.title}</span>
455 {f.owner && <span className="block font-mono text-xs text-faint">{f.owner}</span>}
456 </span>
457 </form>
458 </li>
459 ))}
460 </ul>
461 )}
462 <form method="post" action="#followups" className="space-y-2">
463 <input type="hidden" name="intent" value="followup_add" />
464 {errorFor("followup_add") && <Notice tone="error">{errorFor("followup_add")}</Notice>}
465 <Input name="title" required maxLength={200} placeholder="Alert on push errors above 1%" aria-label="Follow-up" />
466 <div className="flex gap-2">
467 <Input name="owner" type="email" list="staff" placeholder="Owner (optional)" aria-label="Owner" />
468 <Button type="submit" variant="quiet">
469 Add
470 </Button>
471 </div>
472 </form>
473 </Section>
474
475 <Section title="Links">
476 <ul className="space-y-1.5 text-sm">
477 {isPublic ? (
478 <li>
479 <a href={incident.url} className="inline-flex items-center gap-1.5 text-accent hover:underline">
480 Status page permalink <ExternalLink size={13} aria-hidden="true" />
481 </a>
482 <span className="block truncate font-mono text-xs text-faint">{incident.url}</span>
483 </li>
484 ) : (
485 <li className="text-muted">No public page until it is published.</li>
486 )}
487 <li>
488 <Link to="/audit" className="text-muted hover:text-fg hover:underline">
489 Audit log
490 </Link>
491 </li>
492 </ul>
493 </Section>
494 <datalist id="staff">
495 {staff.map((email) => (
496 <option key={email} value={email} />
497 ))}
498 </datalist>
499 </aside>
500 </div>
501 </main>
502 );
503}