Skip to content

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

540 linesCodeBlameRaw
1import { CheckCircle2, Circle, CircleDot, ExternalLink, FileText, Globe, Lock, Radar } from "lucide-react";
2import { Link, redirect } from "react-router";
3
4import type { AdminIncidentDetail, IncidentSeverity, IncidentStatus, TimelineEntry } from "@g1t/contracts";
5
6import type { Route } from "./+types/incident";
7import { BackLink, ImpactBadge, ImpactPicker, PhaseBadge, SeverityBadge, Timer } from "~/components/incidents";
8import { IncidentChecks } from "~/components/latency";
9import { Badge, Button, Field, Input, Notice, Section, Select, Textarea, When } from "~/components/ui";
10import { text } from "~/lib/forms";
11import {
12 INCIDENT_SEVERITIES,
13 INCIDENT_STATUSES,
14 duration,
15 nextStatus,
16 notifyByDefault,
17 readImpacts,
18 secondsBetween,
19 severityLabel,
20 staffSuggestions,
21 statusLabel,
22} from "~/lib/incidents";
23import { staffEmails, statusAdmin } from "~/lib/services.server";
24import { settle } from "~/lib/settle";
25import { requireStaff } from "~/lib/staff";
26
27export const meta: Route.MetaFunction = ({ loaderData }) => [
28 { title: `${loaderData?.incident?.title ?? "Incident"} · sudo` },
29 { name: "robots", content: "noindex, nofollow" },
30];
31
32/** What the page says after a change, by the code in `?done=`. */
33const DONE: Record<string, string> = {
34 declared: "Declared and published. status.g1t.sh shows it within 30 seconds.",
35 update: "Update posted. The status page shows it within 30 seconds.",
36 note: "Note added. Only staff see it.",
37 resolved: "Resolved. Write the postmortem while it is fresh.",
38 roles: "Roles saved.",
39 published: "Published to the status page.",
40 dismissed: "Dismissed. It never appeared on the status page.",
41 followup: "Follow-up saved.",
42};
43
44export async function loader({ params, request, context }: Route.LoaderArgs) {
45 const staff = requireStaff(context);
46 const [incident, components] = await Promise.all([settle(statusAdmin.incident(params.id)), settle(statusAdmin.components())]);
47 if (incident.ok && !incident.value) throw new Response("No such incident.", { status: 404 });
48 const url = new URL(request.url);
49 const done = url.searchParams.get("declared") ? "declared" : (url.searchParams.get("done") ?? "");
50 const detail = incident.ok ? incident.value : null;
51 return {
52 me: staff.email,
53 incident: detail,
54 components: components.ok ? components.value : [],
55 error: incident.ok ? null : incident.error,
56 done: DONE[done] ?? null,
57 staff: staffSuggestions(staffEmails(), [staff.email, detail?.commander, detail?.communications, ...(detail?.followups.map((f) => f.owner) ?? [])]),
58 now: new Date().toISOString(),
59 };
60}
61
62type ActionData = { error: string; intent: string };
63
64export async function action({ params, request, context }: Route.ActionArgs) {
65 const staff = requireStaff(context);
66 const form = await request.formData();
67 const intent = text(form, "intent");
68 const id = params.id ?? "";
69 const back = (done: string, anchor = "") => redirect(`/incidents/${id}?done=${done}${anchor}`);
70 const failed = (error: string) => ({ error, intent }) satisfies ActionData;
71 const unwrap = async <T,>(call: Promise<{ ok: true; value: T } | { ok: false; error: { message: string } }>) => {
72 const result = await settle(call);
73 if (!result.ok) return { ok: false as const, error: result.error };
74 return result.value.ok ? { ok: true as const, value: result.value.value } : { ok: false as const, error: result.value.error.message };
75 };
76
77 if (intent === "update" || intent === "resolve") {
78 const components = await settle(statusAdmin.components());
79 const keys = components.ok ? components.value.map((c) => c.key) : [];
80 const isPublic = intent === "resolve" || text(form, "visibility") === "public";
81 const status = intent === "resolve" ? "resolved" : (text(form, "status") as IncidentStatus) || null;
82 const r = await unwrap(
83 statusAdmin.update(id, {
84 public: isPublic,
85 message: String(form.get("message") ?? ""),
86 status,
87 severity: (text(form, "severity") as IncidentSeverity) || null,
88 impacts: intent === "resolve" ? [] : readImpacts(form, keys),
89 notify: isPublic && form.get("notify") === "on",
90 by: staff.email,
91 }),
92 );
93 if (!r.ok) return failed(r.error);
94 return back(status === "resolved" ? "resolved" : isPublic ? "update" : "note");
95 }
96 if (intent === "roles") {
97 const r = await unwrap(
98 statusAdmin.roles(id, { commander: text(form, "commander") || null, communications: text(form, "communications") || null, by: staff.email }),
99 );
100 return !r.ok ? failed(r.error) : back("roles", "#roles");
101 }
102 if (intent === "publish") {
103 const r = await unwrap(
104 statusAdmin.publish(id, {
105 title: text(form, "title") || null,
106 message: String(form.get("message") ?? ""),
107 notify: form.get("notify") === "on",
108 by: staff.email,
109 }),
110 );
111 return !r.ok ? failed(r.error) : back("published");
112 }
113 if (intent === "dismiss") {
114 const r = await unwrap(statusAdmin.dismiss(id, { reason: text(form, "reason"), by: staff.email }));
115 return !r.ok ? failed(r.error) : back("dismissed");
116 }
117 if (intent === "followup_add") {
118 const r = await unwrap(statusAdmin.addFollowUp(id, { title: text(form, "title"), owner: text(form, "owner") || null, by: staff.email }));
119 return !r.ok ? failed(r.error) : back("followup", "#followups");
120 }
121 if (intent === "followup_set") {
122 const r = await unwrap(statusAdmin.setFollowUp(id, text(form, "followup"), { done: text(form, "done") === "1", by: staff.email }));
123 return !r.ok ? failed(r.error) : back("followup", "#followups");
124 }
125 return failed("Unknown action.");
126}
127
128const KIND_ICON = { update: Globe, note: Lock, detected: Radar, followup: CheckCircle2, postmortem: FileText } as const;
129
130function EntryLine({ entry }: { entry: TimelineEntry }) {
131 const who = entry.by === "status" ? "the checks" : entry.by;
132 const meta = (
133 <span className="text-xs text-faint">
134 <When at={entry.at} time /> · <span className="font-mono">{who}</span>
135 </span>
136 );
137 if (entry.kind === "update") {
138 return (
139 <li className="rounded-md border border-accent/30 bg-accent/6 px-3.5 py-3">
140 <div className="flex flex-wrap items-center gap-x-2 gap-y-1">
141 <Globe size={14} aria-hidden="true" className="text-accent" />
142 <span className="text-xs font-semibold text-accent">Public update</span>
143 {entry.status && <Badge tone="lavender">{statusLabel(entry.status)}</Badge>}
144 {meta}
145 {entry.notified != null && <span className="text-xs text-faint">· emailed {entry.notified}</span>}
146 </div>
147 <p className="mt-1.5 text-sm whitespace-pre-line text-fg">{entry.text}</p>
148 </li>
149 );
150 }
151 if (entry.kind === "note") {
152 return (
153 <li className="rounded-md border border-line bg-raised/40 px-3.5 py-3">
154 <div className="flex flex-wrap items-center gap-x-2 gap-y-1">
155 <Lock size={13} aria-hidden="true" className="text-faint" />
156 <span className="text-xs font-medium text-muted">Internal note</span>
157 {meta}
158 </div>
159 <p className="mt-1.5 text-sm whitespace-pre-line text-fg-soft">{entry.text}</p>
160 </li>
161 );
162 }
163 const Icon = KIND_ICON[entry.kind as keyof typeof KIND_ICON] ?? CircleDot;
164 const loud = entry.kind === "detected" || entry.kind === "failing";
165 return (
166 <li className="flex gap-2.5 px-1 py-1">
167 <Icon size={13} aria-hidden="true" className={`mt-1 shrink-0 ${loud ? "text-warn" : "text-faint"}`} />
168 <p className="min-w-0 text-sm text-muted">
169 <span className={loud ? "text-fg-soft" : ""}>{entry.text}</span> {meta}
170 </p>
171 </li>
172 );
173}
174
175function Header({ incident, now }: { incident: AdminIncidentDetail; now: string }) {
176 const open = incident.resolved_at == null;
177 const d = incident.durations;
178 return (
179 <>
180 <div className="mt-3 flex flex-wrap items-center gap-2">
181 <SeverityBadge severity={incident.severity} />
182 <PhaseBadge incident={incident} />
183 {incident.source === "detected" && <Badge tone="warn">Detected by the checks</Badge>}
184 {incident.visibility !== "public" && <Badge>Not on the status page</Badge>}
185 </div>
186 <h1 className="mt-2 text-2xl font-semibold tracking-tight wrap-break-word">{incident.title}</h1>
187 <p className="mt-1 text-sm text-muted">
188 Impact began <When at={incident.started_at} time /> · declared by{" "}
189 <span className="font-mono text-xs">{incident.created_by === "status" ? "the checks" : incident.created_by}</span>
190 {incident.commander && (
191 <>
192 {" "}
193 · IC <span className="font-mono text-xs">{incident.commander}</span>
194 </>
195 )}
196 {incident.communications && (
197 <>
198 {" "}
199 · comms <span className="font-mono text-xs">{incident.communications}</span>
200 </>
201 )}
202 </p>
203 <div className="mt-4 grid grid-cols-2 divide-line rounded-lg border border-line bg-surface sm:grid-cols-4 sm:divide-x [&>*:nth-child(-n+2)]:border-b [&>*:nth-child(-n+2)]:border-line sm:[&>*:nth-child(-n+2)]:border-b-0">
204 <Timer
205 label={open ? "Open for" : "Lasted"}
206 value={duration(open ? secondsBetween(incident.started_at, now) : d.to_resolve)}
207 tone={open ? "warn" : "mint"}
208 hint={open ? `${severityLabel(incident.severity)} · ${statusLabel(incident.status)}` : <When at={incident.resolved_at} time />}
209 />
210 <Timer
211 label="To acknowledge"
212 value={duration(d.to_acknowledge)}
213 hint={incident.acknowledged_at ? <When at={incident.acknowledged_at} time /> : "Not yet"}
214 />
215 <Timer label="To mitigate" value={duration(d.to_mitigate)} hint={incident.mitigated_at ? <When at={incident.mitigated_at} time /> : "At monitoring"} />
216 <Timer label="To resolve" value={duration(d.to_resolve)} hint={incident.resolved_at ? <When at={incident.resolved_at} time /> : "Still open"} />
217 </div>
218 </>
219 );
220}
221
222function UpdateForm({ incident, components, error }: { incident: AdminIncidentDetail; components: { key: string; name: string }[]; error: string | null }) {
223 const draft = incident.visibility === "draft";
224 const current = new Map(incident.components.map((c) => [c.key, c.impact]));
225 return (
226 <Section
227 id="update"
228 title={draft ? "Add a note" : "Post an update"}
229 description={draft ? "Drafts take notes and changes; publish it to post publicly." : "A public update goes on the status page; a note stays here."}
230 >
231 <form method="post" action="#update" className="space-y-4">
232 <input type="hidden" name="intent" value="update" />
233 {error && <Notice tone="error">{error}</Notice>}
234 {!draft && (
235 <fieldset className="flex flex-wrap gap-2" aria-label="Who sees it">
236 <label className="inline-flex cursor-pointer items-center gap-2 rounded-md border border-line px-3 py-1.5 text-sm has-checked:border-accent/60 has-checked:bg-accent/10">
237 <input type="radio" name="visibility" value="public" defaultChecked className="accent-(--g1t-accent)" />
238 <Globe size={14} aria-hidden="true" /> Public update
239 </label>
240 <label className="inline-flex cursor-pointer items-center gap-2 rounded-md border border-line px-3 py-1.5 text-sm has-checked:border-line-strong has-checked:bg-raised">
241 <input type="radio" name="visibility" value="internal" className="accent-(--g1t-accent)" />
242 <Lock size={14} aria-hidden="true" /> Internal note
243 </label>
244 </fieldset>
245 )}
246 <div className="grid gap-3 sm:grid-cols-2">
247 <Field label="Status" hint={draft ? undefined : "Changing it needs a public update."}>
248 <Select name="status" defaultValue="">
249 <option value="">Keep: {statusLabel(incident.status)}</option>
250 {INCIDENT_STATUSES.filter((s) => s.value !== incident.status).map((s) => (
251 <option key={s.value} value={s.value}>
252 {s.label}
253 {s.value === nextStatus(incident.status) ? " (next)" : ""}
254 </option>
255 ))}
256 </Select>
257 </Field>
258 <Field label="Severity">
259 <Select name="severity" defaultValue="">
260 <option value="">Keep: {severityLabel(incident.severity)}</option>
261 {INCIDENT_SEVERITIES.filter((s) => s.value !== incident.severity).map((s) => (
262 <option key={s.value} value={s.value}>
263 {s.label}
264 </option>
265 ))}
266 </Select>
267 </Field>
268 </div>
269 <details className="group rounded-md">
270 <summary className="cursor-pointer text-sm text-muted select-none hover:text-fg">Change the parts' impact</summary>
271 <div className="mt-2">
272 <ImpactPicker components={components} current={current} blank />
273 </div>
274 </details>
275 <Field label="Message">
276 <Textarea
277 name="message"
278 rows={4}
279 maxLength={4000}
280 placeholder={draft ? "What you found, for the team." : "What changed, in a sentence or two, for the people affected."}
281 />
282 </Field>
283 <div className="flex flex-wrap items-center justify-between gap-3">
284 {!draft ? (
285 <label className="flex items-center gap-2 text-sm">
286 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-(--g1t-accent)" />
287 Email subscribers (public updates only)
288 </label>
289 ) : (
290 <span />
291 )}
292 <Button type="submit" variant="lavender">
293 Post
294 </Button>
295 </div>
296 </form>
297 </Section>
298 );
299}
300
301export default function Incident({ loaderData, actionData }: Route.ComponentProps) {
302 const { incident, components, error, done, staff, now } = loaderData;
303 const failed = actionData as ActionData | undefined;
304 const errorFor = (intent: string) => (failed?.intent === intent ? failed.error : null);
305 if (!incident) {
306 return (
307 <main className="mx-auto max-w-6xl px-4 py-8 sm:py-10">
308 <BackLink to="/incidents">Incidents</BackLink>
309 <div className="mt-4">
310 <Notice tone="warn">The status worker did not answer: {error}</Notice>
311 </div>
312 </main>
313 );
314 }
315 const names = new Map(components.map((c) => [c.key, c.name]));
316 const open = incident.resolved_at == null;
317 const isPublic = incident.visibility === "public";
318 const affected = incident.components.filter((c) => c.impact !== "operational");
319 const timeline = [...incident.timeline].reverse();
320
321 return (
322 <main className="mx-auto max-w-6xl px-4 py-8 sm:py-10">
323 <BackLink to={incident.visibility === "draft" ? "/incidents?tab=drafts" : open ? "/incidents" : "/incidents?tab=resolved"}>Incidents</BackLink>
324 <Header incident={incident} now={now} />
325
326 <div className="mt-4 space-y-3">
327 {done && <Notice tone="ok">{done}</Notice>}
328 {failed && !["update", "roles", "followup_add", "followup_set", "resolve", "publish", "dismiss"].includes(failed.intent) && (
329 <Notice tone="error">{failed.error}</Notice>
330 )}
331 </div>
332
333 {incident.visibility === "draft" && (
334 <section className="mt-5 rounded-lg border border-warn/35 bg-warn/6 p-4 sm:p-5">
335 <h2 className="font-semibold">This is a draft</h2>
336 <p className="mt-1 text-sm text-muted">
337 {incident.source === "detected"
338 ? "The checks failed or were slow on four of five checks in a row and made this. Nobody outside sees it until you publish it. If it was a blip, dismiss it; if no one picks it up and its parts stay healthy for 10 minutes, it is dismissed on its own. Left unacknowledged, the alert goes out again after 45 minutes, then every 6 hours."
339 : "Nobody outside sees it until you publish it."}
340 </p>
341 <div className="mt-4 grid gap-4 lg:grid-cols-[minmax(0,1fr)_18rem]">
342 <form method="post" className="space-y-3">
343 <input type="hidden" name="intent" value="publish" />
344 {errorFor("publish") && <Notice tone="error">{errorFor("publish")}</Notice>}
345 <Field label="Public title">
346 <Input name="title" defaultValue={incident.title.replace(/^Detected: /, "")} maxLength={120} required />
347 </Field>
348 <Field label="First public update">
349 <Textarea name="message" rows={3} required maxLength={4000} placeholder="We are looking into reports that…" />
350 </Field>
351 <div className="flex flex-wrap items-center justify-between gap-3">
352 <label className="flex items-center gap-2 text-sm">
353 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-(--g1t-accent)" />
354 Email subscribers
355 </label>
356 <Button type="submit" variant="lavender">
357 Publish to the status page
358 </Button>
359 </div>
360 </form>
361 <form method="post" className="space-y-3 lg:border-l lg:border-warn/20 lg:pl-4">
362 <input type="hidden" name="intent" value="dismiss" />
363 {errorFor("dismiss") && <Notice tone="error">{errorFor("dismiss")}</Notice>}
364 <Field label="Not an incident?">
365 <Input name="reason" maxLength={500} placeholder="A blip: answered again within minutes" />
366 </Field>
367 <Button type="submit" variant="quiet">
368 Dismiss draft
369 </Button>
370 </form>
371 </div>
372 </section>
373 )}
374
375 <div className="mt-6 grid gap-6 lg:grid-cols-[minmax(0,1fr)_20rem]">
376 <div className="min-w-0 space-y-6">
377 {incident.visibility !== "dismissed" && <UpdateForm incident={incident} components={components} error={errorFor("update")} />}
378 {(incident.checks?.length ?? 0) > 0 && (
379 <Section
380 id="checks"
381 title="Checks"
382 description="How long each check of its parts took, from 30 minutes before it began to 30 minutes after it ended (at most a day), with the data centre each ran from. Kept for 7 days."
383 >
384 <IncidentChecks checks={incident.checks} names={names} />
385 </Section>
386 )}
387 <Section title="Timeline" description="Newest first. Public updates are what the status page shows; everything else is for staff.">
388 <ol className="space-y-2">
389 {timeline.map((entry) => (
390 <EntryLine key={entry.id} entry={entry} />
391 ))}
392 </ol>
393 </Section>
394 </div>
395
396 <aside className="min-w-0 space-y-6">
397 {open && isPublic && (
398 <Section id="resolve" title="Resolve" description="Closes it, posts a last public update, and stops colouring the parts.">
399 <form method="post" action="#resolve" className="space-y-3">
400 <input type="hidden" name="intent" value="resolve" />
401 {errorFor("resolve") && <Notice tone="error">{errorFor("resolve")}</Notice>}
402 <Textarea
403 name="message"
404 rows={3}
405 required
406 maxLength={4000}
407 aria-label="Resolution update"
408 placeholder="Pushes work again. A bad deploy was rolled back; nothing was lost."
409 />
410 <label className="flex items-center gap-2 text-sm">
411 <input type="checkbox" name="notify" defaultChecked={notifyByDefault(incident.severity)} className="accent-(--g1t-accent)" />
412 Email subscribers
413 </label>
414 <Button type="submit" variant="lavender" className="w-full">
415 <CheckCircle2 size={15} aria-hidden="true" /> Resolve incident
416 </Button>
417 </form>
418 </Section>
419 )}
420
421 {!open && isPublic && (
422 <Section
423 title="Postmortem"
424 description={
425 incident.postmortem_published_at ? "Published on the incident's page." : incident.postmortem ? "Drafted, not published." : "Not started."
426 }
427 >
428 <div className="flex flex-wrap items-center gap-2">
429 <Link to={`/incidents/${incident.id}/postmortem`} className="inline-flex items-center gap-1.5 text-sm font-medium text-accent hover:underline">
430 <FileText size={14} aria-hidden="true" />
431 {incident.postmortem ? "Edit the postmortem" : "Write the postmortem"}
432 </Link>
433 {incident.postmortem_published_at && <Badge tone="mint">Published</Badge>}
434 </div>
435 </Section>
436 )}
437
438 <Section title="Parts" description={open ? "What the status page shows for each while it is open." : "What it did to each."}>
439 {affected.length === 0 ? (
440 <p className="text-sm text-muted">None marked.</p>
441 ) : (
442 <ul className="space-y-2">
443 {affected.map((c) => (
444 <li key={c.key} className="flex items-center justify-between gap-2 text-sm">
445 <span className="min-w-0 truncate">{names.get(c.key) ?? c.key}</span>
446 <ImpactBadge impact={c.impact} />
447 </li>
448 ))}
449 </ul>
450 )}
451 </Section>
452
453 <Section id="roles" title="Roles">
454 <form method="post" action="#roles" className="space-y-3">
455 <input type="hidden" name="intent" value="roles" />
456 {errorFor("roles") && <Notice tone="error">{errorFor("roles")}</Notice>}
457 <Field label="Incident commander">
458 <Input name="commander" type="email" list="staff" defaultValue={incident.commander ?? ""} placeholder="Nobody" />
459 </Field>
460 <Field label="Communications">
461 <Input name="communications" type="email" list="staff" defaultValue={incident.communications ?? ""} placeholder="Nobody" />
462 </Field>
463 <Button type="submit" variant="quiet">
464 Save roles
465 </Button>
466 </form>
467 </Section>
468
469 <Section id="followups" title="Follow-ups" description="What to change so it does not happen again.">
470 {errorFor("followup_set") && (
471 <div className="mb-3">
472 <Notice tone="error">{errorFor("followup_set")}</Notice>
473 </div>
474 )}
475 {incident.followups.length > 0 && (
476 <ul className="mb-4 space-y-1.5">
477 {incident.followups.map((f) => (
478 <li key={f.id}>
479 <form method="post" action="#followups" className="flex items-start gap-2">
480 <input type="hidden" name="intent" value="followup_set" />
481 <input type="hidden" name="followup" value={f.id} />
482 <input type="hidden" name="done" value={f.done_at ? "0" : "1"} />
483 <button
484 type="submit"
485 aria-label={f.done_at ? `Mark “${f.title}” not done` : `Mark “${f.title}” done`}
486 className="mt-0.5 shrink-0 rounded text-faint hover:text-accent"
487 >
488 {f.done_at ? <CheckCircle2 size={16} className="text-success" /> : <Circle size={16} />}
489 </button>
490 <span className="min-w-0 text-sm">
491 <span className={f.done_at ? "text-faint line-through" : ""}>{f.title}</span>
492 {f.owner && <span className="block font-mono text-xs text-faint">{f.owner}</span>}
493 </span>
494 </form>
495 </li>
496 ))}
497 </ul>
498 )}
499 <form method="post" action="#followups" className="space-y-2">
500 <input type="hidden" name="intent" value="followup_add" />
501 {errorFor("followup_add") && <Notice tone="error">{errorFor("followup_add")}</Notice>}
502 <Input name="title" required maxLength={200} placeholder="Alert on push errors above 1%" aria-label="Follow-up" />
503 <div className="flex gap-2">
504 <Input name="owner" type="email" list="staff" placeholder="Owner (optional)" aria-label="Owner" />
505 <Button type="submit" variant="quiet">
506 Add
507 </Button>
508 </div>
509 </form>
510 </Section>
511
512 <Section title="Links">
513 <ul className="space-y-1.5 text-sm">
514 {isPublic ? (
515 <li>
516 <a href={incident.url} className="inline-flex items-center gap-1.5 text-accent hover:underline">
517 Status page permalink <ExternalLink size={13} aria-hidden="true" />
518 </a>
519 <span className="block truncate font-mono text-xs text-faint">{incident.url}</span>
520 </li>
521 ) : (
522 <li className="text-muted">No public page until it is published.</li>
523 )}
524 <li>
525 <Link to="/audit" className="text-muted hover:text-fg hover:underline">
526 Audit log
527 </Link>
528 </li>
529 </ul>
530 </Section>
531 <datalist id="staff">
532 {staff.map((email) => (
533 <option key={email} value={email} />
534 ))}
535 </datalist>
536 </aside>
537 </div>
538 </main>
539 );
540}