Sign in

flagon-io/g1t

public

g1t/apps/status/src/index.ts

810 lines37,599 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1/**
2 * status.g1t.sh: whether each part of g1t is working, how it has done over
3 * 90 days, and what staff have said about incidents and maintenance.
4 *
5 * A Worker of its own, apart from the site, so it stays up when g1t does
6 * not. Every minute a cron checks each part over the public internet, as
7 * people reach it, and keeps the result in D1. The same run moves planned
8 * maintenance along, and drafts an incident for staff when a part keeps
9 * failing (detect.ts). Pages are drawn from what is kept, never by
10 * checking on the spot, and kept at the edge for 30 seconds.
11 *
12 * Staff run incidents from sudo, through the `StatusAdmin` entrypoint,
13 * which only a service binding reaches. Every change there is audited.
14 *
15 * GET / the page
16 * GET /status.json the same as JSON (snake_case, CORS open)
17 * GET /badge.svg a small badge
18 * GET /incidents/<id> an incident's updates and postmortem
19 * GET /maintenance/<id> a maintenance window's updates
20 * GET /history the last 12 months, by month
21 * GET /feed.xml, /feed.json every public update, newest first
22 * GET /subscribe subscribing by email
23 * POST /subscribe asks for a subscription: a confirmation email
24 * GET|POST /subscribe/confirm?token= confirms (GET shows a button: link scanners must not confirm)
25 * GET|POST /unsubscribe?token= leaves (POST also takes RFC 8058 one-click)
26 */
27import { WorkerEntrypoint } from "cloudflare:workers";
28import {
29 type AdminIncident,
30 type AdminIncidentDetail,
31 type AdminMaintenance,
32 type DeclareIncident,
33 type FollowUp,
34 type IncidentChange,
35 type MaintenanceChange,
36 type NewMaintenance,
37 type Postmortem,
38 type PostmortemFields,
39 type PublishIncident,
40 type Result,
41 type RolesChange,
42 type StatusAdminApi,
43 type StatusAuditEntry,
44 type StatusBoard,
45 billingClient,
46 fail,
47 ok,
48} from "@g1t/contracts";
49import bricolage from "@g1t/theme/fonts/bricolage-grotesque-latin.woff2";
50import hanken from "@g1t/theme/fonts/hanken-grotesk-latin.woff2";
51import plexMono from "@g1t/theme/fonts/ibm-plex-mono-latin-400.woff2";
52
53import { type Targets, components } from "./components.ts";
54import { DETECT_AFTER, detect, detectedImpact, draftTitle } from "./detect.ts";
55import { type EmailBinding, type Sender, alertLetter, bindingSender, confirmLetter, render as renderMail, unsubscribeHeaders, updateLetter } from "./email.ts";
56import { atom, feedItems, jsonFeed } from "./feed.ts";
57import {
58 type Entry,
59 applyChange,
60 applyRoles,
61 checkChange,
62 checkDeclare,
63 checkFollowUp,
64 checkMaintenance,
65 checkMaintenanceChange,
66 checkPostmortem,
67 checkPublish,
68 checkRoles,
69 postmortemReady,
70 SEVERITY_LABEL,
71 shortId,
72} from "./incidents.ts";
73import { INCIDENT_STATUS, type PageModel, buildPage, classify, underMaintenance } from "./model.ts";
74import { stamp } from "./postmortem.ts";
75import { runCheck } from "./probe.ts";
76import {
77 FAVICON,
78 SCRIPT,
79 type PageOptions,
80 renderBadge,
81 renderHistory,
82 renderIncident,
83 renderMaintenance,
84 renderMessage,
85 renderPage,
86 renderSubscribe,
87} from "./render.ts";
88import {
89 type Observation,
90 addFollowUp,
91 addSystemLines,
92 auditLog,
93 board,
94 confirmSubscription,
95 createIncident,
96 dueMaintenance,
97 facts,
98 incidentDetail,
99 load,
100 loadHistory,
101 loadPublicIncident,
102 loadPublicMaintenance,
103 loadStreaks,
104 maintenanceById,
105 maintenanceUrl,
106 maintenanceUpdate,
107 openCount,
108 openRefs,
109 publishPostmortem,
110 recipients,
111 record,
112 requestSubscription,
113 saveIncident,
114 savePostmortem,
115 saveStreaks,
116 scheduleMaintenance,
117 setFollowUp,
118 unsubscribe,
119} from "./store.ts";
120import { CONFIRM_TTL_MS, RESEND_AFTER_MS, chosenParts, hashToken, newToken, normalizeEmail, readUnsubscribeToken, unsubscribeToken } from "./subscribers.ts";
121
122export interface Env extends Partial<Targets> {
123 DB: D1Database;
124 /** Billing, for reading its price book. Optional: without it, billing is not listed. */
125 BILLING?: Fetcher;
126 /** Where help is. */
127 SUPPORT_URL?: string;
128 /**
129 * The site's address as people's browsers reach it, for the page's links,
130 * when the checks reach it by another (self-hosted: `http://g1t:8787`
131 * inside Compose). SITE_URL when empty.
132 */
133 PUBLIC_SITE_URL?: string;
134 /** The page's share card; empty for none. */
135 OG_IMAGE?: string;
136 /** This page's own address, for links in email and feeds made outside a request. */
137 STATUS_URL?: string;
138 /** Where sudo is, for the staff alert's link. */
139 SUDO_URL?: string;
140 /** Who hears about detected drafts. Empty sends none. */
141 STATUS_ALERT_EMAIL?: string;
142 /** The From of every email. */
143 STATUS_FROM?: string;
144 /** Signs unsubscribe links (a secret). Without it, email subscriptions are off; the feeds still work. */
145 STATUS_SECRET?: string;
146 /** Cloudflare Email Sending (`send_email`). Without it, nothing is emailed. */
147 EMAIL?: EmailBinding;
148}
149
150/** How long the edge keeps a page or the JSON. */
151const CACHE_SECONDS = 30;
152/** Older than this, a visit asks for a round of checks too (a missed cron, or `wrangler dev`). */
153const BEHIND_MS = 3 * 60 * 1000;
154/** How many subscribers one update emails at most, within a Worker's limits. */
155const MAX_RECIPIENTS = 900;
156
157function parts(env: Env) {
158 return components(env, env.BILLING != null);
159}
160
161function names(env: Env): Map<string, string> {
162 return new Map(parts(env).map((p) => [p.key, p.name]));
163}
164
165/** This page's address: the request's own, or STATUS_URL outside a request. */
166function originOf(env: Env, url?: URL): string {
167 return (url?.origin ?? env.STATUS_URL ?? "https://status.g1t.sh").replace(/\/+$/, "");
168}
169
170function sender(env: Env): Sender | null {
171 return bindingSender(env.EMAIL, env.STATUS_FROM || undefined);
172}
173
174/** Whether subscribers can sign up: a way to send, and a secret to sign their links. */
175function emailOn(env: Env): boolean {
176 return sender(env) != null && !!env.STATUS_SECRET;
177}
178
179/** One round of checks, kept. Parts under maintenance are checked but not tallied. */
180export async function checkAll(env: Env, now = new Date()): Promise<Observation[]> {
181 const billing = env.BILLING;
182 const list = parts(env);
183 const observations = await Promise.all(
184 list.map(async (info): Promise<Observation> => {
185 const result = await runCheck(info.check, {
186 fetch: (url, init) => fetch(url, init),
187 billing: billing ? () => billingClient(billing).prices() : null,
188 });
189 const { state, detail } = classify(result);
190 return { component: info.key, state, detail, latency_ms: result ? Math.round(result.ms) : null };
191 }),
192 );
193 const { maintenance } = await load(env.DB, now, originOf(env));
194 await record(env.DB, observations, now, underMaintenance(maintenance, now));
195 return observations;
196}
197
198// --- Email ---------------------------------------------------------------------------
199
200/**
201 * Emails confirmed subscribers who want news about `about`, in the
202 * background. Returns how many it will email, or null when email is off.
203 */
204async function notify(
205 env: Env,
206 ctx: { waitUntil(p: Promise<unknown>): void },
207 about: string[],
208 mail: { heading: string; text: string; url: string },
209): Promise<number | null> {
210 const send = sender(env);
211 const secret = env.STATUS_SECRET;
212 if (!send || !secret) return null;
213 const list = (await recipients(env.DB, about)).slice(0, MAX_RECIPIENTS);
214 const origin = originOf(env);
215 const affects = about.map((k) => names(env).get(k) ?? k);
216 ctx.waitUntil(
217 (async () => {
218 let failed = 0;
219 for (let i = 0; i < list.length; i += 6) {
220 await Promise.all(
221 list.slice(i, i + 6).map(async (r) => {
222 const link = `${origin}/unsubscribe?token=${encodeURIComponent(await unsubscribeToken(secret, r.id))}`;
223 const { text, html } = renderMail(updateLetter({ heading: mail.heading, text: mail.text, url: mail.url, affects, unsubscribe: link }));
224 await send.send({ to: r.email, subject: mail.heading, text, html, headers: unsubscribeHeaders(link) }).catch(() => void (failed += 1));
225 }),
226 );
227 }
228 console.log(JSON.stringify({ event: "status.notified", sent: list.length - failed, failed }));
229 })(),
230 );
231 return list.length;
232}
233
234// --- The cron: detection and maintenance --------------------------------------------------
235
236async function afterChecks(env: Env, ctx: { waitUntil(p: Promise<unknown>): void }, observations: Observation[], now: Date): Promise<void> {
237 const origin = originOf(env);
238 const named = names(env);
239 // Maintenance whose window opened or closed.
240 for (const m of await dueMaintenance(env.DB, now, origin)) {
241 const ended = Date.parse(m.ends_at) <= now.getTime();
242 const text = ended ? "The maintenance is complete." : "The maintenance has begun.";
243 const notified = m.notify ? await notify(env, ctx, m.components, { heading: `${ended ? "Completed" : "In progress"}: ${m.title}`, text, url: m.url }) : null;
244 await maintenanceUpdate(env.DB, m.id, ended ? "completed" : "in_progress", text, "status", notified, now, {
245 action: ended ? "maintenance_completed" : "maintenance_started",
246 detail: `${m.title} (on schedule)`,
247 });
248 }
249 // Detection.
250 const [streaks, open, page] = await Promise.all([loadStreaks(env.DB), openRefs(env.DB), load(env.DB, now, origin)]);
251 const found = detect(streaks, observations, open, underMaintenance(page.maintenance, now), now);
252 await saveStreaks(env.DB, found.streaks);
253 const lines = [
254 ...found.failing.map((f) => ({
255 incident: f.incident,
256 kind: "failing" as const,
257 text: `${named.get(f.key) ?? f.key}: checks ${f.state === "down" ? "failing" : "slow"} ${DETECT_AFTER} times in a row since ${stamp(f.since)}.`,
258 })),
259 ...found.recovered.map((r) => ({
260 incident: r.incident,
261 kind: "recovered" as const,
262 text: `${named.get(r.key) ?? r.key}: answering again, after ${r.checks} failed or slow checks since ${stamp(r.since)}.`,
263 })),
264 ];
265 await addSystemLines(env.DB, lines, now);
266 if (found.draft.length) {
267 const core = new Set(parts(env).filter((p) => p.core).map((p) => p.key));
268 const title = draftTitle(found.draft.map((d) => ({ name: named.get(d.key) ?? d.key, state: d.state })));
269 const since = found.draft.map((d) => d.since).sort()[0]!;
270 const id = await createIncident(
271 env.DB,
272 {
273 title,
274 severity: found.draft.some((d) => d.state === "down" && core.has(d.key)) ? "sev2" : "sev3",
275 status: "investigating",
276 visibility: "draft",
277 source: "detected",
278 components: found.draft.map((d) => ({ key: d.key, impact: detectedImpact(d.state) })),
279 started_at: since,
280 acknowledged_at: null,
281 commander: null,
282 communications: null,
283 by: "status",
284 },
285 [
286 {
287 kind: "detected",
288 public: false,
289 status: null,
290 text: `The checks failed ${DETECT_AFTER} times in a row: ${found.draft.map((d) => `${named.get(d.key) ?? d.key} (${d.state === "down" ? "not answering" : "slow"})`).join(", ")}. Not on the status page until it is published.`,
291 },
292 ],
293 now,
294 { action: "incident_detected", detail: title },
295 );
296 console.warn(JSON.stringify({ event: "status.detected", id, parts: found.draft.map((d) => d.key) }));
297 const to = (env.STATUS_ALERT_EMAIL ?? "").trim();
298 const send = sender(env);
299 if (to && send) {
300 const link = `${(env.SUDO_URL || "https://sudo.g1t.sh").replace(/\/+$/, "")}/incidents/${id}`;
301 const { text, html } = renderMail(alertLetter({ title, parts: found.draft.map((d) => named.get(d.key) ?? d.key), since: stamp(since), link }));
302 ctx.waitUntil(send.send({ to, subject: `[g1t status] ${title}`, text, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
303 }
304 }
305}
306
307// --- Pages -------------------------------------------------------------------------------
308
309async function model(env: Env, now: Date, origin: string): Promise<PageModel> {
310 const stored = await load(env.DB, now, origin);
311 return buildPage({
312 parts: parts(env),
313 current: stored.current,
314 checkedAt: stored.checkedAt,
315 days: stored.days,
316 incidents: stored.incidents,
317 maintenance: stored.maintenance,
318 now,
319 });
320}
321
322/** When this isolate last asked for a catch-up round, so a busy page asks once. */
323let caughtUpAt = 0;
324
325function catchUp(env: Env, ctx: ExecutionContext, page: PageModel, now: Date) {
326 const checked = page.report.checked_at ? Date.parse(page.report.checked_at) : 0;
327 if (now.getTime() - checked < BEHIND_MS || now.getTime() - caughtUpAt < BEHIND_MS) return;
328 caughtUpAt = now.getTime();
329 ctx.waitUntil(checkAll(env, now).catch((error) => console.error(JSON.stringify({ event: "status.catch_up_failed", error: String(error) }))));
330}
331
332const PAGE_POLICY = [
333 "default-src 'none'",
334 "script-src 'self'",
335 "style-src 'unsafe-inline'",
336 "font-src 'self'",
337 "img-src 'self' data:",
338 "base-uri 'none'",
339 "form-action 'self'",
340 "frame-ancestors 'none'",
341].join("; ");
342
343const COMMON = {
344 "x-content-type-options": "nosniff",
345 "referrer-policy": "strict-origin-when-cross-origin",
346};
347
348function edgeCache(): Cache | null {
349 return (globalThis as unknown as { caches?: { default?: Cache } }).caches?.default ?? null;
350}
351
352/** A response from the edge cache, or made and kept there. */
353async function cached(request: Request, ctx: ExecutionContext, make: () => Promise<Response>): Promise<Response> {
354 const cache = edgeCache();
355 const url = new URL(request.url);
356 const key = new Request(url.origin + url.pathname, { method: "GET" });
357 if (cache) {
358 const hit = await cache.match(key).catch(() => undefined);
359 if (hit) return hit;
360 }
361 const response = await make();
362 if (cache && response.ok) ctx.waitUntil(cache.put(key, response.clone()).catch(() => undefined));
363 return response;
364}
365
366const FONTS: Record<string, ArrayBuffer> = {
367 "/fonts/hanken-grotesk.woff2": hanken,
368 "/fonts/bricolage-grotesque.woff2": bricolage,
369 "/fonts/ibm-plex-mono.woff2": plexMono,
370};
371
372function html(body: string, cacheControl: string, status = 200): Response {
373 return new Response(body, {
374 status,
375 headers: { "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "content-security-policy": PAGE_POLICY, ...COMMON },
376 });
377}
378
379async function form(request: Request): Promise<FormData> {
380 try {
381 return await request.formData();
382 } catch {
383 return new FormData();
384 }
385}
386
387/** Subscribing, confirming and leaving: the page's only writes. */
388async function subscriptions(request: Request, env: Env, ctx: ExecutionContext, url: URL, options: PageOptions): Promise<Response | null> {
389 const path = url.pathname;
390 const noStore = "no-store";
391 const message = (title: string, text: string, status = 200, f?: { action: string; fields: Record<string, string>; button: string }) =>
392 html(renderMessage({ ...options, selfUrl: `${url.origin}${path}` }, { title, text, form: f }), noStore, status);
393 const post = request.method === "POST";
394
395 if (path === "/subscribe" && post) {
396 if (!emailOn(env)) return message("Email updates are not available", "Follow the Atom or JSON feed instead.", 503);
397 const data = await form(request);
398 if (String(data.get("website") ?? "")) return message("Check your inbox", "If the address is right, a confirmation link is on its way.");
399 const email = normalizeEmail(data.get("email"));
400 if (!email) return message("That is not an email address", "Go back and check it.", 400);
401 const chosen = chosenParts(data.getAll("components").map(String), parts(env).map((p) => p.key));
402 const token = newToken();
403 const { send } = await requestSubscription(env.DB, email, chosen, await hashToken(token), new Date(), CONFIRM_TTL_MS, RESEND_AFTER_MS);
404 if (send) {
405 const link = `${url.origin}/subscribe/confirm?token=${encodeURIComponent(token)}`;
406 const { text, html: body } = renderMail(confirmLetter(link, chosen ? chosen.map((k) => names(env).get(k) ?? k) : null));
407 ctx.waitUntil(sender(env)!.send({ to: email, subject: "Confirm your subscription to g1t status", text, html: body }).catch((e) => console.error(JSON.stringify({ event: "status.confirm_failed", error: String(e) }))));
408 }
409 return message("Check your inbox", "If the address is right, a confirmation link is on its way. It works for 24 hours.");
410 }
411 if (path === "/subscribe/confirm") {
412 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
413 if (!token) return message("That link is incomplete", "Copy the whole link from the email.", 400);
414 if (!post) return message("Confirm your subscription", "One more step: confirm to start getting emails about incidents and maintenance.", 200, { action: "/subscribe/confirm", fields: { token }, button: "Confirm subscription" });
415 const done = await confirmSubscription(env.DB, await hashToken(token), new Date());
416 if (!done) return message("That link has expired", "Confirmation links work for 24 hours and once. Subscribe again for a new one.", 410);
417 return message("You are subscribed", `${done.email} will get an email when g1t posts an incident or maintenance${done.parts ? ` affecting ${done.parts.map((k) => names(env).get(k) ?? k).join(", ")}` : ""}. Every email has a link to unsubscribe.`);
418 }
419 if (path === "/unsubscribe") {
420 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
421 const id = env.STATUS_SECRET && token ? await readUnsubscribeToken(env.STATUS_SECRET, token) : null;
422 if (!id) return message("That link is not valid", "Use the unsubscribe link at the bottom of any email from g1t status.", 400);
423 if (!post) return message("Unsubscribe", "Stop getting emails from g1t status?", 200, { action: "/unsubscribe", fields: { token }, button: "Unsubscribe" });
424 await unsubscribe(env.DB, id);
425 return message("You are unsubscribed", "You will not get any more emails from g1t status. You can subscribe again at any time.");
426 }
427 return null;
428}
429
430async function handle(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
431 const url = new URL(request.url);
432 const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, "") : url.pathname;
433 if (request.method === "OPTIONS" && (path === "/status.json" || path === "/feed.json")) {
434 return new Response(null, {
435 status: 204,
436 headers: { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, HEAD", "access-control-max-age": "86400" },
437 });
438 }
439 const now = new Date();
440 const origin = originOf(env, url);
441 const site = env.PUBLIC_SITE_URL || env.SITE_URL || url.origin;
442 const options: PageOptions = {
443 siteUrl: site,
444 supportUrl: env.SUPPORT_URL || `${site}/support`,
445 ogImage: env.OG_IMAGE ?? "",
446 selfUrl: `${url.origin}${path === "/" ? "/" : path}`,
447 now,
448 email: emailOn(env),
449 };
450
451 if (request.method === "POST") {
452 const answer = await subscriptions(request, env, ctx, url, options);
453 return answer ?? new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD", ...COMMON } });
454 }
455 if (request.method !== "GET" && request.method !== "HEAD") {
456 return new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD, POST", ...COMMON } });
457 }
458 const fresh = (cacheSeconds = CACHE_SECONDS) => `public, max-age=${cacheSeconds}`;
459 const notFound = () => html(renderMessage(options, { title: "Not found", text: "There is nothing at this address." }), fresh(), 404);
460
461 switch (path) {
462 case "/":
463 return cached(request, ctx, async () => {
464 const page = await model(env, now, origin);
465 catchUp(env, ctx, page, now);
466 return html(renderPage(page, options), fresh());
467 });
468 case "/status.json":
469 return cached(request, ctx, async () => {
470 const page = await model(env, now, origin);
471 catchUp(env, ctx, page, now);
472 return Response.json(page.report, { headers: { "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON } });
473 });
474 case "/badge.svg":
475 return cached(request, ctx, async () => {
476 const page = await model(env, now, origin);
477 return new Response(renderBadge(page.report.overall.state, page.report.overall.title), {
478 headers: { "content-type": "image/svg+xml", "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON },
479 });
480 });
481 case "/history":
482 return cached(request, ctx, async () => {
483 const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 11, 1));
484 const { incidents, maintenance } = await loadHistory(env.DB, since, origin);
485 return html(renderHistory(incidents, maintenance, options), fresh());
486 });
487 case "/feed.xml":
488 case "/feed.json":
489 return cached(request, ctx, async () => {
490 const { incidents, maintenance } = await loadHistory(env.DB, new Date(now.getTime() - 365 * 86_400_000), origin);
491 const items = feedItems(incidents, maintenance);
492 const feed = { origin, title: "g1t status", updated: now.toISOString() };
493 const headers = { "cache-control": fresh(60), "access-control-allow-origin": "*", ...COMMON };
494 return path === "/feed.xml"
495 ? new Response(atom(items, feed), { headers: { "content-type": "application/atom+xml; charset=utf-8", ...headers } })
496 : Response.json(jsonFeed(items, feed), { headers: { "content-type": "application/feed+json; charset=utf-8", ...headers } });
497 });
498 case "/subscribe":
499 return html(renderSubscribe(options, parts(env).map(({ key, name }) => ({ key, name }))), fresh(300));
500 case "/subscribe/confirm":
501 case "/unsubscribe":
502 return (await subscriptions(request, env, ctx, url, options))!;
503 case "/status.js":
504 return new Response(SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
505 case "/favicon.svg":
506 case "/favicon.ico":
507 return new Response(FAVICON, { headers: { "content-type": "image/svg+xml", "cache-control": fresh(86400), ...COMMON } });
508 case "/robots.txt":
509 return new Response("User-agent: *\nAllow: /\nDisallow: /subscribe/confirm\nDisallow: /unsubscribe\n", {
510 headers: { "content-type": "text/plain", "cache-control": fresh(86400) },
511 });
512 }
513 const incident = /^\/incidents\/([a-z0-9-]{1,64})$/.exec(path);
514 if (incident) {
515 return cached(request, ctx, async () => {
516 const found = await loadPublicIncident(env.DB, incident[1]!, origin);
517 return found ? html(renderIncident(found.incident, found.postmortem, names(env), options), fresh()) : notFound();
518 });
519 }
520 const maintenance = /^\/maintenance\/([a-z0-9-]{1,64})$/.exec(path);
521 if (maintenance) {
522 return cached(request, ctx, async () => {
523 const found = await loadPublicMaintenance(env.DB, maintenance[1]!, origin);
524 return found ? html(renderMaintenance(found, names(env), options), fresh()) : notFound();
525 });
526 }
527 const font = FONTS[path];
528 if (font) {
529 return new Response(font, { headers: { "content-type": "font/woff2", "cache-control": "public, max-age=31536000, immutable", ...COMMON } });
530 }
531 return notFound();
532}
533
534export default {
535 async fetch(request, env, ctx) {
536 try {
537 return await handle(request, env, ctx);
538 } catch (error) {
539 console.error(JSON.stringify({ event: "status.failed", path: new URL(request.url).pathname, error: String(error) }));
540 return new Response("The status page could not be drawn. Try again in a minute.", {
541 status: 503,
542 headers: { "content-type": "text/plain; charset=utf-8", "retry-after": "60", ...COMMON },
543 });
544 }
545 },
546 async scheduled(_controller, env, ctx) {
547 const now = new Date();
548 ctx.waitUntil(
549 checkAll(env, now)
550 .then(async (observations) => {
551 const failing = observations.filter((o) => o.state === "down" || o.state === "degraded");
552 if (failing.length) console.warn(JSON.stringify({ event: "status.trouble", parts: failing }));
553 await afterChecks(env, ctx, observations, now);
554 })
555 .catch((error) => console.error(JSON.stringify({ event: "status.cron_failed", error: String(error) }))),
556 );
557 },
558} satisfies ExportedHandler<Env>;
559
560// --- Staff ---------------------------------------------------------------------------------
561
562/**
563 * Staff only: running incidents and maintenance. Reached only through a
564 * service binding (sudo's `STATUS`); status.g1t.sh's own address cannot.
565 */
566export class StatusAdmin extends WorkerEntrypoint<Env> implements StatusAdminApi {
567 private known() {
568 return parts(this.env).map((p) => p.key);
569 }
570
571 private origin() {
572 return originOf(this.env);
573 }
574
575 private async detail(id: string): Promise<AdminIncidentDetail | null> {
576 return incidentDetail(this.env.DB, String(id), this.origin(), names(this.env));
577 }
578
579 private async summary(id: string): Promise<AdminIncident> {
580 const { timeline: _t, followups: _f, postmortem: _p, postmortem_draft: _d, url: _u, ...incident } = (await this.detail(id))!;
581 return incident;
582 }
583
584 /** Emails a public update to subscribers when asked; the count to keep with it. */
585 private async announce(incident: { id: string; title: string; components: { key: string; impact: string }[] }, status: keyof typeof INCIDENT_STATUS, text: string, wanted: boolean) {
586 if (!wanted) return null;
587 const about = incident.components.filter((c) => c.impact !== "operational").map((c) => c.key);
588 return notify(this.env, this.ctx, about, {
589 heading: `${INCIDENT_STATUS[status]}: ${incident.title}`,
590 text,
591 url: `${this.origin()}/incidents/${incident.id}`,
592 });
593 }
594
595 async components(): Promise<{ key: string; name: string }[]> {
596 return parts(this.env).map(({ key, name }) => ({ key, name }));
597 }
598
599 async board(): Promise<StatusBoard> {
600 return { ...(await board(this.env.DB, new Date(), this.origin())), email: emailOn(this.env) };
601 }
602
603 async incident(id: string): Promise<AdminIncidentDetail | null> {
604 return this.detail(id);
605 }
606
607 async openCount(): Promise<number> {
608 return openCount(this.env.DB);
609 }
610
611 async declare(input: DeclareIncident): Promise<Result<AdminIncident>> {
612 const checked = checkDeclare(input, this.known());
613 if (!checked.ok) return fail("invalid", checked.error);
614 const v = checked.value;
615 const now = new Date();
616 const entries: (Entry & { notified?: number | null })[] = [
617 { kind: "declared", public: false, status: null, text: `Declared ${SEVERITY_LABEL[v.severity]}.` },
618 ];
619 if (v.commander) entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${v.commander}.` });
620 if (v.communications) entries.push({ kind: "role", public: false, status: null, text: `Communications: ${v.communications}.` });
621 const id = shortId();
622 const status = v.status ?? "investigating";
623 const notified = await this.announce({ id, title: v.title, components: v.components }, status, v.message, v.notify);
624 entries.push({ kind: "update", public: true, status, text: v.message, notified });
625 await createIncident(
626 this.env.DB,
627 {
628 title: v.title,
629 severity: v.severity,
630 status,
631 visibility: "public",
632 source: "declared",
633 components: v.components,
634 started_at: v.started_at ?? now.toISOString(),
635 acknowledged_at: now.toISOString(),
636 commander: v.commander ?? null,
637 communications: v.communications ?? null,
638 by: v.by,
639 },
640 entries,
641 now,
642 { action: "incident_declared", detail: `${SEVERITY_LABEL[v.severity]}: ${v.title}` },
643 id,
644 );
645 console.log(JSON.stringify({ event: "status.incident_declared", id, by: v.by }));
646 return ok(await this.summary(id));
647 }
648
649 async update(id: string, change: IncidentChange): Promise<Result<AdminIncident>> {
650 const checked = checkChange(change, this.known());
651 if (!checked.ok) return fail("invalid", checked.error);
652 const existing = await this.detail(id);
653 if (!existing) return fail("not_found", "No such incident.");
654 const now = new Date();
655 const applied = applyChange(facts(existing), checked.value, now, names(this.env));
656 if (!applied.ok) return fail("invalid", applied.error);
657 const { next, entries } = applied.value;
658 const update = entries.find((e) => e.kind === "update");
659 const notified = update
660 ? await this.announce({ id: existing.id, title: existing.title, components: next.components }, next.status, update.text, checked.value.notify === true)
661 : null;
662 const lines = entries.map((e) => (e === update ? { ...e, notified } : e));
663 const action = next.status === "resolved" && existing.status !== "resolved" ? "incident_resolved" : update ? "incident_update" : "incident_note";
664 await saveIncident(this.env.DB, existing.id, next, lines, now, checked.value.by, {
665 action,
666 detail: entries.map((e) => (e.kind === "update" || e.kind === "note" ? `${e.kind === "update" ? "Public" : "Note"}: ${e.text}` : e.text)).join(" ").slice(0, 500),
667 });
668 return ok(await this.summary(existing.id));
669 }
670
671 async roles(id: string, change: RolesChange): Promise<Result<AdminIncident>> {
672 const checked = checkRoles(change);
673 if (!checked.ok) return fail("invalid", checked.error);
674 const existing = await this.detail(id);
675 if (!existing) return fail("not_found", "No such incident.");
676 const now = new Date();
677 const { next, entries } = applyRoles(facts(existing), checked.value, now);
678 if (!entries.length) return ok(await this.summary(existing.id));
679 await saveIncident(this.env.DB, existing.id, next, entries, now, checked.value.by, { action: "incident_roles", detail: entries.map((e) => e.text).join(" ") });
680 return ok(await this.summary(existing.id));
681 }
682
683 async publish(id: string, input: PublishIncident): Promise<Result<AdminIncident>> {
684 const checked = checkPublish(input);
685 if (!checked.ok) return fail("invalid", checked.error);
686 const existing = await this.detail(id);
687 if (!existing) return fail("not_found", "No such incident.");
688 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be published.");
689 const now = new Date();
690 const at = now.toISOString();
691 const title = checked.value.title ?? existing.title;
692 const next = { ...facts(existing), visibility: "public" as const, acknowledged_at: existing.acknowledged_at ?? at, title, published_at: at };
693 const notified = await this.announce({ id: existing.id, title, components: existing.components }, existing.status, checked.value.message, checked.value.notify);
694 await saveIncident(
695 this.env.DB,
696 existing.id,
697 next,
698 [
699 ...(existing.acknowledged_at ? [] : [{ kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged." }]),
700 { kind: "published", public: false, status: null, text: title !== existing.title ? `Published as “${title}”.` : "Published to the status page." },
701 { kind: "update", public: true, status: existing.status, text: checked.value.message, notified },
702 ],
703 now,
704 checked.value.by,
705 { action: "incident_published", detail: title },
706 );
707 return ok(await this.summary(existing.id));
708 }
709
710 async dismiss(id: string, input: { reason: string; by: string }): Promise<Result<AdminIncident>> {
711 const existing = await this.detail(id);
712 if (!existing) return fail("not_found", "No such incident.");
713 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be dismissed; resolve a published incident instead.");
714 const by = String(input?.by ?? "").trim();
715 if (!by) return fail("invalid", "Who is making the change is missing.");
716 const reason = String(input?.reason ?? "").trim().slice(0, 500) || "No reason given.";
717 const now = new Date();
718 const at = now.toISOString();
719 const next = { ...facts(existing), visibility: "dismissed" as const, status: "resolved" as const, acknowledged_at: existing.acknowledged_at ?? at, resolved_at: at };
720 await saveIncident(this.env.DB, existing.id, next, [{ kind: "dismissed", public: false, status: null, text: `Dismissed: ${reason}` }], now, by, {
721 action: "incident_dismissed",
722 detail: `${existing.title}: ${reason}`,
723 });
724 return ok(await this.summary(existing.id));
725 }
726
727 async addFollowUp(id: string, input: { title: string; owner: string | null; by: string }): Promise<Result<FollowUp>> {
728 const checked = checkFollowUp(input);
729 if (!checked.ok) return fail("invalid", checked.error);
730 if (!(await this.detail(id))) return fail("not_found", "No such incident.");
731 return ok(await addFollowUp(this.env.DB, String(id), checked.value, new Date()));
732 }
733
734 async setFollowUp(id: string, followUp: string, input: { done: boolean; by: string }): Promise<Result<FollowUp>> {
735 const by = String(input?.by ?? "").trim();
736 if (!by) return fail("invalid", "Who is making the change is missing.");
737 const done = await setFollowUp(this.env.DB, String(id), String(followUp), input.done === true, by, new Date());
738 return done ? ok(done) : fail("not_found", "No such follow-up.");
739 }
740
741 async savePostmortem(id: string, input: PostmortemFields & { by: string }): Promise<Result<Postmortem>> {
742 const checked = checkPostmortem(input);
743 if (!checked.ok) return fail("invalid", checked.error);
744 const existing = await this.detail(id);
745 if (!existing) return fail("not_found", "No such incident.");
746 if (existing.visibility !== "public") return fail("conflict", "Only a published incident has a postmortem.");
747 const { by, ...fields } = checked.value;
748 await savePostmortem(this.env.DB, existing.id, fields, by, new Date());
749 return ok((await this.detail(existing.id))!.postmortem!);
750 }
751
752 async publishPostmortem(id: string, input: { publish: boolean; by: string }): Promise<Result<Postmortem>> {
753 const by = String(input?.by ?? "").trim();
754 if (!by) return fail("invalid", "Who is making the change is missing.");
755 const existing = await this.detail(id);
756 if (!existing) return fail("not_found", "No such incident.");
757 if (!existing.postmortem) return fail("conflict", "Save the postmortem before publishing it.");
758 if (input.publish) {
759 if (!existing.resolved_at) return fail("conflict", "Resolve the incident before publishing its postmortem.");
760 const missing = postmortemReady(existing.postmortem);
761 if (missing) return fail("invalid", missing);
762 }
763 await publishPostmortem(this.env.DB, existing.id, input.publish === true, by, new Date());
764 return ok((await this.detail(existing.id))!.postmortem!);
765 }
766
767 async scheduleMaintenance(input: NewMaintenance): Promise<Result<AdminMaintenance>> {
768 const checked = checkMaintenance(input, this.known());
769 if (!checked.ok) return fail("invalid", checked.error);
770 const v = checked.value;
771 const now = new Date();
772 const id = shortId();
773 const notified = v.notify
774 ? await notify(this.env, this.ctx, v.components, {
775 heading: `Planned maintenance: ${v.title}`,
776 text: `${v.message}\n\nWhen: ${stamp(v.starts_at)} to ${stamp(v.ends_at)}.`,
777 url: maintenanceUrl(this.origin(), id),
778 })
779 : null;
780 const made = await scheduleMaintenance(this.env.DB, v, notified, now, id);
781 return ok((await maintenanceById(this.env.DB, made, this.origin()))!);
782 }
783
784 async changeMaintenance(id: string, change: MaintenanceChange): Promise<Result<AdminMaintenance>> {
785 const checked = checkMaintenanceChange(change);
786 if (!checked.ok) return fail("invalid", checked.error);
787 const existing = await maintenanceById(this.env.DB, String(id), this.origin());
788 if (!existing) return fail("not_found", "No such maintenance.");
789 const v = checked.value;
790 if (existing.state === "completed" || existing.state === "cancelled") return fail("conflict", `This maintenance is ${existing.state}.`);
791 if (v.action === "start" && existing.state !== "scheduled") return fail("conflict", "It has already started.");
792 const state = v.action === "start" ? "in_progress" : v.action === "complete" ? "completed" : v.action === "cancel" ? "cancelled" : null;
793 const text =
794 v.message ||
795 (v.action === "start" ? "The maintenance has begun." : v.action === "complete" ? "The maintenance is complete." : "This maintenance is cancelled.");
796 const word = { update: "Update", start: "In progress", complete: "Completed", cancel: "Cancelled" }[v.action];
797 const notified = v.notify ? await notify(this.env, this.ctx, existing.components, { heading: `${word}: ${existing.title}`, text, url: existing.url }) : null;
798 await maintenanceUpdate(this.env.DB, existing.id, state, text, v.by, notified, new Date(), {
799 action: `maintenance_${v.action === "update" ? "update" : v.action === "start" ? "started" : v.action === "complete" ? "completed" : "cancelled"}`,
800 detail: `${existing.title}: ${text}`,
801 });
802 return ok((await maintenanceById(this.env.DB, existing.id, this.origin()))!);
803 }
804
805 async audit(filter?: { before?: string | null }): Promise<StatusAuditEntry[]> {
806 const before = filter?.before && !Number.isNaN(Date.parse(filter.before)) ? filter.before : null;
807 return auditLog(this.env.DB, before);
808 }
809}
810