Skip to content
Sign in

flagon-io/g1t

public
0
958 linesCodeBlameRaw
1/**
2 * status.g1t.sh: whether each part of g1t is working, how it has done over
3 * 90 days, and what staff have said about incidents and maintenance.
4 *
5 * A Worker of its own, apart from the site, so it stays up when g1t does
6 * not. Every minute a cron checks each part over the public internet, as
7 * people reach it, and keeps the result in D1. The same run moves planned
8 * maintenance along, and drafts an incident for staff when a part keeps
9 * failing (detect.ts). Pages are drawn from what is kept, never by
10 * checking on the spot, and kept at the edge for 30 seconds.
11 *
12 * Staff run incidents from sudo, through the `StatusAdmin` entrypoint,
13 * which only a service binding reaches. Every change there is audited.
14 *
15 * GET / the page
16 * GET /status.json the same as JSON (snake_case, CORS open)
17 * GET /badge.svg a small badge
18 * GET /incidents/<id> an incident's updates and postmortem
19 * GET /maintenance/<id> a maintenance window's updates
20 * GET /history the last 12 months, by month
21 * GET /feed.xml, /feed.json every public update, newest first
22 * GET /subscribe subscribing by email
23 * POST /subscribe asks for a subscription: a confirmation email
24 * GET|POST /subscribe/confirm?token= confirms (GET shows a button: link scanners must not confirm)
25 * GET|POST /unsubscribe?token= leaves (POST also takes RFC 8058 one-click)
26 * POST /deploys the deploy tool: a deploy started or finished (bearer STATUS_DEPLOY_TOKEN)
27 */
28import { WorkerEntrypoint } from "cloudflare:workers";
29import {
30 type AdminIncident,
31 type AdminIncidentDetail,
32 type AdminMaintenance,
33 type DeclareIncident,
34 type FollowUp,
35 type IncidentChange,
36 type MaintenanceChange,
37 type NewMaintenance,
38 type Postmortem,
39 type PostmortemFields,
40 type PublishIncident,
41 type Result,
42 type RolesChange,
43 type StatusAdminApi,
44 type StatusAuditEntry,
45 type StatusBoard,
46 billingClient,
47 fail,
48 ok,
49} from "@g1t/contracts";
50import bricolage from "@g1t/theme/fonts/bricolage-grotesque-latin.woff2";
51import hanken from "@g1t/theme/fonts/hanken-grotesk-latin.woff2";
52import plexMono from "@g1t/theme/fonts/ibm-plex-mono-latin-400.woff2";
53
54import { type Targets, components } from "./components.ts";
55import {
56 autoDismissText,
57 deployChange,
58 deployQuiet,
59 detect,
60 detectedImpact,
61 draftTitle,
62 minutesWords,
63 quietUntil,
64 recoverySentence,
65 settleDrafts,
66 staleDrafts,
67 staleText,
68 troubleSentence,
69 troubledNow,
70} from "./detect.ts";
71import {
72 type EmailBinding,
73 type Sender,
74 alertLetter,
75 bindingSender,
76 confirmLetter,
77 recoveredLetter,
78 render as renderMail,
79 staleLetter,
80 unsubscribeHeaders,
81 updateLetter,
82} from "./email.ts";
83import { atom, feedItems, jsonFeed } from "./feed.ts";
84import {
85 type Entry,
86 applyChange,
87 applyRoles,
88 checkChange,
89 checkDeclare,
90 checkFollowUp,
91 checkMaintenance,
92 checkMaintenanceChange,
93 checkPostmortem,
94 checkPublish,
95 checkRoles,
96 postmortemReady,
97 SEVERITY_LABEL,
98 shortId,
99} from "./incidents.ts";
100import { INCIDENT_STATUS, type PageModel, SLOW_MS, buildPage, classify, underMaintenance } from "./model.ts";
101import { stamp } from "./postmortem.ts";
102import { type StorageReport, probe } from "./probe.ts";
103import { readZone } from "./time.ts";
104import {
105 FAVICON,
106 SCRIPT,
107 type PageOptions,
108 renderBadge,
109 renderHistory,
110 renderIncident,
111 renderMaintenance,
112 renderMessage,
113 renderPage,
114 renderSubscribe,
115} from "./render.ts";
116import {
117 type Observation,
118 addFollowUp,
119 addSystemLines,
120 auditLog,
121 autoDismiss,
122 board,
123 confirmSubscription,
124 createIncident,
125 dueMaintenance,
126 facts,
127 incidentDetail,
128 load,
129 loadDeploy,
130 loadHistory,
131 loadPublicIncident,
132 loadPublicMaintenance,
133 loadStreaks,
134 maintenanceById,
135 maintenanceUrl,
136 maintenanceUpdate,
137 openCount,
138 openRefs,
139 publishPostmortem,
140 recipients,
141 record,
142 requestSubscription,
143 saveDeploy,
144 saveHealthy,
145 saveIncident,
146 savePostmortem,
147 saveReminders,
148 saveStreaks,
149 scheduleMaintenance,
150 setFollowUp,
151 unsubscribe,
152 watchedDrafts,
153} from "./store.ts";
154import { CONFIRM_TTL_MS, RESEND_AFTER_MS, chosenParts, hashToken, newToken, normalizeEmail, readUnsubscribeToken, unsubscribeToken } from "./subscribers.ts";
155
156export interface Env extends Partial<Targets> {
157 DB: D1Database;
158 /** Billing, for reading its price book. Optional: without it, billing is not listed. */
159 BILLING?: Fetcher;
160 /** The repos service, for git storage's recent health. Optional: without it, git storage is not listed. */
161 REPOS?: Fetcher;
162 /** Where help is. */
163 SUPPORT_URL?: string;
164 /**
165 * The site's address as people's browsers reach it, for the page's links,
166 * when the checks reach it by another (self-hosted: `http://g1t:8787`
167 * inside Compose). SITE_URL when empty.
168 */
169 PUBLIC_SITE_URL?: string;
170 /** The page's share card; empty for none. */
171 OG_IMAGE?: string;
172 /** This page's own address, for links in email and feeds made outside a request. */
173 STATUS_URL?: string;
174 /** Where sudo is, for the staff alert's link. */
175 SUDO_URL?: string;
176 /** Who hears about detected drafts. Empty sends none. */
177 STATUS_ALERT_EMAIL?: string;
178 /** The From of every email. */
179 STATUS_FROM?: string;
180 /** Signs unsubscribe links (a secret). Without it, email subscriptions are off; the feeds still work. */
181 STATUS_SECRET?: string;
182 /** Cloudflare Email Sending (`send_email`). Without it, nothing is emailed. */
183 EMAIL?: EmailBinding;
184 /**
185 * The deploy tool's bearer token for `POST /deploys` (a secret). Without
186 * it, deploys are not announced and detection does not hold off for them.
187 */
188 STATUS_DEPLOY_TOKEN?: string;
189}
190
191/** How long the edge keeps a page or the JSON. */
192const CACHE_SECONDS = 30;
193/** Older than this, a visit asks for a round of checks too (a missed cron, or `wrangler dev`). */
194const BEHIND_MS = 3 * 60 * 1000;
195/** How many subscribers one update emails at most, within a Worker's limits. */
196const MAX_RECIPIENTS = 900;
197
198function parts(env: Env) {
199 return components(env, env.BILLING != null, env.REPOS != null);
200}
201
202function names(env: Env): Map<string, string> {
203 return new Map(parts(env).map((p) => [p.key, p.name]));
204}
205
206/** This page's address: the request's own, or STATUS_URL outside a request. */
207function originOf(env: Env, url?: URL): string {
208 return (url?.origin ?? env.STATUS_URL ?? "https://status.g1t.sh").replace(/\/+$/, "");
209}
210
211function sender(env: Env): Sender | null {
212 return bindingSender(env.EMAIL, env.STATUS_FROM || undefined);
213}
214
215/** Whether subscribers can sign up: a way to send, and a secret to sign their links. */
216function emailOn(env: Env): boolean {
217 return sender(env) != null && !!env.STATUS_SECRET;
218}
219
220/** Git storage's last five minutes, from the repos service (`store_health`). */
221async function storeHealth(repos: Fetcher): Promise<StorageReport> {
222 const response = await repos.fetch("https://service/rpc/store_health", {
223 method: "POST",
224 headers: { "content-type": "application/json" },
225 body: JSON.stringify({ minutes: 5 }),
226 });
227 if (!response.ok) throw new Error(`store_health failed with status ${response.status}`);
228 return (await response.json()) as StorageReport;
229}
230
231/** One round of checks, kept. Parts under maintenance are checked but not tallied. */
232export async function checkAll(env: Env, now = new Date()): Promise<Observation[]> {
233 const billing = env.BILLING;
234 const repos = env.REPOS;
235 const list = parts(env);
236 const results = await Promise.all(
237 list.map(async (info) => {
238 // A slow answer is asked again at once before it counts (probe.ts `probe`).
239 const result = await probe(
240 info.check,
241 {
242 fetch: (url, init) => fetch(url, init),
243 billing: billing ? () => billingClient(billing).prices() : null,
244 storage: repos ? () => storeHealth(repos) : null,
245 },
246 info.slowMs ?? SLOW_MS,
247 );
248 return { info, result };
249 }),
250 );
251 // Checks through a binding have no cf-ray of their own: they ran where the others did.
252 const roundColo = results.find((r) => r.result?.colo)?.result?.colo ?? null;
253 const observations = results.map(({ info, result }): Observation => {
254 const { state, detail } = classify(result, info.slowMs);
255 return {
256 component: info.key,
257 state,
258 detail,
259 latency_ms: result ? Math.round(result.ms) : null,
260 colo: result ? (result.colo ?? roundColo) : null,
261 first_ms: result?.first_ms != null ? Math.round(result.first_ms) : null,
262 };
263 });
264 const { maintenance } = await load(env.DB, now, originOf(env));
265 await record(env.DB, observations, now, underMaintenance(maintenance, now));
266 return observations;
267}
268
269// --- Email ---------------------------------------------------------------------------
270
271/**
272 * Emails confirmed subscribers who want news about `about`, in the
273 * background. Returns how many it will email, or null when email is off.
274 */
275async function notify(
276 env: Env,
277 ctx: { waitUntil(p: Promise<unknown>): void },
278 about: string[],
279 mail: { heading: string; text: string; url: string },
280): Promise<number | null> {
281 const send = sender(env);
282 const secret = env.STATUS_SECRET;
283 if (!send || !secret) return null;
284 const list = (await recipients(env.DB, about)).slice(0, MAX_RECIPIENTS);
285 const origin = originOf(env);
286 const affects = about.map((k) => names(env).get(k) ?? k);
287 ctx.waitUntil(
288 (async () => {
289 let failed = 0;
290 for (let i = 0; i < list.length; i += 6) {
291 await Promise.all(
292 list.slice(i, i + 6).map(async (r) => {
293 const link = `${origin}/unsubscribe?token=${encodeURIComponent(await unsubscribeToken(secret, r.id))}`;
294 const { text, html } = renderMail(updateLetter({ heading: mail.heading, text: mail.text, url: mail.url, affects, unsubscribe: link }));
295 await send.send({ to: r.email, subject: mail.heading, text, html, headers: unsubscribeHeaders(link) }).catch(() => void (failed += 1));
296 }),
297 );
298 }
299 console.log(JSON.stringify({ event: "status.notified", sent: list.length - failed, failed }));
300 })(),
301 );
302 return list.length;
303}
304
305// --- The cron: detection and maintenance --------------------------------------------------
306
307async function afterChecks(env: Env, ctx: { waitUntil(p: Promise<unknown>): void }, observations: Observation[], now: Date): Promise<void> {
308 const origin = originOf(env);
309 const named = names(env);
310 // Maintenance whose window opened or closed.
311 for (const m of await dueMaintenance(env.DB, now, origin)) {
312 const ended = Date.parse(m.ends_at) <= now.getTime();
313 const text = ended ? "The maintenance is complete." : "The maintenance has begun.";
314 const notified = m.notify ? await notify(env, ctx, m.components, { heading: `${ended ? "Completed" : "In progress"}: ${m.title}`, text, url: m.url }) : null;
315 await maintenanceUpdate(env.DB, m.id, ended ? "completed" : "in_progress", text, "status", notified, now, {
316 action: ended ? "maintenance_completed" : "maintenance_started",
317 detail: `${m.title} (on schedule)`,
318 });
319 }
320 // Detection. A deploy restarts services: during one, and briefly after, trouble is counted but not drafted.
321 const [streaks, open, page, deploy] = await Promise.all([loadStreaks(env.DB), openRefs(env.DB), load(env.DB, now, origin), loadDeploy(env.DB)]);
322 const quiet = deployQuiet(deploy, now);
323 const found = detect(streaks, observations, open, underMaintenance(page.maintenance, now), now, { quiet });
324 await saveStreaks(env.DB, found.streaks);
325 if (found.held.length) console.log(JSON.stringify({ event: "status.held_for_deploy", parts: found.held, deploy: deploy?.id ?? null }));
326 const name = (key: string) => named.get(key) ?? key;
327 const slowMs = (key: string) => parts(env).find((p) => p.key === key)?.slowMs ?? SLOW_MS;
328 const lines = [
329 ...found.failing.map((f) => ({ incident: f.incident, kind: "failing" as const, text: troubleSentence(name(f.key), f, stamp(f.since), slowMs(f.key)) })),
330 ...found.recovered.map((r) => ({ incident: r.incident, kind: "recovered" as const, text: recoverySentence(name(r.key), r, stamp(r.since)) })),
331 ];
332 await addSystemLines(env.DB, lines, now);
333 const sudo = (id: string) => `${(env.SUDO_URL || "https://sudo.g1t.sh").replace(/\/+$/, "")}/incidents/${id}`;
334 const alertTo = (env.STATUS_ALERT_EMAIL ?? "").trim();
335 const send = sender(env);
336 if (found.draft.length) {
337 const core = new Set(parts(env).filter((p) => p.core).map((p) => p.key));
338 const title = draftTitle(found.draft.map((d) => ({ name: name(d.key), state: d.state })));
339 const since = found.draft.map((d) => d.since).sort()[0]!;
340 const said = found.draft.map((d) => troubleSentence(name(d.key), d, stamp(d.since), slowMs(d.key)));
341 // Trouble that began in a deploy and outlasted it: say so, it is the first thing to rule out.
342 const note = deploy && deployQuiet(deploy, new Date(since)) ? `It began during a deploy (started ${stamp(deploy.started_at)}) and outlasted it.` : null;
343 const id = await createIncident(
344 env.DB,
345 {
346 title,
347 severity: found.draft.some((d) => d.state === "down" && core.has(d.key)) ? "sev2" : "sev3",
348 status: "investigating",
349 visibility: "draft",
350 source: "detected",
351 components: found.draft.map((d) => ({ key: d.key, impact: detectedImpact(d.state) })),
352 started_at: since,
353 acknowledged_at: null,
354 commander: null,
355 communications: null,
356 by: "status",
357 },
358 [
359 {
360 kind: "detected",
361 public: false,
362 status: null,
363 text: `${said.join(" ")}${note ? ` ${note}` : ""} Not on the status page until it is published.`,
364 },
365 ],
366 now,
367 { action: "incident_detected", detail: title },
368 );
369 console.warn(JSON.stringify({ event: "status.detected", id, parts: found.draft.map((d) => d.key), since }));
370 if (alertTo && send) {
371 const { text, html } = renderMail(alertLetter({ title, lines: said, link: sudo(id), ...(note ? { note } : {}) }));
372 ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${title}`, text, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
373 }
374 }
375 // Detected drafts no one picked up, whose parts have stayed healthy long enough: dismissed, with a word to staff.
376 // A part counts as healthy from its first good check; a run that is still going but answered well last time does not hold a draft up.
377 const troubled = new Set(found.streaks.filter(troubledNow).map((s) => s.component));
378 const watched = await watchedDrafts(env.DB);
379 const settled = settleDrafts(watched, troubled, now);
380 await saveHealthy(env.DB, settled.healthy);
381 const dismissed = new Set<string>();
382 for (const d of settled.dismiss) {
383 const text = autoDismissText(d.lasted_ms, stamp(d.recovered_at));
384 if (!(await autoDismiss(env.DB, d.id, d.recovered_at, text, now))) continue;
385 dismissed.add(d.id);
386 console.log(JSON.stringify({ event: "status.auto_dismissed", id: d.id, lasted_ms: d.lasted_ms }));
387 if (alertTo && send) {
388 const letter = recoveredLetter({ title: d.title, text, link: sudo(d.id) });
389 const { text: body, html } = renderMail(letter);
390 ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
391 }
392 }
393 // Drafts still waiting for someone: the alert goes out again after 45 minutes, then every 6 hours.
394 const waiting = watched.filter((d) => !dismissed.has(d.id));
395 const stale = staleDrafts(waiting, now);
396 const emailed = !!(alertTo && send);
397 await saveReminders(
398 env.DB,
399 waiting.map((d) => d.id),
400 stale.map((d) => ({ id: d.id, text: staleText(d.waiting_ms, emailed) })),
401 now,
402 );
403 for (const d of stale) {
404 console.warn(JSON.stringify({ event: "status.draft_waiting", id: d.id, waiting_ms: d.waiting_ms }));
405 if (!emailed) continue;
406 const letter = staleLetter({ title: d.title, waiting: minutesWords(d.waiting_ms), link: sudo(d.id) });
407 const { text: body, html } = renderMail(letter);
408 ctx.waitUntil(send!.send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
409 }
410}
411
412/**
413 * The deploy tool's word that a deploy started or finished:
414 * `POST /deploys` with `Authorization: Bearer <STATUS_DEPLOY_TOKEN>` and
415 * `{"phase": "started" | "finished", "id": "<run or commit>"}`. Without
416 * the secret set, there is no such address.
417 */
418async function deployHook(request: Request, env: Env): Promise<Response> {
419 const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store", ...COMMON } });
420 const token = (env.STATUS_DEPLOY_TOKEN ?? "").trim();
421 if (!token) return json({ error: { code: "not_found", message: "Not found." } }, 404);
422 const given = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "").trim();
423 if (!(await sameSecret(given, token))) return json({ error: { code: "unauthorized", message: "A valid deploy token is required." } }, 401);
424 let body: { phase?: unknown; id?: unknown } = {};
425 try {
426 body = (await request.json()) as typeof body;
427 } catch {
428 // Checked below.
429 }
430 const phase = body.phase === "started" || body.phase === "finished" ? body.phase : null;
431 if (!phase) return json({ error: { code: "invalid", message: 'phase must be "started" or "finished".' } }, 400);
432 const id = typeof body.id === "string" && body.id.trim() ? body.id.trim().slice(0, 100) : null;
433 const now = new Date();
434 const window = deployChange(await loadDeploy(env.DB), phase, id, now);
435 await saveDeploy(env.DB, window);
436 console.log(JSON.stringify({ event: `status.deploy_${phase}`, id, running: window.running }));
437 return json({ deploy: window, quiet_until: quietUntil(window) });
438}
439
440/** Compares two secrets in constant time, by their hashes. */
441async function sameSecret(a: string, b: string): Promise<boolean> {
442 const digest = async (v: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(v)));
443 const [x, y] = await Promise.all([digest(a), digest(b)]);
444 let diff = a.length === 0 ? 1 : 0;
445 for (let i = 0; i < x.length; i++) diff |= x[i]! ^ y[i]!;
446 return diff === 0;
447}
448
449// --- Pages -------------------------------------------------------------------------------
450
451async function model(env: Env, now: Date, origin: string): Promise<PageModel> {
452 const stored = await load(env.DB, now, origin);
453 return buildPage({
454 parts: parts(env),
455 current: stored.current,
456 checkedAt: stored.checkedAt,
457 days: stored.days,
458 incidents: stored.incidents,
459 maintenance: stored.maintenance,
460 now,
461 });
462}
463
464/** When this isolate last asked for a catch-up round, so a busy page asks once. */
465let caughtUpAt = 0;
466
467function catchUp(env: Env, ctx: ExecutionContext, page: PageModel, now: Date) {
468 const checked = page.report.checked_at ? Date.parse(page.report.checked_at) : 0;
469 if (now.getTime() - checked < BEHIND_MS || now.getTime() - caughtUpAt < BEHIND_MS) return;
470 caughtUpAt = now.getTime();
471 ctx.waitUntil(checkAll(env, now).catch((error) => console.error(JSON.stringify({ event: "status.catch_up_failed", error: String(error) }))));
472}
473
474const PAGE_POLICY = [
475 "default-src 'none'",
476 "script-src 'self'",
477 "style-src 'unsafe-inline'",
478 "font-src 'self'",
479 "img-src 'self' data:",
480 "base-uri 'none'",
481 "form-action 'self'",
482 "frame-ancestors 'none'",
483].join("; ");
484
485const COMMON = {
486 "x-content-type-options": "nosniff",
487 "referrer-policy": "strict-origin-when-cross-origin",
488};
489
490function edgeCache(): Cache | null {
491 return (globalThis as unknown as { caches?: { default?: Cache } }).caches?.default ?? null;
492}
493
494/**
495 * A response from the edge cache, or made and kept there. Pages that say
496 * times pass the reader's zone, and are kept once per zone.
497 */
498async function cached(request: Request, ctx: ExecutionContext, make: () => Promise<Response>, zone?: string): Promise<Response> {
499 const cache = edgeCache();
500 const url = new URL(request.url);
501 const key = new Request(`${url.origin}${url.pathname}${zone ? `?zone=${encodeURIComponent(zone)}` : ""}`, { method: "GET" });
502 if (cache) {
503 const hit = await cache.match(key).catch(() => undefined);
504 if (hit) return hit;
505 }
506 const response = await make();
507 if (cache && response.ok) ctx.waitUntil(cache.put(key, response.clone()).catch(() => undefined));
508 return response;
509}
510
511const FONTS: Record<string, ArrayBuffer> = {
512 "/fonts/hanken-grotesk.woff2": hanken,
513 "/fonts/bricolage-grotesque.woff2": bricolage,
514 "/fonts/ibm-plex-mono.woff2": plexMono,
515};
516
517function html(body: string, cacheControl: string, status = 200): Response {
518 return new Response(body, {
519 status,
520 headers: { "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "content-security-policy": PAGE_POLICY, ...COMMON },
521 });
522}
523
524async function form(request: Request): Promise<FormData> {
525 try {
526 return await request.formData();
527 } catch {
528 return new FormData();
529 }
530}
531
532/** Subscribing, confirming and leaving: the page's only writes. */
533async function subscriptions(request: Request, env: Env, ctx: ExecutionContext, url: URL, options: PageOptions): Promise<Response | null> {
534 const path = url.pathname;
535 const noStore = "no-store";
536 const message = (title: string, text: string, status = 200, f?: { action: string; fields: Record<string, string>; button: string }) =>
537 html(renderMessage({ ...options, selfUrl: `${url.origin}${path}` }, { title, text, form: f }), noStore, status);
538 const post = request.method === "POST";
539
540 if (path === "/subscribe" && post) {
541 if (!emailOn(env)) return message("Email updates are not available", "Follow the Atom or JSON feed instead.", 503);
542 const data = await form(request);
543 if (String(data.get("website") ?? "")) return message("Check your inbox", "If the address is right, a confirmation link is on its way.");
544 const email = normalizeEmail(data.get("email"));
545 if (!email) return message("That is not an email address", "Go back and check it.", 400);
546 const chosen = chosenParts(data.getAll("components").map(String), parts(env).map((p) => p.key));
547 const token = newToken();
548 const { send } = await requestSubscription(env.DB, email, chosen, await hashToken(token), new Date(), CONFIRM_TTL_MS, RESEND_AFTER_MS);
549 if (send) {
550 const link = `${url.origin}/subscribe/confirm?token=${encodeURIComponent(token)}`;
551 const { text, html: body } = renderMail(confirmLetter(link, chosen ? chosen.map((k) => names(env).get(k) ?? k) : null));
552 ctx.waitUntil(sender(env)!.send({ to: email, subject: "Confirm your subscription to g1t status", text, html: body }).catch((e) => console.error(JSON.stringify({ event: "status.confirm_failed", error: String(e) }))));
553 }
554 return message("Check your inbox", "If the address is right, a confirmation link is on its way. It works for 24 hours.");
555 }
556 if (path === "/subscribe/confirm") {
557 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
558 if (!token) return message("That link is incomplete", "Copy the whole link from the email.", 400);
559 if (!post) return message("Confirm your subscription", "One more step: confirm to start getting emails about incidents and maintenance.", 200, { action: "/subscribe/confirm", fields: { token }, button: "Confirm subscription" });
560 const done = await confirmSubscription(env.DB, await hashToken(token), new Date());
561 if (!done) return message("That link has expired", "Confirmation links work for 24 hours and once. Subscribe again for a new one.", 410);
562 return message("You are subscribed", `${done.email} will get an email when g1t posts an incident or maintenance${done.parts ? ` affecting ${done.parts.map((k) => names(env).get(k) ?? k).join(", ")}` : ""}. Every email has a link to unsubscribe.`);
563 }
564 if (path === "/unsubscribe") {
565 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
566 const id = env.STATUS_SECRET && token ? await readUnsubscribeToken(env.STATUS_SECRET, token) : null;
567 if (!id) return message("That link is not valid", "Use the unsubscribe link at the bottom of any email from g1t status.", 400);
568 if (!post) return message("Unsubscribe", "Stop getting emails from g1t status?", 200, { action: "/unsubscribe", fields: { token }, button: "Unsubscribe" });
569 await unsubscribe(env.DB, id);
570 return message("You are unsubscribed", "You will not get any more emails from g1t status. You can subscribe again at any time.");
571 }
572 return null;
573}
574
575async function handle(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
576 const url = new URL(request.url);
577 const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, "") : url.pathname;
578 if (request.method === "OPTIONS" && (path === "/status.json" || path === "/feed.json")) {
579 return new Response(null, {
580 status: 204,
581 headers: { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, HEAD", "access-control-max-age": "86400" },
582 });
583 }
584 const now = new Date();
585 const origin = originOf(env, url);
586 const site = env.PUBLIC_SITE_URL || env.SITE_URL || url.origin;
587 const options: PageOptions = {
588 siteUrl: site,
589 supportUrl: env.SUPPORT_URL || `${site}/support`,
590 ogImage: env.OG_IMAGE ?? "",
591 selfUrl: `${url.origin}${path === "/" ? "/" : path}`,
592 now,
593 email: emailOn(env),
594 zone: readZone(request.headers.get("cookie"), (request as { cf?: { timezone?: unknown } }).cf?.timezone),
595 };
596
597 if (request.method === "POST" && path === "/deploys") return deployHook(request, env);
598 if (request.method === "POST") {
599 const answer = await subscriptions(request, env, ctx, url, options);
600 return answer ?? new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD", ...COMMON } });
601 }
602 if (request.method !== "GET" && request.method !== "HEAD") {
603 return new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD, POST", ...COMMON } });
604 }
605 const fresh = (cacheSeconds = CACHE_SECONDS) => `public, max-age=${cacheSeconds}`;
606 const notFound = () => html(renderMessage(options, { title: "Not found", text: "There is nothing at this address." }), fresh(), 404);
607
608 switch (path) {
609 case "/":
610 return cached(request, ctx, async () => {
611 const page = await model(env, now, origin);
612 catchUp(env, ctx, page, now);
613 return html(renderPage(page, options), fresh());
614 }, options.zone);
615 case "/status.json":
616 return cached(request, ctx, async () => {
617 const page = await model(env, now, origin);
618 catchUp(env, ctx, page, now);
619 return Response.json(page.report, { headers: { "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON } });
620 });
621 case "/badge.svg":
622 return cached(request, ctx, async () => {
623 const page = await model(env, now, origin);
624 return new Response(renderBadge(page.report.overall.state, page.report.overall.title), {
625 headers: { "content-type": "image/svg+xml", "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON },
626 });
627 });
628 case "/history":
629 return cached(request, ctx, async () => {
630 const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 11, 1));
631 const { incidents, maintenance } = await loadHistory(env.DB, since, origin);
632 return html(renderHistory(incidents, maintenance, options), fresh());
633 }, options.zone);
634 case "/feed.xml":
635 case "/feed.json":
636 return cached(request, ctx, async () => {
637 const { incidents, maintenance } = await loadHistory(env.DB, new Date(now.getTime() - 365 * 86_400_000), origin);
638 const items = feedItems(incidents, maintenance);
639 const feed = { origin, title: "g1t status", updated: now.toISOString() };
640 const headers = { "cache-control": fresh(60), "access-control-allow-origin": "*", ...COMMON };
641 return path === "/feed.xml"
642 ? new Response(atom(items, feed), { headers: { "content-type": "application/atom+xml; charset=utf-8", ...headers } })
643 : Response.json(jsonFeed(items, feed), { headers: { "content-type": "application/feed+json; charset=utf-8", ...headers } });
644 });
645 case "/subscribe":
646 return html(renderSubscribe(options, parts(env).map(({ key, name }) => ({ key, name }))), fresh(300));
647 case "/subscribe/confirm":
648 case "/unsubscribe":
649 return (await subscriptions(request, env, ctx, url, options))!;
650 case "/status.js":
651 return new Response(SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
652 case "/favicon.svg":
653 case "/favicon.ico":
654 return new Response(FAVICON, { headers: { "content-type": "image/svg+xml", "cache-control": fresh(86400), ...COMMON } });
655 case "/robots.txt":
656 return new Response("User-agent: *\nAllow: /\nDisallow: /subscribe/confirm\nDisallow: /unsubscribe\n", {
657 headers: { "content-type": "text/plain", "cache-control": fresh(86400) },
658 });
659 }
660 const incident = /^\/incidents\/([a-z0-9-]{1,64})$/.exec(path);
661 if (incident) {
662 return cached(request, ctx, async () => {
663 const found = await loadPublicIncident(env.DB, incident[1]!, origin);
664 return found ? html(renderIncident(found.incident, found.postmortem, names(env), options), fresh()) : notFound();
665 }, options.zone);
666 }
667 const maintenance = /^\/maintenance\/([a-z0-9-]{1,64})$/.exec(path);
668 if (maintenance) {
669 return cached(request, ctx, async () => {
670 const found = await loadPublicMaintenance(env.DB, maintenance[1]!, origin);
671 return found ? html(renderMaintenance(found, names(env), options), fresh()) : notFound();
672 }, options.zone);
673 }
674 const font = FONTS[path];
675 if (font) {
676 return new Response(font, { headers: { "content-type": "font/woff2", "cache-control": "public, max-age=31536000, immutable", ...COMMON } });
677 }
678 return notFound();
679}
680
681export default {
682 async fetch(request, env, ctx) {
683 try {
684 return await handle(request, env, ctx);
685 } catch (error) {
686 console.error(JSON.stringify({ event: "status.failed", path: new URL(request.url).pathname, error: String(error) }));
687 return new Response("The status page could not be drawn. Try again in a minute.", {
688 status: 503,
689 headers: { "content-type": "text/plain; charset=utf-8", "retry-after": "60", ...COMMON },
690 });
691 }
692 },
693 async scheduled(_controller, env, ctx) {
694 const now = new Date();
695 ctx.waitUntil(
696 checkAll(env, now)
697 .then(async (observations) => {
698 const failing = observations.filter((o) => o.state === "down" || o.state === "degraded");
699 if (failing.length) console.warn(JSON.stringify({ event: "status.trouble", parts: failing }));
700 await afterChecks(env, ctx, observations, now);
701 })
702 .catch((error) => console.error(JSON.stringify({ event: "status.cron_failed", error: String(error) }))),
703 );
704 },
705} satisfies ExportedHandler<Env>;
706
707// --- Staff ---------------------------------------------------------------------------------
708
709/**
710 * Staff only: running incidents and maintenance. Reached only through a
711 * service binding (sudo's `STATUS`); status.g1t.sh's own address cannot.
712 */
713export class StatusAdmin extends WorkerEntrypoint<Env> implements StatusAdminApi {
714 private known() {
715 return parts(this.env).map((p) => p.key);
716 }
717
718 private origin() {
719 return originOf(this.env);
720 }
721
722 private async detail(id: string): Promise<AdminIncidentDetail | null> {
723 const limits = new Map(parts(this.env).map((p) => [p.key, p.slowMs ?? SLOW_MS]));
724 return incidentDetail(this.env.DB, String(id), this.origin(), names(this.env), { limits });
725 }
726
727 private async summary(id: string): Promise<AdminIncident> {
728 const { timeline: _t, followups: _f, postmortem: _p, postmortem_draft: _d, url: _u, checks: _c, ...incident } = (await this.detail(id))!;
729 return incident;
730 }
731
732 /** Emails a public update to subscribers when asked; the count to keep with it. */
733 private async announce(incident: { id: string; title: string; components: { key: string; impact: string }[] }, status: keyof typeof INCIDENT_STATUS, text: string, wanted: boolean) {
734 if (!wanted) return null;
735 const about = incident.components.filter((c) => c.impact !== "operational").map((c) => c.key);
736 return notify(this.env, this.ctx, about, {
737 heading: `${INCIDENT_STATUS[status]}: ${incident.title}`,
738 text,
739 url: `${this.origin()}/incidents/${incident.id}`,
740 });
741 }
742
743 async components(): Promise<{ key: string; name: string }[]> {
744 return parts(this.env).map(({ key, name }) => ({ key, name }));
745 }
746
747 async board(): Promise<StatusBoard> {
748 return { ...(await board(this.env.DB, new Date(), this.origin())), email: emailOn(this.env) };
749 }
750
751 async incident(id: string): Promise<AdminIncidentDetail | null> {
752 return this.detail(id);
753 }
754
755 async openCount(): Promise<number> {
756 return openCount(this.env.DB);
757 }
758
759 async declare(input: DeclareIncident): Promise<Result<AdminIncident>> {
760 const checked = checkDeclare(input, this.known());
761 if (!checked.ok) return fail("invalid", checked.error);
762 const v = checked.value;
763 const now = new Date();
764 const entries: (Entry & { notified?: number | null })[] = [
765 { kind: "declared", public: false, status: null, text: `Declared ${SEVERITY_LABEL[v.severity]}.` },
766 ];
767 if (v.commander) entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${v.commander}.` });
768 if (v.communications) entries.push({ kind: "role", public: false, status: null, text: `Communications: ${v.communications}.` });
769 const id = shortId();
770 const status = v.status ?? "investigating";
771 const notified = await this.announce({ id, title: v.title, components: v.components }, status, v.message, v.notify);
772 entries.push({ kind: "update", public: true, status, text: v.message, notified });
773 await createIncident(
774 this.env.DB,
775 {
776 title: v.title,
777 severity: v.severity,
778 status,
779 visibility: "public",
780 source: "declared",
781 components: v.components,
782 started_at: v.started_at ?? now.toISOString(),
783 acknowledged_at: now.toISOString(),
784 commander: v.commander ?? null,
785 communications: v.communications ?? null,
786 by: v.by,
787 },
788 entries,
789 now,
790 { action: "incident_declared", detail: `${SEVERITY_LABEL[v.severity]}: ${v.title}` },
791 id,
792 );
793 console.log(JSON.stringify({ event: "status.incident_declared", id, by: v.by }));
794 return ok(await this.summary(id));
795 }
796
797 async update(id: string, change: IncidentChange): Promise<Result<AdminIncident>> {
798 const checked = checkChange(change, this.known());
799 if (!checked.ok) return fail("invalid", checked.error);
800 const existing = await this.detail(id);
801 if (!existing) return fail("not_found", "No such incident.");
802 const now = new Date();
803 const applied = applyChange(facts(existing), checked.value, now, names(this.env));
804 if (!applied.ok) return fail("invalid", applied.error);
805 const { next, entries } = applied.value;
806 const update = entries.find((e) => e.kind === "update");
807 const notified = update
808 ? await this.announce({ id: existing.id, title: existing.title, components: next.components }, next.status, update.text, checked.value.notify === true)
809 : null;
810 const lines = entries.map((e) => (e === update ? { ...e, notified } : e));
811 const action = next.status === "resolved" && existing.status !== "resolved" ? "incident_resolved" : update ? "incident_update" : "incident_note";
812 await saveIncident(this.env.DB, existing.id, next, lines, now, checked.value.by, {
813 action,
814 detail: entries.map((e) => (e.kind === "update" || e.kind === "note" ? `${e.kind === "update" ? "Public" : "Note"}: ${e.text}` : e.text)).join(" ").slice(0, 500),
815 });
816 return ok(await this.summary(existing.id));
817 }
818
819 async roles(id: string, change: RolesChange): Promise<Result<AdminIncident>> {
820 const checked = checkRoles(change);
821 if (!checked.ok) return fail("invalid", checked.error);
822 const existing = await this.detail(id);
823 if (!existing) return fail("not_found", "No such incident.");
824 const now = new Date();
825 const { next, entries } = applyRoles(facts(existing), checked.value, now);
826 if (!entries.length) return ok(await this.summary(existing.id));
827 await saveIncident(this.env.DB, existing.id, next, entries, now, checked.value.by, { action: "incident_roles", detail: entries.map((e) => e.text).join(" ") });
828 return ok(await this.summary(existing.id));
829 }
830
831 async publish(id: string, input: PublishIncident): Promise<Result<AdminIncident>> {
832 const checked = checkPublish(input);
833 if (!checked.ok) return fail("invalid", checked.error);
834 const existing = await this.detail(id);
835 if (!existing) return fail("not_found", "No such incident.");
836 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be published.");
837 const now = new Date();
838 const at = now.toISOString();
839 const title = checked.value.title ?? existing.title;
840 const next = { ...facts(existing), visibility: "public" as const, acknowledged_at: existing.acknowledged_at ?? at, title, published_at: at };
841 const notified = await this.announce({ id: existing.id, title, components: existing.components }, existing.status, checked.value.message, checked.value.notify);
842 await saveIncident(
843 this.env.DB,
844 existing.id,
845 next,
846 [
847 ...(existing.acknowledged_at ? [] : [{ kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged." }]),
848 { kind: "published", public: false, status: null, text: title !== existing.title ? `Published as “${title}”.` : "Published to the status page." },
849 { kind: "update", public: true, status: existing.status, text: checked.value.message, notified },
850 ],
851 now,
852 checked.value.by,
853 { action: "incident_published", detail: title },
854 );
855 return ok(await this.summary(existing.id));
856 }
857
858 async dismiss(id: string, input: { reason: string; by: string }): Promise<Result<AdminIncident>> {
859 const existing = await this.detail(id);
860 if (!existing) return fail("not_found", "No such incident.");
861 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be dismissed; resolve a published incident instead.");
862 const by = String(input?.by ?? "").trim();
863 if (!by) return fail("invalid", "Who is making the change is missing.");
864 const reason = String(input?.reason ?? "").trim().slice(0, 500) || "No reason given.";
865 const now = new Date();
866 const at = now.toISOString();
867 const next = { ...facts(existing), visibility: "dismissed" as const, status: "resolved" as const, acknowledged_at: existing.acknowledged_at ?? at, resolved_at: at };
868 await saveIncident(this.env.DB, existing.id, next, [{ kind: "dismissed", public: false, status: null, text: `Dismissed: ${reason}` }], now, by, {
869 action: "incident_dismissed",
870 detail: `${existing.title}: ${reason}`,
871 });
872 return ok(await this.summary(existing.id));
873 }
874
875 async addFollowUp(id: string, input: { title: string; owner: string | null; by: string }): Promise<Result<FollowUp>> {
876 const checked = checkFollowUp(input);
877 if (!checked.ok) return fail("invalid", checked.error);
878 if (!(await this.detail(id))) return fail("not_found", "No such incident.");
879 return ok(await addFollowUp(this.env.DB, String(id), checked.value, new Date()));
880 }
881
882 async setFollowUp(id: string, followUp: string, input: { done: boolean; by: string }): Promise<Result<FollowUp>> {
883 const by = String(input?.by ?? "").trim();
884 if (!by) return fail("invalid", "Who is making the change is missing.");
885 const done = await setFollowUp(this.env.DB, String(id), String(followUp), input.done === true, by, new Date());
886 return done ? ok(done) : fail("not_found", "No such follow-up.");
887 }
888
889 async savePostmortem(id: string, input: PostmortemFields & { by: string }): Promise<Result<Postmortem>> {
890 const checked = checkPostmortem(input);
891 if (!checked.ok) return fail("invalid", checked.error);
892 const existing = await this.detail(id);
893 if (!existing) return fail("not_found", "No such incident.");
894 if (existing.visibility !== "public") return fail("conflict", "Only a published incident has a postmortem.");
895 const { by, ...fields } = checked.value;
896 await savePostmortem(this.env.DB, existing.id, fields, by, new Date());
897 return ok((await this.detail(existing.id))!.postmortem!);
898 }
899
900 async publishPostmortem(id: string, input: { publish: boolean; by: string }): Promise<Result<Postmortem>> {
901 const by = String(input?.by ?? "").trim();
902 if (!by) return fail("invalid", "Who is making the change is missing.");
903 const existing = await this.detail(id);
904 if (!existing) return fail("not_found", "No such incident.");
905 if (!existing.postmortem) return fail("conflict", "Save the postmortem before publishing it.");
906 if (input.publish) {
907 if (!existing.resolved_at) return fail("conflict", "Resolve the incident before publishing its postmortem.");
908 const missing = postmortemReady(existing.postmortem);
909 if (missing) return fail("invalid", missing);
910 }
911 await publishPostmortem(this.env.DB, existing.id, input.publish === true, by, new Date());
912 return ok((await this.detail(existing.id))!.postmortem!);
913 }
914
915 async scheduleMaintenance(input: NewMaintenance): Promise<Result<AdminMaintenance>> {
916 const checked = checkMaintenance(input, this.known());
917 if (!checked.ok) return fail("invalid", checked.error);
918 const v = checked.value;
919 const now = new Date();
920 const id = shortId();
921 const notified = v.notify
922 ? await notify(this.env, this.ctx, v.components, {
923 heading: `Planned maintenance: ${v.title}`,
924 text: `${v.message}\n\nWhen: ${stamp(v.starts_at)} to ${stamp(v.ends_at)}.`,
925 url: maintenanceUrl(this.origin(), id),
926 })
927 : null;
928 const made = await scheduleMaintenance(this.env.DB, v, notified, now, id);
929 return ok((await maintenanceById(this.env.DB, made, this.origin()))!);
930 }
931
932 async changeMaintenance(id: string, change: MaintenanceChange): Promise<Result<AdminMaintenance>> {
933 const checked = checkMaintenanceChange(change);
934 if (!checked.ok) return fail("invalid", checked.error);
935 const existing = await maintenanceById(this.env.DB, String(id), this.origin());
936 if (!existing) return fail("not_found", "No such maintenance.");
937 const v = checked.value;
938 if (existing.state === "completed" || existing.state === "cancelled") return fail("conflict", `This maintenance is ${existing.state}.`);
939 if (v.action === "start" && existing.state !== "scheduled") return fail("conflict", "It has already started.");
940 const state = v.action === "start" ? "in_progress" : v.action === "complete" ? "completed" : v.action === "cancel" ? "cancelled" : null;
941 const text =
942 v.message ||
943 (v.action === "start" ? "The maintenance has begun." : v.action === "complete" ? "The maintenance is complete." : "This maintenance is cancelled.");
944 const word = { update: "Update", start: "In progress", complete: "Completed", cancel: "Cancelled" }[v.action];
945 const notified = v.notify ? await notify(this.env, this.ctx, existing.components, { heading: `${word}: ${existing.title}`, text, url: existing.url }) : null;
946 await maintenanceUpdate(this.env.DB, existing.id, state, text, v.by, notified, new Date(), {
947 action: `maintenance_${v.action === "update" ? "update" : v.action === "start" ? "started" : v.action === "complete" ? "completed" : "cancelled"}`,
948 detail: `${existing.title}: ${text}`,
949 });
950 return ok((await maintenanceById(this.env.DB, existing.id, this.origin()))!);
951 }
952
953 async audit(filter?: { before?: string | null }): Promise<StatusAuditEntry[]> {
954 const before = filter?.before && !Number.isNaN(Date.parse(filter.before)) ? filter.before : null;
955 return auditLog(this.env.DB, before);
956 }
957}
958