Skip to content
Sign up

flagon-io/g1t

public
0
978 linesCodeBlameRaw
1/**
2 * status.g1t.sh: whether each part of g1t is working, how it has done over
3 * 90 days, and what staff have said about incidents and maintenance.
4 *
5 * A Worker of its own, apart from the site, so it stays up when g1t does
6 * not. Every minute a cron checks each part over the public internet, as
7 * people reach it, and keeps the result in D1. The same run moves planned
8 * maintenance along, and drafts an incident for staff when a part keeps
9 * failing (detect.ts). Pages are drawn from what is kept, never by
10 * checking on the spot, and kept at the edge for 30 seconds.
11 *
12 * Staff run incidents from sudo, through the `StatusAdmin` entrypoint,
13 * which only a service binding reaches. Every change there is audited.
14 *
15 * GET / the page
16 * GET /status.json the same as JSON (snake_case, CORS open)
17 * GET /badge.svg a small badge
18 * GET /theme.js, /status.js the page's scripts: its theme before paint; hover detail and the theme switch
19 * GET /incidents/<id> an incident's updates and postmortem
20 * GET /maintenance/<id> a maintenance window's updates
21 * GET /history the last 12 months, by month
22 * GET /feed.xml, /feed.json every public update, newest first
23 * GET /subscribe subscribing by email
24 * POST /subscribe asks for a subscription: a confirmation email
25 * GET|POST /subscribe/confirm?token= confirms (GET shows a button: link scanners must not confirm)
26 * GET|POST /unsubscribe?token= leaves (POST also takes RFC 8058 one-click)
27 * POST /deploys the deploy tool: a deploy started or finished (bearer STATUS_DEPLOY_TOKEN)
28 */
29import { WorkerEntrypoint } from "cloudflare:workers";
30import {
31 type AdminIncident,
32 type AdminIncidentDetail,
33 type AdminMaintenance,
34 type DeclareIncident,
35 type FollowUp,
36 type IncidentChange,
37 type MaintenanceChange,
38 type NewMaintenance,
39 type Postmortem,
40 type PostmortemFields,
41 type PublishIncident,
42 type Result,
43 type RolesChange,
44 type StatusAdminApi,
45 type StatusAuditEntry,
46 type StatusBoard,
47 billingClient,
48 fail,
49 ok,
50} from "@g1t/contracts";
51import { LIMIT_PERIOD_SECONDS, type RateLimitBinding, clientAddress, isLimited, secretKey } from "@g1t/contracts/rate-limits";
52import bricolage from "@g1t/theme/fonts/bricolage-grotesque-latin.woff2";
53import hanken from "@g1t/theme/fonts/hanken-grotesk-latin.woff2";
54import plexMono from "@g1t/theme/fonts/ibm-plex-mono-latin-400.woff2";
55
56import { type Targets, components } from "./components.ts";
57import {
58 autoDismissText,
59 deployChange,
60 deployQuiet,
61 detect,
62 detectedImpact,
63 draftTitle,
64 minutesWords,
65 quietUntil,
66 recoverySentence,
67 settleDrafts,
68 staleDrafts,
69 staleText,
70 troubleSentence,
71 troubledNow,
72} from "./detect.ts";
73import {
74 type EmailBinding,
75 type Sender,
76 alertLetter,
77 bindingSender,
78 confirmLetter,
79 recoveredLetter,
80 render as renderMail,
81 staleLetter,
82 unsubscribeHeaders,
83 updateLetter,
84} from "./email.ts";
85import { atom, feedItems, jsonFeed } from "./feed.ts";
86import {
87 type Entry,
88 applyChange,
89 applyRoles,
90 checkChange,
91 checkDeclare,
92 checkFollowUp,
93 checkMaintenance,
94 checkMaintenanceChange,
95 checkPostmortem,
96 checkPublish,
97 checkRoles,
98 postmortemReady,
99 SEVERITY_LABEL,
100 shortId,
101} from "./incidents.ts";
102import { INCIDENT_STATUS, type PageModel, SLOW_MS, buildPage, classify, underMaintenance } from "./model.ts";
103import { stamp } from "./postmortem.ts";
104import { type StorageReport, probe } from "./probe.ts";
105import { readZone } from "./time.ts";
106import {
107 FAVICON,
108 SCRIPT,
109 THEME_SCRIPT,
110 type PageOptions,
111 renderBadge,
112 renderHistory,
113 renderIncident,
114 renderMaintenance,
115 renderMessage,
116 renderPage,
117 renderSubscribe,
118} from "./render.ts";
119import {
120 type Observation,
121 addFollowUp,
122 addSystemLines,
123 auditLog,
124 autoDismiss,
125 board,
126 confirmSubscription,
127 createIncident,
128 dueMaintenance,
129 facts,
130 incidentDetail,
131 load,
132 loadDeploy,
133 loadHistory,
134 loadPublicIncident,
135 loadPublicMaintenance,
136 loadStreaks,
137 maintenanceById,
138 maintenanceUrl,
139 maintenanceUpdate,
140 openCount,
141 openRefs,
142 publishPostmortem,
143 recipients,
144 record,
145 requestSubscription,
146 saveDeploy,
147 saveHealthy,
148 saveIncident,
149 savePostmortem,
150 saveReminders,
151 saveStreaks,
152 scheduleMaintenance,
153 setFollowUp,
154 unsubscribe,
155 watchedDrafts,
156} from "./store.ts";
157import { CONFIRM_TTL_MS, RESEND_AFTER_MS, chosenParts, hashToken, newToken, normalizeEmail, readUnsubscribeToken, unsubscribeToken } from "./subscribers.ts";
158
159export interface Env extends Partial<Targets> {
160 DB: D1Database;
161 /** Billing, for reading its price book. Optional: without it, billing is not listed. */
162 BILLING?: Fetcher;
163 /** The repos service, for git storage's recent health. Optional: without it, git storage is not listed. */
164 REPOS?: Fetcher;
165 /** Where help is. */
166 SUPPORT_URL?: string;
167 /**
168 * The site's address as people's browsers reach it, for the page's links,
169 * when the checks reach it by another (self-hosted: `http://g1t:8787`
170 * inside Compose). SITE_URL when empty.
171 */
172 PUBLIC_SITE_URL?: string;
173 /** The page's share card; empty for none. */
174 OG_IMAGE?: string;
175 /** This page's own address, for links in email and feeds made outside a request. */
176 STATUS_URL?: string;
177 /** Where sudo is, for the staff alert's link. */
178 SUDO_URL?: string;
179 /** Who hears about detected drafts. Empty sends none. */
180 STATUS_ALERT_EMAIL?: string;
181 /** The From of every email. */
182 STATUS_FROM?: string;
183 /** Signs unsubscribe links (a secret). Without it, email subscriptions are off; the feeds still work. */
184 STATUS_SECRET?: string;
185 /** Cloudflare Email Sending (`send_email`). Without it, nothing is emailed. */
186 EMAIL?: EmailBinding;
187 /**
188 * The deploy tool's bearer token for `POST /deploys` (a secret). Without
189 * it, deploys are not announced and detection does not hold off for them.
190 */
191 STATUS_DEPLOY_TOKEN?: string;
192 /**
193 * Asking for a subscription, per client address and per email address
194 * (RATE_LIMITS in packages/contracts). Without them, only the resend
195 * window (RESEND_AFTER_MS) holds back repeated emails to one address.
196 */
197 STATUS_SUBSCRIBE_LIMIT?: RateLimitBinding;
198 STATUS_EMAIL_LIMIT?: RateLimitBinding;
199}
200
201/** How long the edge keeps a page or the JSON. */
202const CACHE_SECONDS = 30;
203/** Older than this, a visit asks for a round of checks too (a missed cron, or `wrangler dev`). */
204const BEHIND_MS = 3 * 60 * 1000;
205/** How many subscribers one update emails at most, within a Worker's limits. */
206const MAX_RECIPIENTS = 900;
207
208function parts(env: Env) {
209 return components(env, env.BILLING != null, env.REPOS != null);
210}
211
212function names(env: Env): Map<string, string> {
213 return new Map(parts(env).map((p) => [p.key, p.name]));
214}
215
216/** This page's address: the request's own, or STATUS_URL outside a request. */
217function originOf(env: Env, url?: URL): string {
218 return (url?.origin ?? env.STATUS_URL ?? "https://status.g1t.sh").replace(/\/+$/, "");
219}
220
221function sender(env: Env): Sender | null {
222 return bindingSender(env.EMAIL, env.STATUS_FROM || undefined);
223}
224
225/** Whether subscribers can sign up: a way to send, and a secret to sign their links. */
226function emailOn(env: Env): boolean {
227 return sender(env) != null && !!env.STATUS_SECRET;
228}
229
230/** Git storage's last five minutes, from the repos service (`store_health`). */
231async function storeHealth(repos: Fetcher): Promise<StorageReport> {
232 const response = await repos.fetch("https://service/rpc/store_health", {
233 method: "POST",
234 headers: { "content-type": "application/json" },
235 body: JSON.stringify({ minutes: 5 }),
236 });
237 if (!response.ok) throw new Error(`store_health failed with status ${response.status}`);
238 return (await response.json()) as StorageReport;
239}
240
241/** One round of checks, kept. Parts under maintenance are checked but not tallied. */
242export async function checkAll(env: Env, now = new Date()): Promise<Observation[]> {
243 const billing = env.BILLING;
244 const repos = env.REPOS;
245 const list = parts(env);
246 const results = await Promise.all(
247 list.map(async (info) => {
248 // A slow answer is asked again at once before it counts (probe.ts `probe`).
249 const result = await probe(
250 info.check,
251 {
252 fetch: (url, init) => fetch(url, init),
253 billing: billing ? () => billingClient(billing).prices() : null,
254 storage: repos ? () => storeHealth(repos) : null,
255 },
256 info.slowMs ?? SLOW_MS,
257 );
258 return { info, result };
259 }),
260 );
261 // Checks through a binding have no cf-ray of their own: they ran where the others did.
262 const roundColo = results.find((r) => r.result?.colo)?.result?.colo ?? null;
263 const observations = results.map(({ info, result }): Observation => {
264 const { state, detail } = classify(result, info.slowMs);
265 return {
266 component: info.key,
267 state,
268 detail,
269 latency_ms: result ? Math.round(result.ms) : null,
270 colo: result ? (result.colo ?? roundColo) : null,
271 first_ms: result?.first_ms != null ? Math.round(result.first_ms) : null,
272 };
273 });
274 const { maintenance } = await load(env.DB, now, originOf(env));
275 await record(env.DB, observations, now, underMaintenance(maintenance, now));
276 return observations;
277}
278
279// --- Email ---------------------------------------------------------------------------
280
281/**
282 * Emails confirmed subscribers who want news about `about`, in the
283 * background. Returns how many it will email, or null when email is off.
284 */
285async function notify(
286 env: Env,
287 ctx: { waitUntil(p: Promise<unknown>): void },
288 about: string[],
289 mail: { heading: string; text: string; url: string },
290): Promise<number | null> {
291 const send = sender(env);
292 const secret = env.STATUS_SECRET;
293 if (!send || !secret) return null;
294 const list = (await recipients(env.DB, about)).slice(0, MAX_RECIPIENTS);
295 const origin = originOf(env);
296 const affects = about.map((k) => names(env).get(k) ?? k);
297 ctx.waitUntil(
298 (async () => {
299 let failed = 0;
300 for (let i = 0; i < list.length; i += 6) {
301 await Promise.all(
302 list.slice(i, i + 6).map(async (r) => {
303 const link = `${origin}/unsubscribe?token=${encodeURIComponent(await unsubscribeToken(secret, r.id))}`;
304 const { text, html } = renderMail(updateLetter({ heading: mail.heading, text: mail.text, url: mail.url, affects, unsubscribe: link }));
305 await send.send({ to: r.email, subject: mail.heading, text, html, headers: unsubscribeHeaders(link) }).catch(() => void (failed += 1));
306 }),
307 );
308 }
309 console.log(JSON.stringify({ event: "status.notified", sent: list.length - failed, failed }));
310 })(),
311 );
312 return list.length;
313}
314
315// --- The cron: detection and maintenance --------------------------------------------------
316
317async function afterChecks(env: Env, ctx: { waitUntil(p: Promise<unknown>): void }, observations: Observation[], now: Date): Promise<void> {
318 const origin = originOf(env);
319 const named = names(env);
320 // Maintenance whose window opened or closed.
321 for (const m of await dueMaintenance(env.DB, now, origin)) {
322 const ended = Date.parse(m.ends_at) <= now.getTime();
323 const text = ended ? "The maintenance is complete." : "The maintenance has begun.";
324 const notified = m.notify ? await notify(env, ctx, m.components, { heading: `${ended ? "Completed" : "In progress"}: ${m.title}`, text, url: m.url }) : null;
325 await maintenanceUpdate(env.DB, m.id, ended ? "completed" : "in_progress", text, "status", notified, now, {
326 action: ended ? "maintenance_completed" : "maintenance_started",
327 detail: `${m.title} (on schedule)`,
328 });
329 }
330 // Detection. A deploy restarts services: during one, and briefly after, trouble is counted but not drafted.
331 const [streaks, open, page, deploy] = await Promise.all([loadStreaks(env.DB), openRefs(env.DB), load(env.DB, now, origin), loadDeploy(env.DB)]);
332 const quiet = deployQuiet(deploy, now);
333 const found = detect(streaks, observations, open, underMaintenance(page.maintenance, now), now, { quiet });
334 await saveStreaks(env.DB, found.streaks);
335 if (found.held.length) console.log(JSON.stringify({ event: "status.held_for_deploy", parts: found.held, deploy: deploy?.id ?? null }));
336 const name = (key: string) => named.get(key) ?? key;
337 const slowMs = (key: string) => parts(env).find((p) => p.key === key)?.slowMs ?? SLOW_MS;
338 const lines = [
339 ...found.failing.map((f) => ({ incident: f.incident, kind: "failing" as const, text: troubleSentence(name(f.key), f, stamp(f.since), slowMs(f.key)) })),
340 ...found.recovered.map((r) => ({ incident: r.incident, kind: "recovered" as const, text: recoverySentence(name(r.key), r, stamp(r.since)) })),
341 ];
342 await addSystemLines(env.DB, lines, now);
343 const sudo = (id: string) => `${(env.SUDO_URL || "https://sudo.g1t.sh").replace(/\/+$/, "")}/incidents/${id}`;
344 const alertTo = (env.STATUS_ALERT_EMAIL ?? "").trim();
345 const send = sender(env);
346 if (found.draft.length) {
347 const core = new Set(parts(env).filter((p) => p.core).map((p) => p.key));
348 const title = draftTitle(found.draft.map((d) => ({ name: name(d.key), state: d.state })));
349 const since = found.draft.map((d) => d.since).sort()[0]!;
350 const said = found.draft.map((d) => troubleSentence(name(d.key), d, stamp(d.since), slowMs(d.key)));
351 // Trouble that began in a deploy and outlasted it: say so, it is the first thing to rule out.
352 const note = deploy && deployQuiet(deploy, new Date(since)) ? `It began during a deploy (started ${stamp(deploy.started_at)}) and outlasted it.` : null;
353 const id = await createIncident(
354 env.DB,
355 {
356 title,
357 severity: found.draft.some((d) => d.state === "down" && core.has(d.key)) ? "sev2" : "sev3",
358 status: "investigating",
359 visibility: "draft",
360 source: "detected",
361 components: found.draft.map((d) => ({ key: d.key, impact: detectedImpact(d.state) })),
362 started_at: since,
363 acknowledged_at: null,
364 commander: null,
365 communications: null,
366 by: "status",
367 },
368 [
369 {
370 kind: "detected",
371 public: false,
372 status: null,
373 text: `${said.join(" ")}${note ? ` ${note}` : ""} Not on the status page until it is published.`,
374 },
375 ],
376 now,
377 { action: "incident_detected", detail: title },
378 );
379 console.warn(JSON.stringify({ event: "status.detected", id, parts: found.draft.map((d) => d.key), since }));
380 if (alertTo && send) {
381 const { text, html } = renderMail(alertLetter({ title, lines: said, link: sudo(id), ...(note ? { note } : {}) }));
382 ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${title}`, text, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
383 }
384 }
385 // Detected drafts no one picked up, whose parts have stayed healthy long enough: dismissed, with a word to staff.
386 // A part counts as healthy from its first good check; a run that is still going but answered well last time does not hold a draft up.
387 const troubled = new Set(found.streaks.filter(troubledNow).map((s) => s.component));
388 const watched = await watchedDrafts(env.DB);
389 const settled = settleDrafts(watched, troubled, now);
390 await saveHealthy(env.DB, settled.healthy);
391 const dismissed = new Set<string>();
392 for (const d of settled.dismiss) {
393 const text = autoDismissText(d.lasted_ms, stamp(d.recovered_at));
394 if (!(await autoDismiss(env.DB, d.id, d.recovered_at, text, now))) continue;
395 dismissed.add(d.id);
396 console.log(JSON.stringify({ event: "status.auto_dismissed", id: d.id, lasted_ms: d.lasted_ms }));
397 if (alertTo && send) {
398 const letter = recoveredLetter({ title: d.title, text, link: sudo(d.id) });
399 const { text: body, html } = renderMail(letter);
400 ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
401 }
402 }
403 // Drafts still waiting for someone: the alert goes out again after 45 minutes, then every 6 hours.
404 const waiting = watched.filter((d) => !dismissed.has(d.id));
405 const stale = staleDrafts(waiting, now);
406 const emailed = !!(alertTo && send);
407 await saveReminders(
408 env.DB,
409 waiting.map((d) => d.id),
410 stale.map((d) => ({ id: d.id, text: staleText(d.waiting_ms, emailed) })),
411 now,
412 );
413 for (const d of stale) {
414 console.warn(JSON.stringify({ event: "status.draft_waiting", id: d.id, waiting_ms: d.waiting_ms }));
415 if (!emailed) continue;
416 const letter = staleLetter({ title: d.title, waiting: minutesWords(d.waiting_ms), link: sudo(d.id) });
417 const { text: body, html } = renderMail(letter);
418 ctx.waitUntil(send!.send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))));
419 }
420}
421
422/**
423 * The deploy tool's word that a deploy started or finished:
424 * `POST /deploys` with `Authorization: Bearer <STATUS_DEPLOY_TOKEN>` and
425 * `{"phase": "started" | "finished", "id": "<run or commit>"}`. Without
426 * the secret set, there is no such address.
427 */
428async function deployHook(request: Request, env: Env): Promise<Response> {
429 const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store", ...COMMON } });
430 const token = (env.STATUS_DEPLOY_TOKEN ?? "").trim();
431 if (!token) return json({ error: { code: "not_found", message: "Not found." } }, 404);
432 const given = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "").trim();
433 if (!(await sameSecret(given, token))) return json({ error: { code: "unauthorized", message: "A valid deploy token is required." } }, 401);
434 let body: { phase?: unknown; id?: unknown } = {};
435 try {
436 body = (await request.json()) as typeof body;
437 } catch {
438 // Checked below.
439 }
440 const phase = body.phase === "started" || body.phase === "finished" ? body.phase : null;
441 if (!phase) return json({ error: { code: "invalid", message: 'phase must be "started" or "finished".' } }, 400);
442 const id = typeof body.id === "string" && body.id.trim() ? body.id.trim().slice(0, 100) : null;
443 const now = new Date();
444 const window = deployChange(await loadDeploy(env.DB), phase, id, now);
445 await saveDeploy(env.DB, window);
446 console.log(JSON.stringify({ event: `status.deploy_${phase}`, id, running: window.running }));
447 return json({ deploy: window, quiet_until: quietUntil(window) });
448}
449
450/** Compares two secrets in constant time, by their hashes. */
451async function sameSecret(a: string, b: string): Promise<boolean> {
452 const digest = async (v: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(v)));
453 const [x, y] = await Promise.all([digest(a), digest(b)]);
454 let diff = a.length === 0 ? 1 : 0;
455 for (let i = 0; i < x.length; i++) diff |= x[i]! ^ y[i]!;
456 return diff === 0;
457}
458
459// --- Pages -------------------------------------------------------------------------------
460
461async function model(env: Env, now: Date, origin: string): Promise<PageModel> {
462 const stored = await load(env.DB, now, origin);
463 return buildPage({
464 parts: parts(env),
465 current: stored.current,
466 checkedAt: stored.checkedAt,
467 days: stored.days,
468 incidents: stored.incidents,
469 maintenance: stored.maintenance,
470 now,
471 });
472}
473
474/** When this isolate last asked for a catch-up round, so a busy page asks once. */
475let caughtUpAt = 0;
476
477function catchUp(env: Env, ctx: ExecutionContext, page: PageModel, now: Date) {
478 const checked = page.report.checked_at ? Date.parse(page.report.checked_at) : 0;
479 if (now.getTime() - checked < BEHIND_MS || now.getTime() - caughtUpAt < BEHIND_MS) return;
480 caughtUpAt = now.getTime();
481 ctx.waitUntil(checkAll(env, now).catch((error) => console.error(JSON.stringify({ event: "status.catch_up_failed", error: String(error) }))));
482}
483
484const PAGE_POLICY = [
485 "default-src 'none'",
486 "script-src 'self'",
487 "style-src 'unsafe-inline'",
488 "font-src 'self'",
489 "img-src 'self' data:",
490 "base-uri 'none'",
491 "form-action 'self'",
492 "frame-ancestors 'none'",
493].join("; ");
494
495const COMMON = {
496 "x-content-type-options": "nosniff",
497 "referrer-policy": "strict-origin-when-cross-origin",
498};
499
500function edgeCache(): Cache | null {
501 return (globalThis as unknown as { caches?: { default?: Cache } }).caches?.default ?? null;
502}
503
504/**
505 * A response from the edge cache, or made and kept there. Pages that say
506 * times pass the reader's zone, and are kept once per zone.
507 */
508async function cached(request: Request, ctx: ExecutionContext, make: () => Promise<Response>, zone?: string): Promise<Response> {
509 const cache = edgeCache();
510 const url = new URL(request.url);
511 const key = new Request(`${url.origin}${url.pathname}${zone ? `?zone=${encodeURIComponent(zone)}` : ""}`, { method: "GET" });
512 if (cache) {
513 const hit = await cache.match(key).catch(() => undefined);
514 if (hit) return hit;
515 }
516 const response = await make();
517 if (cache && response.ok) ctx.waitUntil(cache.put(key, response.clone()).catch(() => undefined));
518 return response;
519}
520
521const FONTS: Record<string, ArrayBuffer> = {
522 "/fonts/hanken-grotesk.woff2": hanken,
523 "/fonts/bricolage-grotesque.woff2": bricolage,
524 "/fonts/ibm-plex-mono.woff2": plexMono,
525};
526
527function html(body: string, cacheControl: string, status = 200): Response {
528 return new Response(body, {
529 status,
530 headers: { "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "content-security-policy": PAGE_POLICY, ...COMMON },
531 });
532}
533
534async function form(request: Request): Promise<FormData> {
535 try {
536 return await request.formData();
537 } catch {
538 return new FormData();
539 }
540}
541
542/** Subscribing, confirming and leaving: the page's only writes. */
543async function subscriptions(request: Request, env: Env, ctx: ExecutionContext, url: URL, options: PageOptions): Promise<Response | null> {
544 const path = url.pathname;
545 const noStore = "no-store";
546 const message = (title: string, text: string, status = 200, f?: { action: string; fields: Record<string, string>; button: string }) =>
547 html(renderMessage({ ...options, selfUrl: `${url.origin}${path}` }, { title, text, form: f }), noStore, status);
548 const post = request.method === "POST";
549
550 if (path === "/subscribe" && post) {
551 if (!emailOn(env)) return message("Email updates are not available", "Follow the Atom or JSON feed instead.", 503);
552 // Each request can send an email: limited per client, then per address.
553 const tooMany = () => {
554 const answer = message("Too many requests", "Wait a minute and try again.", 429);
555 answer.headers.set("retry-after", String(LIMIT_PERIOD_SECONDS));
556 return answer;
557 };
558 if (await isLimited(env.STATUS_SUBSCRIBE_LIMIT, `ip:${clientAddress(request)}`)) return tooMany();
559 const data = await form(request);
560 if (String(data.get("website") ?? "")) return message("Check your inbox", "If the address is right, a confirmation link is on its way.");
561 const email = normalizeEmail(data.get("email"));
562 if (!email) return message("That is not an email address", "Go back and check it.", 400);
563 if (await isLimited(env.STATUS_EMAIL_LIMIT, await secretKey("email", email))) return tooMany();
564 const chosen = chosenParts(data.getAll("components").map(String), parts(env).map((p) => p.key));
565 const token = newToken();
566 const { send } = await requestSubscription(env.DB, email, chosen, await hashToken(token), new Date(), CONFIRM_TTL_MS, RESEND_AFTER_MS);
567 if (send) {
568 const link = `${url.origin}/subscribe/confirm?token=${encodeURIComponent(token)}`;
569 const { text, html: body } = renderMail(confirmLetter(link, chosen ? chosen.map((k) => names(env).get(k) ?? k) : null));
570 ctx.waitUntil(sender(env)!.send({ to: email, subject: "Confirm your subscription to g1t status", text, html: body }).catch((e) => console.error(JSON.stringify({ event: "status.confirm_failed", error: String(e) }))));
571 }
572 return message("Check your inbox", "If the address is right, a confirmation link is on its way. It works for 24 hours.");
573 }
574 if (path === "/subscribe/confirm") {
575 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
576 if (!token) return message("That link is incomplete", "Copy the whole link from the email.", 400);
577 if (!post) return message("Confirm your subscription", "One more step: confirm to start getting emails about incidents and maintenance.", 200, { action: "/subscribe/confirm", fields: { token }, button: "Confirm subscription" });
578 const done = await confirmSubscription(env.DB, await hashToken(token), new Date());
579 if (!done) return message("That link has expired", "Confirmation links work for 24 hours and once. Subscribe again for a new one.", 410);
580 return message("You are subscribed", `${done.email} will get an email when g1t posts an incident or maintenance${done.parts ? ` affecting ${done.parts.map((k) => names(env).get(k) ?? k).join(", ")}` : ""}. Every email has a link to unsubscribe.`);
581 }
582 if (path === "/unsubscribe") {
583 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
584 const id = env.STATUS_SECRET && token ? await readUnsubscribeToken(env.STATUS_SECRET, token) : null;
585 if (!id) return message("That link is not valid", "Use the unsubscribe link at the bottom of any email from g1t status.", 400);
586 if (!post) return message("Unsubscribe", "Stop getting emails from g1t status?", 200, { action: "/unsubscribe", fields: { token }, button: "Unsubscribe" });
587 await unsubscribe(env.DB, id);
588 return message("You are unsubscribed", "You will not get any more emails from g1t status. You can subscribe again at any time.");
589 }
590 return null;
591}
592
593async function handle(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
594 const url = new URL(request.url);
595 const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, "") : url.pathname;
596 if (request.method === "OPTIONS" && (path === "/status.json" || path === "/feed.json")) {
597 return new Response(null, {
598 status: 204,
599 headers: { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, HEAD", "access-control-max-age": "86400" },
600 });
601 }
602 const now = new Date();
603 const origin = originOf(env, url);
604 const site = env.PUBLIC_SITE_URL || env.SITE_URL || url.origin;
605 const options: PageOptions = {
606 siteUrl: site,
607 supportUrl: env.SUPPORT_URL || `${site}/support`,
608 ogImage: env.OG_IMAGE ?? "",
609 selfUrl: `${url.origin}${path === "/" ? "/" : path}`,
610 now,
611 email: emailOn(env),
612 zone: readZone(request.headers.get("cookie"), (request as { cf?: { timezone?: unknown } }).cf?.timezone),
613 };
614
615 if (request.method === "POST" && path === "/deploys") return deployHook(request, env);
616 if (request.method === "POST") {
617 const answer = await subscriptions(request, env, ctx, url, options);
618 return answer ?? new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD", ...COMMON } });
619 }
620 if (request.method !== "GET" && request.method !== "HEAD") {
621 return new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD, POST", ...COMMON } });
622 }
623 const fresh = (cacheSeconds = CACHE_SECONDS) => `public, max-age=${cacheSeconds}`;
624 const notFound = () => html(renderMessage(options, { title: "Not found", text: "There is nothing at this address." }), fresh(), 404);
625
626 switch (path) {
627 case "/":
628 return cached(request, ctx, async () => {
629 const page = await model(env, now, origin);
630 catchUp(env, ctx, page, now);
631 return html(renderPage(page, options), fresh());
632 }, options.zone);
633 case "/status.json":
634 return cached(request, ctx, async () => {
635 const page = await model(env, now, origin);
636 catchUp(env, ctx, page, now);
637 return Response.json(page.report, { headers: { "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON } });
638 });
639 case "/badge.svg":
640 return cached(request, ctx, async () => {
641 const page = await model(env, now, origin);
642 return new Response(renderBadge(page.report.overall.state, page.report.overall.title), {
643 headers: { "content-type": "image/svg+xml", "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON },
644 });
645 });
646 case "/history":
647 return cached(request, ctx, async () => {
648 const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 11, 1));
649 const { incidents, maintenance } = await loadHistory(env.DB, since, origin);
650 return html(renderHistory(incidents, maintenance, options), fresh());
651 }, options.zone);
652 case "/feed.xml":
653 case "/feed.json":
654 return cached(request, ctx, async () => {
655 const { incidents, maintenance } = await loadHistory(env.DB, new Date(now.getTime() - 365 * 86_400_000), origin);
656 const items = feedItems(incidents, maintenance);
657 const feed = { origin, title: "g1t status", updated: now.toISOString() };
658 const headers = { "cache-control": fresh(60), "access-control-allow-origin": "*", ...COMMON };
659 return path === "/feed.xml"
660 ? new Response(atom(items, feed), { headers: { "content-type": "application/atom+xml; charset=utf-8", ...headers } })
661 : Response.json(jsonFeed(items, feed), { headers: { "content-type": "application/feed+json; charset=utf-8", ...headers } });
662 });
663 case "/subscribe":
664 return html(renderSubscribe(options, parts(env).map(({ key, name }) => ({ key, name }))), fresh(300));
665 case "/subscribe/confirm":
666 case "/unsubscribe":
667 return (await subscriptions(request, env, ctx, url, options))!;
668 case "/theme.js":
669 return new Response(THEME_SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
670 case "/status.js":
671 return new Response(SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
672 case "/favicon.svg":
673 case "/favicon.ico":
674 return new Response(FAVICON, { headers: { "content-type": "image/svg+xml", "cache-control": fresh(86400), ...COMMON } });
675 case "/robots.txt":
676 return new Response("User-agent: *\nAllow: /\nDisallow: /subscribe/confirm\nDisallow: /unsubscribe\n", {
677 headers: { "content-type": "text/plain", "cache-control": fresh(86400) },
678 });
679 }
680 const incident = /^\/incidents\/([a-z0-9-]{1,64})$/.exec(path);
681 if (incident) {
682 return cached(request, ctx, async () => {
683 const found = await loadPublicIncident(env.DB, incident[1]!, origin);
684 return found ? html(renderIncident(found.incident, found.postmortem, names(env), options), fresh()) : notFound();
685 }, options.zone);
686 }
687 const maintenance = /^\/maintenance\/([a-z0-9-]{1,64})$/.exec(path);
688 if (maintenance) {
689 return cached(request, ctx, async () => {
690 const found = await loadPublicMaintenance(env.DB, maintenance[1]!, origin);
691 return found ? html(renderMaintenance(found, names(env), options), fresh()) : notFound();
692 }, options.zone);
693 }
694 const font = FONTS[path];
695 if (font) {
696 return new Response(font, { headers: { "content-type": "font/woff2", "cache-control": "public, max-age=31536000, immutable", ...COMMON } });
697 }
698 return notFound();
699}
700
701export default {
702 async fetch(request, env, ctx) {
703 try {
704 return await handle(request, env, ctx);
705 } catch (error) {
706 console.error(JSON.stringify({ event: "status.failed", path: new URL(request.url).pathname, error: String(error) }));
707 return new Response("The status page could not be drawn. Try again in a minute.", {
708 status: 503,
709 headers: { "content-type": "text/plain; charset=utf-8", "retry-after": "60", ...COMMON },
710 });
711 }
712 },
713 async scheduled(_controller, env, ctx) {
714 const now = new Date();
715 ctx.waitUntil(
716 checkAll(env, now)
717 .then(async (observations) => {
718 const failing = observations.filter((o) => o.state === "down" || o.state === "degraded");
719 if (failing.length) console.warn(JSON.stringify({ event: "status.trouble", parts: failing }));
720 await afterChecks(env, ctx, observations, now);
721 })
722 .catch((error) => console.error(JSON.stringify({ event: "status.cron_failed", error: String(error) }))),
723 );
724 },
725} satisfies ExportedHandler<Env>;
726
727// --- Staff ---------------------------------------------------------------------------------
728
729/**
730 * Staff only: running incidents and maintenance. Reached only through a
731 * service binding (sudo's `STATUS`); status.g1t.sh's own address cannot.
732 */
733export class StatusAdmin extends WorkerEntrypoint<Env> implements StatusAdminApi {
734 private known() {
735 return parts(this.env).map((p) => p.key);
736 }
737
738 private origin() {
739 return originOf(this.env);
740 }
741
742 private async detail(id: string): Promise<AdminIncidentDetail | null> {
743 const limits = new Map(parts(this.env).map((p) => [p.key, p.slowMs ?? SLOW_MS]));
744 return incidentDetail(this.env.DB, String(id), this.origin(), names(this.env), { limits });
745 }
746
747 private async summary(id: string): Promise<AdminIncident> {
748 const { timeline: _t, followups: _f, postmortem: _p, postmortem_draft: _d, url: _u, checks: _c, ...incident } = (await this.detail(id))!;
749 return incident;
750 }
751
752 /** Emails a public update to subscribers when asked; the count to keep with it. */
753 private async announce(incident: { id: string; title: string; components: { key: string; impact: string }[] }, status: keyof typeof INCIDENT_STATUS, text: string, wanted: boolean) {
754 if (!wanted) return null;
755 const about = incident.components.filter((c) => c.impact !== "operational").map((c) => c.key);
756 return notify(this.env, this.ctx, about, {
757 heading: `${INCIDENT_STATUS[status]}: ${incident.title}`,
758 text,
759 url: `${this.origin()}/incidents/${incident.id}`,
760 });
761 }
762
763 async components(): Promise<{ key: string; name: string }[]> {
764 return parts(this.env).map(({ key, name }) => ({ key, name }));
765 }
766
767 async board(): Promise<StatusBoard> {
768 return { ...(await board(this.env.DB, new Date(), this.origin())), email: emailOn(this.env) };
769 }
770
771 async incident(id: string): Promise<AdminIncidentDetail | null> {
772 return this.detail(id);
773 }
774
775 async openCount(): Promise<number> {
776 return openCount(this.env.DB);
777 }
778
779 async declare(input: DeclareIncident): Promise<Result<AdminIncident>> {
780 const checked = checkDeclare(input, this.known());
781 if (!checked.ok) return fail("invalid", checked.error);
782 const v = checked.value;
783 const now = new Date();
784 const entries: (Entry & { notified?: number | null })[] = [
785 { kind: "declared", public: false, status: null, text: `Declared ${SEVERITY_LABEL[v.severity]}.` },
786 ];
787 if (v.commander) entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${v.commander}.` });
788 if (v.communications) entries.push({ kind: "role", public: false, status: null, text: `Communications: ${v.communications}.` });
789 const id = shortId();
790 const status = v.status ?? "investigating";
791 const notified = await this.announce({ id, title: v.title, components: v.components }, status, v.message, v.notify);
792 entries.push({ kind: "update", public: true, status, text: v.message, notified });
793 await createIncident(
794 this.env.DB,
795 {
796 title: v.title,
797 severity: v.severity,
798 status,
799 visibility: "public",
800 source: "declared",
801 components: v.components,
802 started_at: v.started_at ?? now.toISOString(),
803 acknowledged_at: now.toISOString(),
804 commander: v.commander ?? null,
805 communications: v.communications ?? null,
806 by: v.by,
807 },
808 entries,
809 now,
810 { action: "incident_declared", detail: `${SEVERITY_LABEL[v.severity]}: ${v.title}` },
811 id,
812 );
813 console.log(JSON.stringify({ event: "status.incident_declared", id, by: v.by }));
814 return ok(await this.summary(id));
815 }
816
817 async update(id: string, change: IncidentChange): Promise<Result<AdminIncident>> {
818 const checked = checkChange(change, this.known());
819 if (!checked.ok) return fail("invalid", checked.error);
820 const existing = await this.detail(id);
821 if (!existing) return fail("not_found", "No such incident.");
822 const now = new Date();
823 const applied = applyChange(facts(existing), checked.value, now, names(this.env));
824 if (!applied.ok) return fail("invalid", applied.error);
825 const { next, entries } = applied.value;
826 const update = entries.find((e) => e.kind === "update");
827 const notified = update
828 ? await this.announce({ id: existing.id, title: existing.title, components: next.components }, next.status, update.text, checked.value.notify === true)
829 : null;
830 const lines = entries.map((e) => (e === update ? { ...e, notified } : e));
831 const action = next.status === "resolved" && existing.status !== "resolved" ? "incident_resolved" : update ? "incident_update" : "incident_note";
832 await saveIncident(this.env.DB, existing.id, next, lines, now, checked.value.by, {
833 action,
834 detail: entries.map((e) => (e.kind === "update" || e.kind === "note" ? `${e.kind === "update" ? "Public" : "Note"}: ${e.text}` : e.text)).join(" ").slice(0, 500),
835 });
836 return ok(await this.summary(existing.id));
837 }
838
839 async roles(id: string, change: RolesChange): Promise<Result<AdminIncident>> {
840 const checked = checkRoles(change);
841 if (!checked.ok) return fail("invalid", checked.error);
842 const existing = await this.detail(id);
843 if (!existing) return fail("not_found", "No such incident.");
844 const now = new Date();
845 const { next, entries } = applyRoles(facts(existing), checked.value, now);
846 if (!entries.length) return ok(await this.summary(existing.id));
847 await saveIncident(this.env.DB, existing.id, next, entries, now, checked.value.by, { action: "incident_roles", detail: entries.map((e) => e.text).join(" ") });
848 return ok(await this.summary(existing.id));
849 }
850
851 async publish(id: string, input: PublishIncident): Promise<Result<AdminIncident>> {
852 const checked = checkPublish(input);
853 if (!checked.ok) return fail("invalid", checked.error);
854 const existing = await this.detail(id);
855 if (!existing) return fail("not_found", "No such incident.");
856 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be published.");
857 const now = new Date();
858 const at = now.toISOString();
859 const title = checked.value.title ?? existing.title;
860 const next = { ...facts(existing), visibility: "public" as const, acknowledged_at: existing.acknowledged_at ?? at, title, published_at: at };
861 const notified = await this.announce({ id: existing.id, title, components: existing.components }, existing.status, checked.value.message, checked.value.notify);
862 await saveIncident(
863 this.env.DB,
864 existing.id,
865 next,
866 [
867 ...(existing.acknowledged_at ? [] : [{ kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged." }]),
868 { kind: "published", public: false, status: null, text: title !== existing.title ? `Published as “${title}”.` : "Published to the status page." },
869 { kind: "update", public: true, status: existing.status, text: checked.value.message, notified },
870 ],
871 now,
872 checked.value.by,
873 { action: "incident_published", detail: title },
874 );
875 return ok(await this.summary(existing.id));
876 }
877
878 async dismiss(id: string, input: { reason: string; by: string }): Promise<Result<AdminIncident>> {
879 const existing = await this.detail(id);
880 if (!existing) return fail("not_found", "No such incident.");
881 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be dismissed; resolve a published incident instead.");
882 const by = String(input?.by ?? "").trim();
883 if (!by) return fail("invalid", "Who is making the change is missing.");
884 const reason = String(input?.reason ?? "").trim().slice(0, 500) || "No reason given.";
885 const now = new Date();
886 const at = now.toISOString();
887 const next = { ...facts(existing), visibility: "dismissed" as const, status: "resolved" as const, acknowledged_at: existing.acknowledged_at ?? at, resolved_at: at };
888 await saveIncident(this.env.DB, existing.id, next, [{ kind: "dismissed", public: false, status: null, text: `Dismissed: ${reason}` }], now, by, {
889 action: "incident_dismissed",
890 detail: `${existing.title}: ${reason}`,
891 });
892 return ok(await this.summary(existing.id));
893 }
894
895 async addFollowUp(id: string, input: { title: string; owner: string | null; by: string }): Promise<Result<FollowUp>> {
896 const checked = checkFollowUp(input);
897 if (!checked.ok) return fail("invalid", checked.error);
898 if (!(await this.detail(id))) return fail("not_found", "No such incident.");
899 return ok(await addFollowUp(this.env.DB, String(id), checked.value, new Date()));
900 }
901
902 async setFollowUp(id: string, followUp: string, input: { done: boolean; by: string }): Promise<Result<FollowUp>> {
903 const by = String(input?.by ?? "").trim();
904 if (!by) return fail("invalid", "Who is making the change is missing.");
905 const done = await setFollowUp(this.env.DB, String(id), String(followUp), input.done === true, by, new Date());
906 return done ? ok(done) : fail("not_found", "No such follow-up.");
907 }
908
909 async savePostmortem(id: string, input: PostmortemFields & { by: string }): Promise<Result<Postmortem>> {
910 const checked = checkPostmortem(input);
911 if (!checked.ok) return fail("invalid", checked.error);
912 const existing = await this.detail(id);
913 if (!existing) return fail("not_found", "No such incident.");
914 if (existing.visibility !== "public") return fail("conflict", "Only a published incident has a postmortem.");
915 const { by, ...fields } = checked.value;
916 await savePostmortem(this.env.DB, existing.id, fields, by, new Date());
917 return ok((await this.detail(existing.id))!.postmortem!);
918 }
919
920 async publishPostmortem(id: string, input: { publish: boolean; by: string }): Promise<Result<Postmortem>> {
921 const by = String(input?.by ?? "").trim();
922 if (!by) return fail("invalid", "Who is making the change is missing.");
923 const existing = await this.detail(id);
924 if (!existing) return fail("not_found", "No such incident.");
925 if (!existing.postmortem) return fail("conflict", "Save the postmortem before publishing it.");
926 if (input.publish) {
927 if (!existing.resolved_at) return fail("conflict", "Resolve the incident before publishing its postmortem.");
928 const missing = postmortemReady(existing.postmortem);
929 if (missing) return fail("invalid", missing);
930 }
931 await publishPostmortem(this.env.DB, existing.id, input.publish === true, by, new Date());
932 return ok((await this.detail(existing.id))!.postmortem!);
933 }
934
935 async scheduleMaintenance(input: NewMaintenance): Promise<Result<AdminMaintenance>> {
936 const checked = checkMaintenance(input, this.known());
937 if (!checked.ok) return fail("invalid", checked.error);
938 const v = checked.value;
939 const now = new Date();
940 const id = shortId();
941 const notified = v.notify
942 ? await notify(this.env, this.ctx, v.components, {
943 heading: `Planned maintenance: ${v.title}`,
944 text: `${v.message}\n\nWhen: ${stamp(v.starts_at)} to ${stamp(v.ends_at)}.`,
945 url: maintenanceUrl(this.origin(), id),
946 })
947 : null;
948 const made = await scheduleMaintenance(this.env.DB, v, notified, now, id);
949 return ok((await maintenanceById(this.env.DB, made, this.origin()))!);
950 }
951
952 async changeMaintenance(id: string, change: MaintenanceChange): Promise<Result<AdminMaintenance>> {
953 const checked = checkMaintenanceChange(change);
954 if (!checked.ok) return fail("invalid", checked.error);
955 const existing = await maintenanceById(this.env.DB, String(id), this.origin());
956 if (!existing) return fail("not_found", "No such maintenance.");
957 const v = checked.value;
958 if (existing.state === "completed" || existing.state === "cancelled") return fail("conflict", `This maintenance is ${existing.state}.`);
959 if (v.action === "start" && existing.state !== "scheduled") return fail("conflict", "It has already started.");
960 const state = v.action === "start" ? "in_progress" : v.action === "complete" ? "completed" : v.action === "cancel" ? "cancelled" : null;
961 const text =
962 v.message ||
963 (v.action === "start" ? "The maintenance has begun." : v.action === "complete" ? "The maintenance is complete." : "This maintenance is cancelled.");
964 const word = { update: "Update", start: "In progress", complete: "Completed", cancel: "Cancelled" }[v.action];
965 const notified = v.notify ? await notify(this.env, this.ctx, existing.components, { heading: `${word}: ${existing.title}`, text, url: existing.url }) : null;
966 await maintenanceUpdate(this.env.DB, existing.id, state, text, v.by, notified, new Date(), {
967 action: `maintenance_${v.action === "update" ? "update" : v.action === "start" ? "started" : v.action === "complete" ? "completed" : "cancelled"}`,
968 detail: `${existing.title}: ${text}`,
969 });
970 return ok((await maintenanceById(this.env.DB, existing.id, this.origin()))!);
971 }
972
973 async audit(filter?: { before?: string | null }): Promise<StatusAuditEntry[]> {
974 const before = filter?.before && !Number.isNaN(Date.parse(filter.before)) ? filter.before : null;
975 return auditLog(this.env.DB, before);
976 }
977}
978