Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
1,063 linesCodeBlameRaw
1/**
2 * status.g1t.sh: whether each part of g1t is working, how it has done over
3 * 90 days, and what staff have said about incidents and maintenance.
4 *
5 * A Worker of its own, apart from the site, so it stays up when g1t does
6 * not. Every minute a cron checks each part over the public internet, as
7 * people reach it, and keeps the result in D1. The same run moves planned
8 * maintenance along, and drafts an incident for staff when a part keeps
9 * failing (detect.ts). Pages are drawn from what is kept, never by
10 * checking on the spot, and kept at the edge for 30 seconds.
11 *
12 * Staff run incidents from sudo, through the `StatusAdmin` entrypoint,
13 * which only a service binding reaches. Every change there is audited.
14 *
15 * GET / the page
16 * GET /status.json the same as JSON (snake_case, CORS open)
17 * GET /badge.svg a small badge
18 * GET /theme.js, /status.js the page's scripts: its theme before paint; hover detail and the theme switch
19 * GET /incidents/<id> an incident's updates and postmortem
20 * GET /maintenance/<id> a maintenance window's updates
21 * GET /history the last 12 months, by month
22 * GET /feed.xml, /feed.json every public update, newest first
23 * GET /subscribe subscribing by email
24 * POST /subscribe asks for a subscription: a confirmation email
25 * GET|POST /subscribe/confirm?token= confirms (GET shows a button: link scanners must not confirm)
26 * GET|POST /unsubscribe?token= leaves (POST also takes RFC 8058 one-click)
27 * POST /deploys the deploy tool: a deploy started or finished (bearer STATUS_DEPLOY_TOKEN)
28 */
29import { WorkerEntrypoint } from "cloudflare:workers";
30import {
31 type AdminIncident,
32 type AdminIncidentDetail,
33 type AdminMaintenance,
34 type DeclareIncident,
35 type FollowUp,
36 type IncidentChange,
37 type MaintenanceChange,
38 type NewMaintenance,
39 type Postmortem,
40 type PostmortemFields,
41 type PublishIncident,
42 type Result,
43 type RolesChange,
44 type StatusAdminApi,
45 type StatusAuditEntry,
46 type StatusBoard,
47 billingClient,
48 fail,
49 ok,
50} from "@g1t/contracts";
51import { LIMIT_PERIOD_SECONDS, type RateLimitBinding, clientAddress, isLimited, secretKey } from "@g1t/contracts/rate-limits";
52import bricolage from "@g1t/theme/fonts/bricolage-grotesque-latin.woff2";
53import hanken from "@g1t/theme/fonts/hanken-grotesk-latin.woff2";
54import plexMono from "@g1t/theme/fonts/ibm-plex-mono-latin-400.woff2";
55
56import { type Targets, components } from "./components.ts";
57import {
58 autoDismissText,
59 deployChange,
60 deployQuiet,
61 detect,
62 detectedImpact,
63 draftTitle,
64 minutesWords,
65 quietUntil,
66 recoverySentence,
67 settleDrafts,
68 staleDrafts,
69 staleText,
70 troubleSentence,
71 troubledNow,
72} from "./detect.ts";
73import {
74 type EmailBinding,
75 type Sender,
76 alertLetter,
77 bindingSender,
78 confirmLetter,
79 recoveredLetter,
80 render as renderMail,
81 staleLetter,
82 unsubscribeHeaders,
83 updateLetter,
84} from "./email.ts";
85import { atom, feedItems, jsonFeed } from "./feed.ts";
86import {
87 type Entry,
88 applyChange,
89 applyRoles,
90 checkChange,
91 checkDeclare,
92 checkFollowUp,
93 checkMaintenance,
94 checkMaintenanceChange,
95 checkPostmortem,
96 checkPublish,
97 checkRoles,
98 postmortemReady,
99 SEVERITY_LABEL,
100 shortId,
101} from "./incidents.ts";
102import { INCIDENT_STATUS, type PageModel, SLOW_MS, buildPage, classify, underMaintenance } from "./model.ts";
103import { stamp } from "./postmortem.ts";
104import { type StorageReport, probe } from "./probe.ts";
105import { readZone } from "./time.ts";
106import {
107 FAVICON,
108 SCRIPT,
109 THEME_SCRIPT,
110 type PageOptions,
111 renderBadge,
112 renderHistory,
113 renderIncident,
114 renderMaintenance,
115 renderMessage,
116 renderPage,
117 renderSubscribe,
118} from "./render.ts";
119import {
120 type Observation,
121 addFollowUp,
122 addSystemLines,
123 auditLog,
124 autoDismiss,
125 board,
126 confirmSubscription,
127 createIncident,
128 dueMaintenance,
129 facts,
130 incidentDetail,
131 load,
132 loadDeploy,
133 loadHistory,
134 loadPublicIncident,
135 loadPublicMaintenance,
136 loadStreaks,
137 maintenanceById,
138 maintenanceUrl,
139 maintenanceUpdate,
140 openCount,
141 openRefs,
142 publishPostmortem,
143 recipients,
144 record,
145 requestSubscription,
146 saveDeploy,
147 saveHealthy,
148 saveIncident,
149 savePostmortem,
150 saveReminders,
151 saveStreaks,
152 scheduleMaintenance,
153 setFollowUp,
154 unsubscribe,
155 watchedDrafts,
156} from "./store.ts";
157import { CONFIRM_TTL_MS, RESEND_AFTER_MS, chosenParts, hashToken, newToken, normalizeEmail, readUnsubscribeToken, unsubscribeToken } from "./subscribers.ts";
158
159export interface Env extends Partial<Targets> {
160 DB: D1Database;
161 /** Billing, for reading its price book. Optional: without it, billing is not listed. */
162 BILLING?: Fetcher;
163 /** The repos service, for git storage's recent health. Optional: without it, git storage is not listed. */
164 REPOS?: Fetcher;
165 /** Where help is. */
166 SUPPORT_URL?: string;
167 /**
168 * The site's address as people's browsers reach it, for the page's links,
169 * when the checks reach it by another (self-hosted: `http://g1t:8787`
170 * inside Compose). SITE_URL when empty.
171 */
172 PUBLIC_SITE_URL?: string;
173 /** The page's share card; empty for none. */
174 OG_IMAGE?: string;
175 /** This page's own address, for links in email and feeds made outside a request. */
176 STATUS_URL?: string;
177 /** Where sudo is, for the staff alert's link. */
178 SUDO_URL?: string;
179 /** Who hears about detected drafts. Empty sends none. */
180 STATUS_ALERT_EMAIL?: string;
181 /** The From of every email. */
182 STATUS_FROM?: string;
183 /** Signs unsubscribe links (a secret). Without it, email subscriptions are off; the feeds still work. */
184 STATUS_SECRET?: string;
185 /** Cloudflare Email Sending (`send_email`). Without it, nothing is emailed. */
186 EMAIL?: EmailBinding;
187 /**
188 * The deploy tool's bearer token for `POST /deploys` (a secret). Without
189 * it, deploys are not announced and detection does not hold off for them.
190 */
191 STATUS_DEPLOY_TOKEN?: string;
192 /**
193 * Asking for a subscription, per client address and per email address
194 * (RATE_LIMITS in packages/contracts). Without them, only the resend
195 * window (RESEND_AFTER_MS) holds back repeated emails to one address.
196 */
197 STATUS_SUBSCRIBE_LIMIT?: RateLimitBinding;
198 STATUS_EMAIL_LIMIT?: RateLimitBinding;
199}
200
201/** How long the edge keeps a page or the JSON. */
202const CACHE_SECONDS = 30;
203/** Older than this, a visit asks for a round of checks too (a missed cron, or `wrangler dev`). */
204const BEHIND_MS = 3 * 60 * 1000;
205/** How many subscribers one update emails at most, within a Worker's limits. */
206const MAX_RECIPIENTS = 900;
207
208function parts(env: Env) {
209 return components(env, env.BILLING != null, env.REPOS != null);
210}
211
212function names(env: Env): Map<string, string> {
213 return new Map(parts(env).map((p) => [p.key, p.name]));
214}
215
216/** This page's address: the request's own, or STATUS_URL outside a request. */
217function originOf(env: Env, url?: URL): string {
218 return (url?.origin ?? env.STATUS_URL ?? "https://status.g1t.sh").replace(/\/+$/, "");
219}
220
221function sender(env: Env): Sender | null {
222 return bindingSender(env.EMAIL, env.STATUS_FROM || undefined);
223}
224
225/** Whether subscribers can sign up: a way to send, and a secret to sign their links. */
226function emailOn(env: Env): boolean {
227 return sender(env) != null && !!env.STATUS_SECRET;
228}
229
230/** Git storage's last five minutes, from the repos service (`store_health`). */
231async function storeHealth(repos: Fetcher): Promise<StorageReport> {
232 const response = await repos.fetch("https://service/rpc/store_health", {
233 method: "POST",
234 headers: { "content-type": "application/json" },
235 body: JSON.stringify({ minutes: 5 }),
236 });
237 if (!response.ok) throw new Error(`store_health failed with status ${response.status}`);
238 return (await response.json()) as StorageReport;
239}
240
241/** One round of checks, kept. Parts under maintenance are checked but not tallied. */
242export async function checkAll(env: Env, now = new Date()): Promise<Observation[]> {
243 const billing = env.BILLING;
244 const repos = env.REPOS;
245 const list = parts(env);
246 const results = await Promise.all(
247 list.map(async (info) => {
248 // A slow answer is asked again at once before it counts (probe.ts `probe`).
249 const result = await probe(
250 info.check,
251 {
252 fetch: (url, init) => fetch(url, init),
253 billing: billing ? () => billingClient(billing).prices() : null,
254 storage: repos ? () => storeHealth(repos) : null,
255 },
256 info.slowMs ?? SLOW_MS,
257 );
258 return { info, result };
259 }),
260 );
261 // Checks through a binding have no cf-ray of their own: they ran where the others did.
262 const roundColo = results.find((r) => r.result?.colo)?.result?.colo ?? null;
263 const observations = results.map(({ info, result }): Observation => {
264 const { state, detail } = classify(result, info.slowMs);
265 return {
266 component: info.key,
267 state,
268 detail,
269 latency_ms: result ? Math.round(result.ms) : null,
270 colo: result ? (result.colo ?? roundColo) : null,
271 first_ms: result?.first_ms != null ? Math.round(result.first_ms) : null,
272 };
273 });
274 const { maintenance } = await load(env.DB, now, originOf(env));
275 await record(env.DB, observations, now, underMaintenance(maintenance, now));
276 return observations;
277}
278
279// --- Email ---------------------------------------------------------------------------
280
281/**
282 * Emails confirmed subscribers who want news about `about`, in the
283 * background. Returns how many it will email, or null when email is off.
284 */
285async function notify(
286 env: Env,
287 ctx: { waitUntil(p: Promise<unknown>): void },
288 about: string[],
289 mail: { heading: string; text: string; url: string },
290): Promise<number | null> {
291 const send = sender(env);
292 const secret = env.STATUS_SECRET;
293 if (!send || !secret) return null;
294 const list = (await recipients(env.DB, about)).slice(0, MAX_RECIPIENTS);
295 const origin = originOf(env);
296 const affects = about.map((k) => names(env).get(k) ?? k);
297 ctx.waitUntil(
298 (async () => {
299 let failed = 0;
300 for (let i = 0; i < list.length; i += 6) {
301 await Promise.all(
302 list.slice(i, i + 6).map(async (r) => {
303 const link = `${origin}/unsubscribe?token=${encodeURIComponent(await unsubscribeToken(secret, r.id))}`;
304 const { text, html } = renderMail(updateLetter({ heading: mail.heading, text: mail.text, url: mail.url, affects, unsubscribe: link }));
305 await send.send({ to: r.email, subject: mail.heading, text, html, headers: unsubscribeHeaders(link) }).catch(() => void (failed += 1));
306 }),
307 );
308 }
309 console.log(JSON.stringify({ event: "status.notified", sent: list.length - failed, failed }));
310 })(),
311 );
312 return list.length;
313}
314
315// --- The cron: detection and maintenance --------------------------------------------------
316
317async function afterChecks(env: Env, ctx: { waitUntil(p: Promise<unknown>): void }, observations: Observation[], now: Date): Promise<void> {
318 const origin = originOf(env);
319 const named = names(env);
320 // Maintenance whose window opened or closed.
321 for (const m of await dueMaintenance(env.DB, now, origin)) {
322 const ended = Date.parse(m.ends_at) <= now.getTime();
323 const text = ended ? "The maintenance is complete." : "The maintenance has begun.";
324 const notified = m.notify
325 ? await notify(env, ctx, m.components, { heading: `${ended ? "Completed" : "In progress"}: ${m.title}`, text, url: m.url })
326 : null;
327 await maintenanceUpdate(env.DB, m.id, ended ? "completed" : "in_progress", text, "status", notified, now, {
328 action: ended ? "maintenance_completed" : "maintenance_started",
329 detail: `${m.title} (on schedule)`,
330 });
331 }
332 // Detection. A deploy restarts services: during one, and briefly after, trouble is counted but not drafted.
333 const [streaks, open, page, deploy] = await Promise.all([loadStreaks(env.DB), openRefs(env.DB), load(env.DB, now, origin), loadDeploy(env.DB)]);
334 const quiet = deployQuiet(deploy, now);
335 const found = detect(streaks, observations, open, underMaintenance(page.maintenance, now), now, { quiet });
336 await saveStreaks(env.DB, found.streaks);
337 if (found.held.length) console.log(JSON.stringify({ event: "status.held_for_deploy", parts: found.held, deploy: deploy?.id ?? null }));
338 const name = (key: string) => named.get(key) ?? key;
339 const slowMs = (key: string) => parts(env).find((p) => p.key === key)?.slowMs ?? SLOW_MS;
340 const lines = [
341 ...found.failing.map((f) => ({ incident: f.incident, kind: "failing" as const, text: troubleSentence(name(f.key), f, stamp(f.since), slowMs(f.key)) })),
342 ...found.recovered.map((r) => ({ incident: r.incident, kind: "recovered" as const, text: recoverySentence(name(r.key), r, stamp(r.since)) })),
343 ];
344 await addSystemLines(env.DB, lines, now);
345 const sudo = (id: string) => `${(env.SUDO_URL || "https://sudo.g1t.sh").replace(/\/+$/, "")}/incidents/${id}`;
346 const alertTo = (env.STATUS_ALERT_EMAIL ?? "").trim();
347 const send = sender(env);
348 if (found.draft.length) {
349 const core = new Set(
350 parts(env)
351 .filter((p) => p.core)
352 .map((p) => p.key),
353 );
354 const title = draftTitle(found.draft.map((d) => ({ name: name(d.key), state: d.state })));
355 const since = found.draft.map((d) => d.since).sort()[0]!;
356 const said = found.draft.map((d) => troubleSentence(name(d.key), d, stamp(d.since), slowMs(d.key)));
357 // Trouble that began in a deploy and outlasted it: say so, it is the first thing to rule out.
358 const note = deploy && deployQuiet(deploy, new Date(since)) ? `It began during a deploy (started ${stamp(deploy.started_at)}) and outlasted it.` : null;
359 const id = await createIncident(
360 env.DB,
361 {
362 title,
363 severity: found.draft.some((d) => d.state === "down" && core.has(d.key)) ? "sev2" : "sev3",
364 status: "investigating",
365 visibility: "draft",
366 source: "detected",
367 components: found.draft.map((d) => ({ key: d.key, impact: detectedImpact(d.state) })),
368 started_at: since,
369 acknowledged_at: null,
370 commander: null,
371 communications: null,
372 by: "status",
373 },
374 [
375 {
376 kind: "detected",
377 public: false,
378 status: null,
379 text: `${said.join(" ")}${note ? ` ${note}` : ""} Not on the status page until it is published.`,
380 },
381 ],
382 now,
383 { action: "incident_detected", detail: title },
384 );
385 console.warn(JSON.stringify({ event: "status.detected", id, parts: found.draft.map((d) => d.key), since }));
386 if (alertTo && send) {
387 const { text, html } = renderMail(alertLetter({ title, lines: said, link: sudo(id), ...(note ? { note } : {}) }));
388 ctx.waitUntil(
389 send
390 .send({ to: alertTo, subject: `[g1t status] ${title}`, text, html })
391 .catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))),
392 );
393 }
394 }
395 // Detected drafts no one picked up, whose parts have stayed healthy long enough: dismissed, with a word to staff.
396 // A part counts as healthy from its first good check; a run that is still going but answered well last time does not hold a draft up.
397 const troubled = new Set(found.streaks.filter(troubledNow).map((s) => s.component));
398 const watched = await watchedDrafts(env.DB);
399 const settled = settleDrafts(watched, troubled, now);
400 await saveHealthy(env.DB, settled.healthy);
401 const dismissed = new Set<string>();
402 for (const d of settled.dismiss) {
403 const text = autoDismissText(d.lasted_ms, stamp(d.recovered_at));
404 if (!(await autoDismiss(env.DB, d.id, d.recovered_at, text, now))) continue;
405 dismissed.add(d.id);
406 console.log(JSON.stringify({ event: "status.auto_dismissed", id: d.id, lasted_ms: d.lasted_ms }));
407 if (alertTo && send) {
408 const letter = recoveredLetter({ title: d.title, text, link: sudo(d.id) });
409 const { text: body, html } = renderMail(letter);
410 ctx.waitUntil(
411 send
412 .send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html })
413 .catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))),
414 );
415 }
416 }
417 // Drafts still waiting for someone: the alert goes out again after 45 minutes, then every 6 hours.
418 const waiting = watched.filter((d) => !dismissed.has(d.id));
419 const stale = staleDrafts(waiting, now);
420 const emailed = !!(alertTo && send);
421 await saveReminders(
422 env.DB,
423 waiting.map((d) => d.id),
424 stale.map((d) => ({ id: d.id, text: staleText(d.waiting_ms, emailed) })),
425 now,
426 );
427 for (const d of stale) {
428 console.warn(JSON.stringify({ event: "status.draft_waiting", id: d.id, waiting_ms: d.waiting_ms }));
429 if (!emailed) continue;
430 const letter = staleLetter({ title: d.title, waiting: minutesWords(d.waiting_ms), link: sudo(d.id) });
431 const { text: body, html } = renderMail(letter);
432 ctx.waitUntil(
433 send!
434 .send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html })
435 .catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) }))),
436 );
437 }
438}
439
440/**
441 * The deploy tool's word that a deploy started or finished:
442 * `POST /deploys` with `Authorization: Bearer <STATUS_DEPLOY_TOKEN>` and
443 * `{"phase": "started" | "finished", "id": "<run or commit>"}`. Without
444 * the secret set, there is no such address.
445 */
446async function deployHook(request: Request, env: Env): Promise<Response> {
447 const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store", ...COMMON } });
448 const token = (env.STATUS_DEPLOY_TOKEN ?? "").trim();
449 if (!token) return json({ error: { code: "not_found", message: "Not found." } }, 404);
450 const given = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "").trim();
451 if (!(await sameSecret(given, token))) return json({ error: { code: "unauthorized", message: "A valid deploy token is required." } }, 401);
452 let body: { phase?: unknown; id?: unknown } = {};
453 try {
454 body = (await request.json()) as typeof body;
455 } catch {
456 // Checked below.
457 }
458 const phase = body.phase === "started" || body.phase === "finished" ? body.phase : null;
459 if (!phase) return json({ error: { code: "invalid", message: 'phase must be "started" or "finished".' } }, 400);
460 const id = typeof body.id === "string" && body.id.trim() ? body.id.trim().slice(0, 100) : null;
461 const now = new Date();
462 const window = deployChange(await loadDeploy(env.DB), phase, id, now);
463 await saveDeploy(env.DB, window);
464 console.log(JSON.stringify({ event: `status.deploy_${phase}`, id, running: window.running }));
465 return json({ deploy: window, quiet_until: quietUntil(window) });
466}
467
468/** Compares two secrets in constant time, by their hashes. */
469async function sameSecret(a: string, b: string): Promise<boolean> {
470 const digest = async (v: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(v)));
471 const [x, y] = await Promise.all([digest(a), digest(b)]);
472 let diff = a.length === 0 ? 1 : 0;
473 for (let i = 0; i < x.length; i++) diff |= x[i]! ^ y[i]!;
474 return diff === 0;
475}
476
477// --- Pages -------------------------------------------------------------------------------
478
479async function model(env: Env, now: Date, origin: string): Promise<PageModel> {
480 const stored = await load(env.DB, now, origin);
481 return buildPage({
482 parts: parts(env),
483 current: stored.current,
484 checkedAt: stored.checkedAt,
485 days: stored.days,
486 incidents: stored.incidents,
487 maintenance: stored.maintenance,
488 now,
489 });
490}
491
492/** When this isolate last asked for a catch-up round, so a busy page asks once. */
493let caughtUpAt = 0;
494
495function catchUp(env: Env, ctx: ExecutionContext, page: PageModel, now: Date) {
496 const checked = page.report.checked_at ? Date.parse(page.report.checked_at) : 0;
497 if (now.getTime() - checked < BEHIND_MS || now.getTime() - caughtUpAt < BEHIND_MS) return;
498 caughtUpAt = now.getTime();
499 ctx.waitUntil(checkAll(env, now).catch((error) => console.error(JSON.stringify({ event: "status.catch_up_failed", error: String(error) }))));
500}
501
502const PAGE_POLICY = [
503 "default-src 'none'",
504 "script-src 'self'",
505 "style-src 'unsafe-inline'",
506 "font-src 'self'",
507 "img-src 'self' data:",
508 "base-uri 'none'",
509 "form-action 'self'",
510 "frame-ancestors 'none'",
511].join("; ");
512
513const COMMON = {
514 "x-content-type-options": "nosniff",
515 "referrer-policy": "strict-origin-when-cross-origin",
516};
517
518function edgeCache(): Cache | null {
519 return (globalThis as unknown as { caches?: { default?: Cache } }).caches?.default ?? null;
520}
521
522/**
523 * A response from the edge cache, or made and kept there. Pages that say
524 * times pass the reader's zone, and are kept once per zone.
525 */
526async function cached(request: Request, ctx: ExecutionContext, make: () => Promise<Response>, zone?: string): Promise<Response> {
527 const cache = edgeCache();
528 const url = new URL(request.url);
529 const key = new Request(`${url.origin}${url.pathname}${zone ? `?zone=${encodeURIComponent(zone)}` : ""}`, { method: "GET" });
530 if (cache) {
531 const hit = await cache.match(key).catch(() => undefined);
532 if (hit) return hit;
533 }
534 const response = await make();
535 if (cache && response.ok) ctx.waitUntil(cache.put(key, response.clone()).catch(() => undefined));
536 return response;
537}
538
539const FONTS: Record<string, ArrayBuffer> = {
540 "/fonts/hanken-grotesk.woff2": hanken,
541 "/fonts/bricolage-grotesque.woff2": bricolage,
542 "/fonts/ibm-plex-mono.woff2": plexMono,
543};
544
545function html(body: string, cacheControl: string, status = 200): Response {
546 return new Response(body, {
547 status,
548 headers: { "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "content-security-policy": PAGE_POLICY, ...COMMON },
549 });
550}
551
552async function form(request: Request): Promise<FormData> {
553 try {
554 return await request.formData();
555 } catch {
556 return new FormData();
557 }
558}
559
560/** Subscribing, confirming and leaving: the page's only writes. */
561async function subscriptions(request: Request, env: Env, ctx: ExecutionContext, url: URL, options: PageOptions): Promise<Response | null> {
562 const path = url.pathname;
563 const noStore = "no-store";
564 const message = (title: string, text: string, status = 200, f?: { action: string; fields: Record<string, string>; button: string }) =>
565 html(renderMessage({ ...options, selfUrl: `${url.origin}${path}` }, { title, text, form: f }), noStore, status);
566 const post = request.method === "POST";
567
568 if (path === "/subscribe" && post) {
569 if (!emailOn(env)) return message("Email updates are not available", "Follow the Atom or JSON feed instead.", 503);
570 // Each request can send an email: limited per client, then per address.
571 const tooMany = () => {
572 const answer = message("Too many requests", "Wait a minute and try again.", 429);
573 answer.headers.set("retry-after", String(LIMIT_PERIOD_SECONDS));
574 return answer;
575 };
576 if (await isLimited(env.STATUS_SUBSCRIBE_LIMIT, `ip:${clientAddress(request)}`)) return tooMany();
577 const data = await form(request);
578 if (String(data.get("website") ?? "")) return message("Check your inbox", "If the address is right, a confirmation link is on its way.");
579 const email = normalizeEmail(data.get("email"));
580 if (!email) return message("That is not an email address", "Go back and check it.", 400);
581 if (await isLimited(env.STATUS_EMAIL_LIMIT, await secretKey("email", email))) return tooMany();
582 const chosen = chosenParts(
583 data.getAll("components").map(String),
584 parts(env).map((p) => p.key),
585 );
586 const token = newToken();
587 const { send } = await requestSubscription(env.DB, email, chosen, await hashToken(token), new Date(), CONFIRM_TTL_MS, RESEND_AFTER_MS);
588 if (send) {
589 const link = `${url.origin}/subscribe/confirm?token=${encodeURIComponent(token)}`;
590 const { text, html: body } = renderMail(confirmLetter(link, chosen ? chosen.map((k) => names(env).get(k) ?? k) : null));
591 ctx.waitUntil(
592 sender(env)!
593 .send({ to: email, subject: "Confirm your subscription to g1t status", text, html: body })
594 .catch((e) => console.error(JSON.stringify({ event: "status.confirm_failed", error: String(e) }))),
595 );
596 }
597 return message("Check your inbox", "If the address is right, a confirmation link is on its way. It works for 24 hours.");
598 }
599 if (path === "/subscribe/confirm") {
600 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
601 if (!token) return message("That link is incomplete", "Copy the whole link from the email.", 400);
602 if (!post)
603 return message("Confirm your subscription", "One more step: confirm to start getting emails about incidents and maintenance.", 200, {
604 action: "/subscribe/confirm",
605 fields: { token },
606 button: "Confirm subscription",
607 });
608 const done = await confirmSubscription(env.DB, await hashToken(token), new Date());
609 if (!done) return message("That link has expired", "Confirmation links work for 24 hours and once. Subscribe again for a new one.", 410);
610 return message(
611 "You are subscribed",
612 `${done.email} will get an email when g1t posts an incident or maintenance${done.parts ? ` affecting ${done.parts.map((k) => names(env).get(k) ?? k).join(", ")}` : ""}. Every email has a link to unsubscribe.`,
613 );
614 }
615 if (path === "/unsubscribe") {
616 const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : "");
617 const id = env.STATUS_SECRET && token ? await readUnsubscribeToken(env.STATUS_SECRET, token) : null;
618 if (!id) return message("That link is not valid", "Use the unsubscribe link at the bottom of any email from g1t status.", 400);
619 if (!post) return message("Unsubscribe", "Stop getting emails from g1t status?", 200, { action: "/unsubscribe", fields: { token }, button: "Unsubscribe" });
620 await unsubscribe(env.DB, id);
621 return message("You are unsubscribed", "You will not get any more emails from g1t status. You can subscribe again at any time.");
622 }
623 return null;
624}
625
626async function handle(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
627 const url = new URL(request.url);
628 const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, "") : url.pathname;
629 if (request.method === "OPTIONS" && (path === "/status.json" || path === "/feed.json")) {
630 return new Response(null, {
631 status: 204,
632 headers: { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, HEAD", "access-control-max-age": "86400" },
633 });
634 }
635 const now = new Date();
636 const origin = originOf(env, url);
637 const site = env.PUBLIC_SITE_URL || env.SITE_URL || url.origin;
638 const options: PageOptions = {
639 siteUrl: site,
640 supportUrl: env.SUPPORT_URL || `${site}/support`,
641 ogImage: env.OG_IMAGE ?? "",
642 selfUrl: `${url.origin}${path === "/" ? "/" : path}`,
643 now,
644 email: emailOn(env),
645 zone: readZone(request.headers.get("cookie"), (request as { cf?: { timezone?: unknown } }).cf?.timezone),
646 };
647
648 if (request.method === "POST" && path === "/deploys") return deployHook(request, env);
649 if (request.method === "POST") {
650 const answer = await subscriptions(request, env, ctx, url, options);
651 return answer ?? new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD", ...COMMON } });
652 }
653 if (request.method !== "GET" && request.method !== "HEAD") {
654 return new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD, POST", ...COMMON } });
655 }
656 const fresh = (cacheSeconds = CACHE_SECONDS) => `public, max-age=${cacheSeconds}`;
657 const notFound = () => html(renderMessage(options, { title: "Not found", text: "There is nothing at this address." }), fresh(), 404);
658
659 switch (path) {
660 case "/":
661 return cached(
662 request,
663 ctx,
664 async () => {
665 const page = await model(env, now, origin);
666 catchUp(env, ctx, page, now);
667 return html(renderPage(page, options), fresh());
668 },
669 options.zone,
670 );
671 case "/status.json":
672 return cached(request, ctx, async () => {
673 const page = await model(env, now, origin);
674 catchUp(env, ctx, page, now);
675 return Response.json(page.report, { headers: { "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON } });
676 });
677 case "/badge.svg":
678 return cached(request, ctx, async () => {
679 const page = await model(env, now, origin);
680 return new Response(renderBadge(page.report.overall.state, page.report.overall.title), {
681 headers: { "content-type": "image/svg+xml", "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON },
682 });
683 });
684 case "/history":
685 return cached(
686 request,
687 ctx,
688 async () => {
689 const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 11, 1));
690 const { incidents, maintenance } = await loadHistory(env.DB, since, origin);
691 return html(renderHistory(incidents, maintenance, options), fresh());
692 },
693 options.zone,
694 );
695 case "/feed.xml":
696 case "/feed.json":
697 return cached(request, ctx, async () => {
698 const { incidents, maintenance } = await loadHistory(env.DB, new Date(now.getTime() - 365 * 86_400_000), origin);
699 const items = feedItems(incidents, maintenance);
700 const feed = { origin, title: "g1t status", updated: now.toISOString() };
701 const headers = { "cache-control": fresh(60), "access-control-allow-origin": "*", ...COMMON };
702 return path === "/feed.xml"
703 ? new Response(atom(items, feed), { headers: { "content-type": "application/atom+xml; charset=utf-8", ...headers } })
704 : Response.json(jsonFeed(items, feed), { headers: { "content-type": "application/feed+json; charset=utf-8", ...headers } });
705 });
706 case "/subscribe":
707 return html(
708 renderSubscribe(
709 options,
710 parts(env).map(({ key, name }) => ({ key, name })),
711 ),
712 fresh(300),
713 );
714 case "/subscribe/confirm":
715 case "/unsubscribe":
716 return (await subscriptions(request, env, ctx, url, options))!;
717 case "/theme.js":
718 return new Response(THEME_SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
719 case "/status.js":
720 return new Response(SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } });
721 case "/favicon.svg":
722 case "/favicon.ico":
723 return new Response(FAVICON, { headers: { "content-type": "image/svg+xml", "cache-control": fresh(86400), ...COMMON } });
724 case "/robots.txt":
725 return new Response("User-agent: *\nAllow: /\nDisallow: /subscribe/confirm\nDisallow: /unsubscribe\n", {
726 headers: { "content-type": "text/plain", "cache-control": fresh(86400) },
727 });
728 }
729 const incident = /^\/incidents\/([a-z0-9-]{1,64})$/.exec(path);
730 if (incident) {
731 return cached(
732 request,
733 ctx,
734 async () => {
735 const found = await loadPublicIncident(env.DB, incident[1]!, origin);
736 return found ? html(renderIncident(found.incident, found.postmortem, names(env), options), fresh()) : notFound();
737 },
738 options.zone,
739 );
740 }
741 const maintenance = /^\/maintenance\/([a-z0-9-]{1,64})$/.exec(path);
742 if (maintenance) {
743 return cached(
744 request,
745 ctx,
746 async () => {
747 const found = await loadPublicMaintenance(env.DB, maintenance[1]!, origin);
748 return found ? html(renderMaintenance(found, names(env), options), fresh()) : notFound();
749 },
750 options.zone,
751 );
752 }
753 const font = FONTS[path];
754 if (font) {
755 return new Response(font, { headers: { "content-type": "font/woff2", "cache-control": "public, max-age=31536000, immutable", ...COMMON } });
756 }
757 return notFound();
758}
759
760export default {
761 async fetch(request, env, ctx) {
762 try {
763 return await handle(request, env, ctx);
764 } catch (error) {
765 console.error(JSON.stringify({ event: "status.failed", path: new URL(request.url).pathname, error: String(error) }));
766 return new Response("The status page could not be drawn. Try again in a minute.", {
767 status: 503,
768 headers: { "content-type": "text/plain; charset=utf-8", "retry-after": "60", ...COMMON },
769 });
770 }
771 },
772 async scheduled(_controller, env, ctx) {
773 const now = new Date();
774 ctx.waitUntil(
775 checkAll(env, now)
776 .then(async (observations) => {
777 const failing = observations.filter((o) => o.state === "down" || o.state === "degraded");
778 if (failing.length) console.warn(JSON.stringify({ event: "status.trouble", parts: failing }));
779 await afterChecks(env, ctx, observations, now);
780 })
781 .catch((error) => console.error(JSON.stringify({ event: "status.cron_failed", error: String(error) }))),
782 );
783 },
784} satisfies ExportedHandler<Env>;
785
786// --- Staff ---------------------------------------------------------------------------------
787
788/**
789 * Staff only: running incidents and maintenance. Reached only through a
790 * service binding (sudo's `STATUS`); status.g1t.sh's own address cannot.
791 */
792export class StatusAdmin extends WorkerEntrypoint<Env> implements StatusAdminApi {
793 private known() {
794 return parts(this.env).map((p) => p.key);
795 }
796
797 private origin() {
798 return originOf(this.env);
799 }
800
801 private async detail(id: string): Promise<AdminIncidentDetail | null> {
802 const limits = new Map(parts(this.env).map((p) => [p.key, p.slowMs ?? SLOW_MS]));
803 return incidentDetail(this.env.DB, String(id), this.origin(), names(this.env), { limits });
804 }
805
806 private async summary(id: string): Promise<AdminIncident> {
807 const { timeline: _t, followups: _f, postmortem: _p, postmortem_draft: _d, url: _u, checks: _c, ...incident } = (await this.detail(id))!;
808 return incident;
809 }
810
811 /** Emails a public update to subscribers when asked; the count to keep with it. */
812 private async announce(
813 incident: { id: string; title: string; components: { key: string; impact: string }[] },
814 status: keyof typeof INCIDENT_STATUS,
815 text: string,
816 wanted: boolean,
817 ) {
818 if (!wanted) return null;
819 const about = incident.components.filter((c) => c.impact !== "operational").map((c) => c.key);
820 return notify(this.env, this.ctx, about, {
821 heading: `${INCIDENT_STATUS[status]}: ${incident.title}`,
822 text,
823 url: `${this.origin()}/incidents/${incident.id}`,
824 });
825 }
826
827 async components(): Promise<{ key: string; name: string }[]> {
828 return parts(this.env).map(({ key, name }) => ({ key, name }));
829 }
830
831 async board(): Promise<StatusBoard> {
832 return { ...(await board(this.env.DB, new Date(), this.origin())), email: emailOn(this.env) };
833 }
834
835 async incident(id: string): Promise<AdminIncidentDetail | null> {
836 return this.detail(id);
837 }
838
839 async openCount(): Promise<number> {
840 return openCount(this.env.DB);
841 }
842
843 async declare(input: DeclareIncident): Promise<Result<AdminIncident>> {
844 const checked = checkDeclare(input, this.known());
845 if (!checked.ok) return fail("invalid", checked.error);
846 const v = checked.value;
847 const now = new Date();
848 const entries: (Entry & { notified?: number | null })[] = [
849 { kind: "declared", public: false, status: null, text: `Declared ${SEVERITY_LABEL[v.severity]}.` },
850 ];
851 if (v.commander) entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${v.commander}.` });
852 if (v.communications) entries.push({ kind: "role", public: false, status: null, text: `Communications: ${v.communications}.` });
853 const id = shortId();
854 const status = v.status ?? "investigating";
855 const notified = await this.announce({ id, title: v.title, components: v.components }, status, v.message, v.notify);
856 entries.push({ kind: "update", public: true, status, text: v.message, notified });
857 await createIncident(
858 this.env.DB,
859 {
860 title: v.title,
861 severity: v.severity,
862 status,
863 visibility: "public",
864 source: "declared",
865 components: v.components,
866 started_at: v.started_at ?? now.toISOString(),
867 acknowledged_at: now.toISOString(),
868 commander: v.commander ?? null,
869 communications: v.communications ?? null,
870 by: v.by,
871 },
872 entries,
873 now,
874 { action: "incident_declared", detail: `${SEVERITY_LABEL[v.severity]}: ${v.title}` },
875 id,
876 );
877 console.log(JSON.stringify({ event: "status.incident_declared", id, by: v.by }));
878 return ok(await this.summary(id));
879 }
880
881 async update(id: string, change: IncidentChange): Promise<Result<AdminIncident>> {
882 const checked = checkChange(change, this.known());
883 if (!checked.ok) return fail("invalid", checked.error);
884 const existing = await this.detail(id);
885 if (!existing) return fail("not_found", "No such incident.");
886 const now = new Date();
887 const applied = applyChange(facts(existing), checked.value, now, names(this.env));
888 if (!applied.ok) return fail("invalid", applied.error);
889 const { next, entries } = applied.value;
890 const update = entries.find((e) => e.kind === "update");
891 const notified = update
892 ? await this.announce({ id: existing.id, title: existing.title, components: next.components }, next.status, update.text, checked.value.notify === true)
893 : null;
894 const lines = entries.map((e) => (e === update ? { ...e, notified } : e));
895 const action = next.status === "resolved" && existing.status !== "resolved" ? "incident_resolved" : update ? "incident_update" : "incident_note";
896 await saveIncident(this.env.DB, existing.id, next, lines, now, checked.value.by, {
897 action,
898 detail: entries
899 .map((e) => (e.kind === "update" || e.kind === "note" ? `${e.kind === "update" ? "Public" : "Note"}: ${e.text}` : e.text))
900 .join(" ")
901 .slice(0, 500),
902 });
903 return ok(await this.summary(existing.id));
904 }
905
906 async roles(id: string, change: RolesChange): Promise<Result<AdminIncident>> {
907 const checked = checkRoles(change);
908 if (!checked.ok) return fail("invalid", checked.error);
909 const existing = await this.detail(id);
910 if (!existing) return fail("not_found", "No such incident.");
911 const now = new Date();
912 const { next, entries } = applyRoles(facts(existing), checked.value, now);
913 if (!entries.length) return ok(await this.summary(existing.id));
914 await saveIncident(this.env.DB, existing.id, next, entries, now, checked.value.by, {
915 action: "incident_roles",
916 detail: entries.map((e) => e.text).join(" "),
917 });
918 return ok(await this.summary(existing.id));
919 }
920
921 async publish(id: string, input: PublishIncident): Promise<Result<AdminIncident>> {
922 const checked = checkPublish(input);
923 if (!checked.ok) return fail("invalid", checked.error);
924 const existing = await this.detail(id);
925 if (!existing) return fail("not_found", "No such incident.");
926 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be published.");
927 const now = new Date();
928 const at = now.toISOString();
929 const title = checked.value.title ?? existing.title;
930 const next = { ...facts(existing), visibility: "public" as const, acknowledged_at: existing.acknowledged_at ?? at, title, published_at: at };
931 const notified = await this.announce(
932 { id: existing.id, title, components: existing.components },
933 existing.status,
934 checked.value.message,
935 checked.value.notify,
936 );
937 await saveIncident(
938 this.env.DB,
939 existing.id,
940 next,
941 [
942 ...(existing.acknowledged_at ? [] : [{ kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged." }]),
943 { kind: "published", public: false, status: null, text: title !== existing.title ? `Published as “${title}”.` : "Published to the status page." },
944 { kind: "update", public: true, status: existing.status, text: checked.value.message, notified },
945 ],
946 now,
947 checked.value.by,
948 { action: "incident_published", detail: title },
949 );
950 return ok(await this.summary(existing.id));
951 }
952
953 async dismiss(id: string, input: { reason: string; by: string }): Promise<Result<AdminIncident>> {
954 const existing = await this.detail(id);
955 if (!existing) return fail("not_found", "No such incident.");
956 if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be dismissed; resolve a published incident instead.");
957 const by = String(input?.by ?? "").trim();
958 if (!by) return fail("invalid", "Who is making the change is missing.");
959 const reason =
960 String(input?.reason ?? "")
961 .trim()
962 .slice(0, 500) || "No reason given.";
963 const now = new Date();
964 const at = now.toISOString();
965 const next = {
966 ...facts(existing),
967 visibility: "dismissed" as const,
968 status: "resolved" as const,
969 acknowledged_at: existing.acknowledged_at ?? at,
970 resolved_at: at,
971 };
972 await saveIncident(this.env.DB, existing.id, next, [{ kind: "dismissed", public: false, status: null, text: `Dismissed: ${reason}` }], now, by, {
973 action: "incident_dismissed",
974 detail: `${existing.title}: ${reason}`,
975 });
976 return ok(await this.summary(existing.id));
977 }
978
979 async addFollowUp(id: string, input: { title: string; owner: string | null; by: string }): Promise<Result<FollowUp>> {
980 const checked = checkFollowUp(input);
981 if (!checked.ok) return fail("invalid", checked.error);
982 if (!(await this.detail(id))) return fail("not_found", "No such incident.");
983 return ok(await addFollowUp(this.env.DB, String(id), checked.value, new Date()));
984 }
985
986 async setFollowUp(id: string, followUp: string, input: { done: boolean; by: string }): Promise<Result<FollowUp>> {
987 const by = String(input?.by ?? "").trim();
988 if (!by) return fail("invalid", "Who is making the change is missing.");
989 const done = await setFollowUp(this.env.DB, String(id), String(followUp), input.done === true, by, new Date());
990 return done ? ok(done) : fail("not_found", "No such follow-up.");
991 }
992
993 async savePostmortem(id: string, input: PostmortemFields & { by: string }): Promise<Result<Postmortem>> {
994 const checked = checkPostmortem(input);
995 if (!checked.ok) return fail("invalid", checked.error);
996 const existing = await this.detail(id);
997 if (!existing) return fail("not_found", "No such incident.");
998 if (existing.visibility !== "public") return fail("conflict", "Only a published incident has a postmortem.");
999 const { by, ...fields } = checked.value;
1000 await savePostmortem(this.env.DB, existing.id, fields, by, new Date());
1001 return ok((await this.detail(existing.id))!.postmortem!);
1002 }
1003
1004 async publishPostmortem(id: string, input: { publish: boolean; by: string }): Promise<Result<Postmortem>> {
1005 const by = String(input?.by ?? "").trim();
1006 if (!by) return fail("invalid", "Who is making the change is missing.");
1007 const existing = await this.detail(id);
1008 if (!existing) return fail("not_found", "No such incident.");
1009 if (!existing.postmortem) return fail("conflict", "Save the postmortem before publishing it.");
1010 if (input.publish) {
1011 if (!existing.resolved_at) return fail("conflict", "Resolve the incident before publishing its postmortem.");
1012 const missing = postmortemReady(existing.postmortem);
1013 if (missing) return fail("invalid", missing);
1014 }
1015 await publishPostmortem(this.env.DB, existing.id, input.publish === true, by, new Date());
1016 return ok((await this.detail(existing.id))!.postmortem!);
1017 }
1018
1019 async scheduleMaintenance(input: NewMaintenance): Promise<Result<AdminMaintenance>> {
1020 const checked = checkMaintenance(input, this.known());
1021 if (!checked.ok) return fail("invalid", checked.error);
1022 const v = checked.value;
1023 const now = new Date();
1024 const id = shortId();
1025 const notified = v.notify
1026 ? await notify(this.env, this.ctx, v.components, {
1027 heading: `Planned maintenance: ${v.title}`,
1028 text: `${v.message}\n\nWhen: ${stamp(v.starts_at)} to ${stamp(v.ends_at)}.`,
1029 url: maintenanceUrl(this.origin(), id),
1030 })
1031 : null;
1032 const made = await scheduleMaintenance(this.env.DB, v, notified, now, id);
1033 return ok((await maintenanceById(this.env.DB, made, this.origin()))!);
1034 }
1035
1036 async changeMaintenance(id: string, change: MaintenanceChange): Promise<Result<AdminMaintenance>> {
1037 const checked = checkMaintenanceChange(change);
1038 if (!checked.ok) return fail("invalid", checked.error);
1039 const existing = await maintenanceById(this.env.DB, String(id), this.origin());
1040 if (!existing) return fail("not_found", "No such maintenance.");
1041 const v = checked.value;
1042 if (existing.state === "completed" || existing.state === "cancelled") return fail("conflict", `This maintenance is ${existing.state}.`);
1043 if (v.action === "start" && existing.state !== "scheduled") return fail("conflict", "It has already started.");
1044 const state = v.action === "start" ? "in_progress" : v.action === "complete" ? "completed" : v.action === "cancel" ? "cancelled" : null;
1045 const text =
1046 v.message ||
1047 (v.action === "start" ? "The maintenance has begun." : v.action === "complete" ? "The maintenance is complete." : "This maintenance is cancelled.");
1048 const word = { update: "Update", start: "In progress", complete: "Completed", cancel: "Cancelled" }[v.action];
1049 const notified = v.notify
1050 ? await notify(this.env, this.ctx, existing.components, { heading: `${word}: ${existing.title}`, text, url: existing.url })
1051 : null;
1052 await maintenanceUpdate(this.env.DB, existing.id, state, text, v.by, notified, new Date(), {
1053 action: `maintenance_${v.action === "update" ? "update" : v.action === "start" ? "started" : v.action === "complete" ? "completed" : "cancelled"}`,
1054 detail: `${existing.title}: ${text}`,
1055 });
1056 return ok((await maintenanceById(this.env.DB, existing.id, this.origin()))!);
1057 }
1058
1059 async audit(filter?: { before?: string | null }): Promise<StatusAuditEntry[]> {
1060 const before = filter?.before && !Number.isNaN(Date.parse(filter.before)) ? filter.before : null;
1061 return auditLog(this.env.DB, before);
1062 }
1063}