Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 1 | /** |
| 2 | * status.g1t.sh: whether each part of g1t is working, how it has done over | |
| 3 | * 90 days, and what staff have said about incidents and maintenance. | |
| 4 | * | |
| 5 | * A Worker of its own, apart from the site, so it stays up when g1t does | |
| 6 | * not. Every minute a cron checks each part over the public internet, as | |
| 7 | * people reach it, and keeps the result in D1. The same run moves planned | |
| 8 | * maintenance along, and drafts an incident for staff when a part keeps | |
| 9 | * failing (detect.ts). Pages are drawn from what is kept, never by | |
| 10 | * checking on the spot, and kept at the edge for 30 seconds. | |
| 11 | * | |
| 12 | * Staff run incidents from sudo, through the `StatusAdmin` entrypoint, | |
| 13 | * which only a service binding reaches. Every change there is audited. | |
| 14 | * | |
| 15 | * GET / the page | |
| 16 | * GET /status.json the same as JSON (snake_case, CORS open) | |
| 17 | * GET /badge.svg a small badge | |
| 18 | * GET /incidents/<id> an incident's updates and postmortem | |
| 19 | * GET /maintenance/<id> a maintenance window's updates | |
| 20 | * GET /history the last 12 months, by month | |
| 21 | * GET /feed.xml, /feed.json every public update, newest first | |
| 22 | * GET /subscribe subscribing by email | |
| 23 | * POST /subscribe asks for a subscription: a confirmation email | |
| 24 | * GET|POST /subscribe/confirm?token= confirms (GET shows a button: link scanners must not confirm) | |
| 25 | * GET|POST /unsubscribe?token= leaves (POST also takes RFC 8058 one-click) | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 26 | * POST /deploys the deploy tool: a deploy started or finished (bearer STATUS_DEPLOY_TOKEN) |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 27 | */ |
| 28 | import { WorkerEntrypoint } from "cloudflare:workers"; | |
| 29 | import { | |
| 30 | type AdminIncident, | |
| 31 | type AdminIncidentDetail, | |
| 32 | type AdminMaintenance, | |
| 33 | type DeclareIncident, | |
| 34 | type FollowUp, | |
| 35 | type IncidentChange, | |
| 36 | type MaintenanceChange, | |
| 37 | type NewMaintenance, | |
| 38 | type Postmortem, | |
| 39 | type PostmortemFields, | |
| 40 | type PublishIncident, | |
| 41 | type Result, | |
| 42 | type RolesChange, | |
| 43 | type StatusAdminApi, | |
| 44 | type StatusAuditEntry, | |
| 45 | type StatusBoard, | |
| 46 | billingClient, | |
| 47 | fail, | |
| 48 | ok, | |
| 49 | } from "@g1t/contracts"; | |
| 50 | import bricolage from "@g1t/theme/fonts/bricolage-grotesque-latin.woff2"; | |
| 51 | import hanken from "@g1t/theme/fonts/hanken-grotesk-latin.woff2"; | |
| 52 | import plexMono from "@g1t/theme/fonts/ibm-plex-mono-latin-400.woff2"; | |
| 53 | ||
| 54 | import { type Targets, components } from "./components.ts"; | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 55 | import { |
| 56 | DEPLOY_GRACE_MS, | |
| 57 | DEPLOY_MAX_MS, | |
| 58 | autoDismissText, | |
| 59 | deployChange, | |
| 60 | deployQuiet, | |
| 61 | detect, | |
| 62 | detectedImpact, | |
| 63 | draftTitle, | |
| 64 | recoverySentence, | |
| 65 | settleDrafts, | |
| 66 | troubleSentence, | |
| 67 | } from "./detect.ts"; | |
| 68 | import { type EmailBinding, type Sender, alertLetter, bindingSender, confirmLetter, recoveredLetter, render as renderMail, unsubscribeHeaders, updateLetter } from "./email.ts"; | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 69 | import { atom, feedItems, jsonFeed } from "./feed.ts"; |
| 70 | import { | |
| 71 | type Entry, | |
| 72 | applyChange, | |
| 73 | applyRoles, | |
| 74 | checkChange, | |
| 75 | checkDeclare, | |
| 76 | checkFollowUp, | |
| 77 | checkMaintenance, | |
| 78 | checkMaintenanceChange, | |
| 79 | checkPostmortem, | |
| 80 | checkPublish, | |
| 81 | checkRoles, | |
| 82 | postmortemReady, | |
| 83 | SEVERITY_LABEL, | |
| 84 | shortId, | |
| 85 | } from "./incidents.ts"; | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 86 | import { INCIDENT_STATUS, type PageModel, SLOW_MS, buildPage, classify, underMaintenance } from "./model.ts"; |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 87 | import { stamp } from "./postmortem.ts"; |
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 88 | import { type StorageReport, runCheck } from "./probe.ts"; |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 89 | import { readZone } from "./time.ts"; |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 90 | import { |
| 91 | FAVICON, | |
| 92 | SCRIPT, | |
| 93 | type PageOptions, | |
| 94 | renderBadge, | |
| 95 | renderHistory, | |
| 96 | renderIncident, | |
| 97 | renderMaintenance, | |
| 98 | renderMessage, | |
| 99 | renderPage, | |
| 100 | renderSubscribe, | |
| 101 | } from "./render.ts"; | |
| 102 | import { | |
| 103 | type Observation, | |
| 104 | addFollowUp, | |
| 105 | addSystemLines, | |
| 106 | auditLog, | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 107 | autoDismiss, |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 108 | board, |
| 109 | confirmSubscription, | |
| 110 | createIncident, | |
| 111 | dueMaintenance, | |
| 112 | facts, | |
| 113 | incidentDetail, | |
| 114 | load, | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 115 | loadDeploy, |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 116 | loadHistory, |
| 117 | loadPublicIncident, | |
| 118 | loadPublicMaintenance, | |
| 119 | loadStreaks, | |
| 120 | maintenanceById, | |
| 121 | maintenanceUrl, | |
| 122 | maintenanceUpdate, | |
| 123 | openCount, | |
| 124 | openRefs, | |
| 125 | publishPostmortem, | |
| 126 | recipients, | |
| 127 | record, | |
| 128 | requestSubscription, | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 129 | saveDeploy, |
| 130 | saveHealthy, | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 131 | saveIncident, |
| 132 | savePostmortem, | |
| 133 | saveStreaks, | |
| 134 | scheduleMaintenance, | |
| 135 | setFollowUp, | |
| 136 | unsubscribe, | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 137 | watchedDrafts, |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 138 | } from "./store.ts"; |
| 139 | import { CONFIRM_TTL_MS, RESEND_AFTER_MS, chosenParts, hashToken, newToken, normalizeEmail, readUnsubscribeToken, unsubscribeToken } from "./subscribers.ts"; | |
| 140 | ||
| 141 | export interface Env extends Partial<Targets> { | |
| 142 | DB: D1Database; | |
| 143 | /** Billing, for reading its price book. Optional: without it, billing is not listed. */ | |
| 144 | BILLING?: Fetcher; | |
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 145 | /** The repos service, for git storage's recent health. Optional: without it, git storage is not listed. */ |
| 146 | REPOS?: Fetcher; | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 147 | /** Where help is. */ |
| 148 | SUPPORT_URL?: string; | |
| 149 | /** | |
| 150 | * The site's address as people's browsers reach it, for the page's links, | |
| 151 | * when the checks reach it by another (self-hosted: `http://g1t:8787` | |
| 152 | * inside Compose). SITE_URL when empty. | |
| 153 | */ | |
| 154 | PUBLIC_SITE_URL?: string; | |
| 155 | /** The page's share card; empty for none. */ | |
| 156 | OG_IMAGE?: string; | |
| 157 | /** This page's own address, for links in email and feeds made outside a request. */ | |
| 158 | STATUS_URL?: string; | |
| 159 | /** Where sudo is, for the staff alert's link. */ | |
| 160 | SUDO_URL?: string; | |
| 161 | /** Who hears about detected drafts. Empty sends none. */ | |
| 162 | STATUS_ALERT_EMAIL?: string; | |
| 163 | /** The From of every email. */ | |
| 164 | STATUS_FROM?: string; | |
| 165 | /** Signs unsubscribe links (a secret). Without it, email subscriptions are off; the feeds still work. */ | |
| 166 | STATUS_SECRET?: string; | |
| 167 | /** Cloudflare Email Sending (`send_email`). Without it, nothing is emailed. */ | |
| 168 | EMAIL?: EmailBinding; | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 169 | /** |
| 170 | * The deploy tool's bearer token for `POST /deploys` (a secret). Without | |
| 171 | * it, deploys are not announced and detection does not hold off for them. | |
| 172 | */ | |
| 173 | STATUS_DEPLOY_TOKEN?: string; | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 174 | } |
| 175 | ||
| 176 | /** How long the edge keeps a page or the JSON. */ | |
| 177 | const CACHE_SECONDS = 30; | |
| 178 | /** Older than this, a visit asks for a round of checks too (a missed cron, or `wrangler dev`). */ | |
| 179 | const BEHIND_MS = 3 * 60 * 1000; | |
| 180 | /** How many subscribers one update emails at most, within a Worker's limits. */ | |
| 181 | const MAX_RECIPIENTS = 900; | |
| 182 | ||
| 183 | function parts(env: Env) { | |
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 184 | return components(env, env.BILLING != null, env.REPOS != null); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 185 | } |
| 186 | ||
| 187 | function names(env: Env): Map<string, string> { | |
| 188 | return new Map(parts(env).map((p) => [p.key, p.name])); | |
| 189 | } | |
| 190 | ||
| 191 | /** This page's address: the request's own, or STATUS_URL outside a request. */ | |
| 192 | function originOf(env: Env, url?: URL): string { | |
| 193 | return (url?.origin ?? env.STATUS_URL ?? "https://status.g1t.sh").replace(/\/+$/, ""); | |
| 194 | } | |
| 195 | ||
| 196 | function sender(env: Env): Sender | null { | |
| 197 | return bindingSender(env.EMAIL, env.STATUS_FROM || undefined); | |
| 198 | } | |
| 199 | ||
| 200 | /** Whether subscribers can sign up: a way to send, and a secret to sign their links. */ | |
| 201 | function emailOn(env: Env): boolean { | |
| 202 | return sender(env) != null && !!env.STATUS_SECRET; | |
| 203 | } | |
| 204 | ||
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 205 | /** Git storage's last five minutes, from the repos service (`store_health`). */ |
| 206 | async function storeHealth(repos: Fetcher): Promise<StorageReport> { | |
| 207 | const response = await repos.fetch("https://service/rpc/store_health", { | |
| 208 | method: "POST", | |
| 209 | headers: { "content-type": "application/json" }, | |
| 210 | body: JSON.stringify({ minutes: 5 }), | |
| 211 | }); | |
| 212 | if (!response.ok) throw new Error(`store_health failed with status ${response.status}`); | |
| 213 | return (await response.json()) as StorageReport; | |
| 214 | } | |
| 215 | ||
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 216 | /** One round of checks, kept. Parts under maintenance are checked but not tallied. */ |
| 217 | export async function checkAll(env: Env, now = new Date()): Promise<Observation[]> { | |
| 218 | const billing = env.BILLING; | |
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 219 | const repos = env.REPOS; |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 220 | const list = parts(env); |
| 221 | const observations = await Promise.all( | |
| 222 | list.map(async (info): Promise<Observation> => { | |
| 223 | const result = await runCheck(info.check, { | |
| 224 | fetch: (url, init) => fetch(url, init), | |
| 225 | billing: billing ? () => billingClient(billing).prices() : null, | |
| Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily | 226 | storage: repos ? () => storeHealth(repos) : null, |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 227 | }); |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 228 | const { state, detail } = classify(result, info.slowMs); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 229 | return { component: info.key, state, detail, latency_ms: result ? Math.round(result.ms) : null }; |
| 230 | }), | |
| 231 | ); | |
| 232 | const { maintenance } = await load(env.DB, now, originOf(env)); | |
| 233 | await record(env.DB, observations, now, underMaintenance(maintenance, now)); | |
| 234 | return observations; | |
| 235 | } | |
| 236 | ||
| 237 | // --- Email --------------------------------------------------------------------------- | |
| 238 | ||
| 239 | /** | |
| 240 | * Emails confirmed subscribers who want news about `about`, in the | |
| 241 | * background. Returns how many it will email, or null when email is off. | |
| 242 | */ | |
| 243 | async function notify( | |
| 244 | env: Env, | |
| 245 | ctx: { waitUntil(p: Promise<unknown>): void }, | |
| 246 | about: string[], | |
| 247 | mail: { heading: string; text: string; url: string }, | |
| 248 | ): Promise<number | null> { | |
| 249 | const send = sender(env); | |
| 250 | const secret = env.STATUS_SECRET; | |
| 251 | if (!send || !secret) return null; | |
| 252 | const list = (await recipients(env.DB, about)).slice(0, MAX_RECIPIENTS); | |
| 253 | const origin = originOf(env); | |
| 254 | const affects = about.map((k) => names(env).get(k) ?? k); | |
| 255 | ctx.waitUntil( | |
| 256 | (async () => { | |
| 257 | let failed = 0; | |
| 258 | for (let i = 0; i < list.length; i += 6) { | |
| 259 | await Promise.all( | |
| 260 | list.slice(i, i + 6).map(async (r) => { | |
| 261 | const link = `${origin}/unsubscribe?token=${encodeURIComponent(await unsubscribeToken(secret, r.id))}`; | |
| 262 | const { text, html } = renderMail(updateLetter({ heading: mail.heading, text: mail.text, url: mail.url, affects, unsubscribe: link })); | |
| 263 | await send.send({ to: r.email, subject: mail.heading, text, html, headers: unsubscribeHeaders(link) }).catch(() => void (failed += 1)); | |
| 264 | }), | |
| 265 | ); | |
| 266 | } | |
| 267 | console.log(JSON.stringify({ event: "status.notified", sent: list.length - failed, failed })); | |
| 268 | })(), | |
| 269 | ); | |
| 270 | return list.length; | |
| 271 | } | |
| 272 | ||
| 273 | // --- The cron: detection and maintenance -------------------------------------------------- | |
| 274 | ||
| 275 | async function afterChecks(env: Env, ctx: { waitUntil(p: Promise<unknown>): void }, observations: Observation[], now: Date): Promise<void> { | |
| 276 | const origin = originOf(env); | |
| 277 | const named = names(env); | |
| 278 | // Maintenance whose window opened or closed. | |
| 279 | for (const m of await dueMaintenance(env.DB, now, origin)) { | |
| 280 | const ended = Date.parse(m.ends_at) <= now.getTime(); | |
| 281 | const text = ended ? "The maintenance is complete." : "The maintenance has begun."; | |
| 282 | const notified = m.notify ? await notify(env, ctx, m.components, { heading: `${ended ? "Completed" : "In progress"}: ${m.title}`, text, url: m.url }) : null; | |
| 283 | await maintenanceUpdate(env.DB, m.id, ended ? "completed" : "in_progress", text, "status", notified, now, { | |
| 284 | action: ended ? "maintenance_completed" : "maintenance_started", | |
| 285 | detail: `${m.title} (on schedule)`, | |
| 286 | }); | |
| 287 | } | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 288 | // Detection. A deploy restarts services: during one, and briefly after, trouble is counted but not drafted. |
| 289 | const [streaks, open, page, deploy] = await Promise.all([loadStreaks(env.DB), openRefs(env.DB), load(env.DB, now, origin), loadDeploy(env.DB)]); | |
| 290 | const quiet = deployQuiet(deploy, now); | |
| 291 | const found = detect(streaks, observations, open, underMaintenance(page.maintenance, now), now, { quiet }); | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 292 | await saveStreaks(env.DB, found.streaks); |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 293 | if (found.held.length) console.log(JSON.stringify({ event: "status.held_for_deploy", parts: found.held, deploy: deploy?.id ?? null })); |
| 294 | const name = (key: string) => named.get(key) ?? key; | |
| 295 | const slowMs = (key: string) => parts(env).find((p) => p.key === key)?.slowMs ?? SLOW_MS; | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 296 | const lines = [ |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 297 | ...found.failing.map((f) => ({ incident: f.incident, kind: "failing" as const, text: troubleSentence(name(f.key), f, stamp(f.since), slowMs(f.key)) })), |
| 298 | ...found.recovered.map((r) => ({ incident: r.incident, kind: "recovered" as const, text: recoverySentence(name(r.key), r, stamp(r.since)) })), | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 299 | ]; |
| 300 | await addSystemLines(env.DB, lines, now); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 301 | const sudo = (id: string) => `${(env.SUDO_URL || "https://sudo.g1t.sh").replace(/\/+$/, "")}/incidents/${id}`; |
| 302 | const alertTo = (env.STATUS_ALERT_EMAIL ?? "").trim(); | |
| 303 | const send = sender(env); | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 304 | if (found.draft.length) { |
| 305 | const core = new Set(parts(env).filter((p) => p.core).map((p) => p.key)); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 306 | const title = draftTitle(found.draft.map((d) => ({ name: name(d.key), state: d.state }))); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 307 | const since = found.draft.map((d) => d.since).sort()[0]!; |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 308 | const said = found.draft.map((d) => troubleSentence(name(d.key), d, stamp(d.since), slowMs(d.key))); |
| 309 | // Trouble that began in a deploy and outlasted it: say so, it is the first thing to rule out. | |
| 310 | const note = deploy && deployQuiet(deploy, new Date(since)) ? `It began during a deploy (started ${stamp(deploy.started_at)}) and outlasted it.` : null; | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 311 | const id = await createIncident( |
| 312 | env.DB, | |
| 313 | { | |
| 314 | title, | |
| 315 | severity: found.draft.some((d) => d.state === "down" && core.has(d.key)) ? "sev2" : "sev3", | |
| 316 | status: "investigating", | |
| 317 | visibility: "draft", | |
| 318 | source: "detected", | |
| 319 | components: found.draft.map((d) => ({ key: d.key, impact: detectedImpact(d.state) })), | |
| 320 | started_at: since, | |
| 321 | acknowledged_at: null, | |
| 322 | commander: null, | |
| 323 | communications: null, | |
| 324 | by: "status", | |
| 325 | }, | |
| 326 | [ | |
| 327 | { | |
| 328 | kind: "detected", | |
| 329 | public: false, | |
| 330 | status: null, | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 331 | text: `${said.join(" ")}${note ? ` ${note}` : ""} Not on the status page until it is published.`, |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 332 | }, |
| 333 | ], | |
| 334 | now, | |
| 335 | { action: "incident_detected", detail: title }, | |
| 336 | ); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 337 | console.warn(JSON.stringify({ event: "status.detected", id, parts: found.draft.map((d) => d.key), since })); |
| 338 | if (alertTo && send) { | |
| 339 | const { text, html } = renderMail(alertLetter({ title, lines: said, link: sudo(id), ...(note ? { note } : {}) })); | |
| 340 | ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${title}`, text, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) })))); | |
| 341 | } | |
| 342 | } | |
| 343 | // Detected drafts no one picked up, whose parts have stayed healthy long enough: dismissed, with a word to staff. | |
| 344 | const troubled = new Set(found.streaks.map((s) => s.component)); | |
| 345 | const settled = settleDrafts(await watchedDrafts(env.DB), troubled, now); | |
| 346 | await saveHealthy(env.DB, settled.healthy); | |
| 347 | for (const d of settled.dismiss) { | |
| 348 | const text = autoDismissText(d.lasted_ms, stamp(d.recovered_at)); | |
| 349 | if (!(await autoDismiss(env.DB, d.id, d.recovered_at, text, now))) continue; | |
| 350 | console.log(JSON.stringify({ event: "status.auto_dismissed", id: d.id, lasted_ms: d.lasted_ms })); | |
| 351 | if (alertTo && send) { | |
| 352 | const letter = recoveredLetter({ title: d.title, text, link: sudo(d.id) }); | |
| 353 | const { text: body, html } = renderMail(letter); | |
| 354 | ctx.waitUntil(send.send({ to: alertTo, subject: `[g1t status] ${letter.heading}`, text: body, html }).catch((e) => console.error(JSON.stringify({ event: "status.alert_failed", error: String(e) })))); | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 355 | } |
| 356 | } | |
| 357 | } | |
| 358 | ||
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 359 | /** |
| 360 | * The deploy tool's word that a deploy started or finished: | |
| 361 | * `POST /deploys` with `Authorization: Bearer <STATUS_DEPLOY_TOKEN>` and | |
| 362 | * `{"phase": "started" | "finished", "id": "<run or commit>"}`. Without | |
| 363 | * the secret set, there is no such address. | |
| 364 | */ | |
| 365 | async function deployHook(request: Request, env: Env): Promise<Response> { | |
| 366 | const json = (body: unknown, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store", ...COMMON } }); | |
| 367 | const token = (env.STATUS_DEPLOY_TOKEN ?? "").trim(); | |
| 368 | if (!token) return json({ error: { code: "not_found", message: "Not found." } }, 404); | |
| 369 | const given = (request.headers.get("authorization") ?? "").replace(/^Bearer\s+/i, "").trim(); | |
| 370 | if (!(await sameSecret(given, token))) return json({ error: { code: "unauthorized", message: "A valid deploy token is required." } }, 401); | |
| 371 | let body: { phase?: unknown; id?: unknown } = {}; | |
| 372 | try { | |
| 373 | body = (await request.json()) as typeof body; | |
| 374 | } catch { | |
| 375 | // Checked below. | |
| 376 | } | |
| 377 | const phase = body.phase === "started" || body.phase === "finished" ? body.phase : null; | |
| 378 | if (!phase) return json({ error: { code: "invalid", message: 'phase must be "started" or "finished".' } }, 400); | |
| 379 | const id = typeof body.id === "string" && body.id.trim() ? body.id.trim().slice(0, 100) : null; | |
| 380 | const now = new Date(); | |
| 381 | const window = deployChange(await loadDeploy(env.DB), phase, id, now); | |
| 382 | await saveDeploy(env.DB, window); | |
| 383 | console.log(JSON.stringify({ event: `status.deploy_${phase}`, id })); | |
| 384 | const quietUntil = window.finished_at ? Date.parse(window.finished_at) + DEPLOY_GRACE_MS : Date.parse(window.started_at) + DEPLOY_MAX_MS; | |
| 385 | return json({ deploy: window, quiet_until: new Date(quietUntil).toISOString() }); | |
| 386 | } | |
| 387 | ||
| 388 | /** Compares two secrets in constant time, by their hashes. */ | |
| 389 | async function sameSecret(a: string, b: string): Promise<boolean> { | |
| 390 | const digest = async (v: string) => new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(v))); | |
| 391 | const [x, y] = await Promise.all([digest(a), digest(b)]); | |
| 392 | let diff = a.length === 0 ? 1 : 0; | |
| 393 | for (let i = 0; i < x.length; i++) diff |= x[i]! ^ y[i]!; | |
| 394 | return diff === 0; | |
| 395 | } | |
| 396 | ||
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 397 | // --- Pages ------------------------------------------------------------------------------- |
| 398 | ||
| 399 | async function model(env: Env, now: Date, origin: string): Promise<PageModel> { | |
| 400 | const stored = await load(env.DB, now, origin); | |
| 401 | return buildPage({ | |
| 402 | parts: parts(env), | |
| 403 | current: stored.current, | |
| 404 | checkedAt: stored.checkedAt, | |
| 405 | days: stored.days, | |
| 406 | incidents: stored.incidents, | |
| 407 | maintenance: stored.maintenance, | |
| 408 | now, | |
| 409 | }); | |
| 410 | } | |
| 411 | ||
| 412 | /** When this isolate last asked for a catch-up round, so a busy page asks once. */ | |
| 413 | let caughtUpAt = 0; | |
| 414 | ||
| 415 | function catchUp(env: Env, ctx: ExecutionContext, page: PageModel, now: Date) { | |
| 416 | const checked = page.report.checked_at ? Date.parse(page.report.checked_at) : 0; | |
| 417 | if (now.getTime() - checked < BEHIND_MS || now.getTime() - caughtUpAt < BEHIND_MS) return; | |
| 418 | caughtUpAt = now.getTime(); | |
| 419 | ctx.waitUntil(checkAll(env, now).catch((error) => console.error(JSON.stringify({ event: "status.catch_up_failed", error: String(error) })))); | |
| 420 | } | |
| 421 | ||
| 422 | const PAGE_POLICY = [ | |
| 423 | "default-src 'none'", | |
| 424 | "script-src 'self'", | |
| 425 | "style-src 'unsafe-inline'", | |
| 426 | "font-src 'self'", | |
| 427 | "img-src 'self' data:", | |
| 428 | "base-uri 'none'", | |
| 429 | "form-action 'self'", | |
| 430 | "frame-ancestors 'none'", | |
| 431 | ].join("; "); | |
| 432 | ||
| 433 | const COMMON = { | |
| 434 | "x-content-type-options": "nosniff", | |
| 435 | "referrer-policy": "strict-origin-when-cross-origin", | |
| 436 | }; | |
| 437 | ||
| 438 | function edgeCache(): Cache | null { | |
| 439 | return (globalThis as unknown as { caches?: { default?: Cache } }).caches?.default ?? null; | |
| 440 | } | |
| 441 | ||
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 442 | /** |
| 443 | * A response from the edge cache, or made and kept there. Pages that say | |
| 444 | * times pass the reader's zone, and are kept once per zone. | |
| 445 | */ | |
| 446 | async function cached(request: Request, ctx: ExecutionContext, make: () => Promise<Response>, zone?: string): Promise<Response> { | |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 447 | const cache = edgeCache(); |
| 448 | const url = new URL(request.url); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 449 | const key = new Request(`${url.origin}${url.pathname}${zone ? `?zone=${encodeURIComponent(zone)}` : ""}`, { method: "GET" }); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 450 | if (cache) { |
| 451 | const hit = await cache.match(key).catch(() => undefined); | |
| 452 | if (hit) return hit; | |
| 453 | } | |
| 454 | const response = await make(); | |
| 455 | if (cache && response.ok) ctx.waitUntil(cache.put(key, response.clone()).catch(() => undefined)); | |
| 456 | return response; | |
| 457 | } | |
| 458 | ||
| 459 | const FONTS: Record<string, ArrayBuffer> = { | |
| 460 | "/fonts/hanken-grotesk.woff2": hanken, | |
| 461 | "/fonts/bricolage-grotesque.woff2": bricolage, | |
| 462 | "/fonts/ibm-plex-mono.woff2": plexMono, | |
| 463 | }; | |
| 464 | ||
| 465 | function html(body: string, cacheControl: string, status = 200): Response { | |
| 466 | return new Response(body, { | |
| 467 | status, | |
| 468 | headers: { "content-type": "text/html; charset=utf-8", "cache-control": cacheControl, "content-security-policy": PAGE_POLICY, ...COMMON }, | |
| 469 | }); | |
| 470 | } | |
| 471 | ||
| 472 | async function form(request: Request): Promise<FormData> { | |
| 473 | try { | |
| 474 | return await request.formData(); | |
| 475 | } catch { | |
| 476 | return new FormData(); | |
| 477 | } | |
| 478 | } | |
| 479 | ||
| 480 | /** Subscribing, confirming and leaving: the page's only writes. */ | |
| 481 | async function subscriptions(request: Request, env: Env, ctx: ExecutionContext, url: URL, options: PageOptions): Promise<Response | null> { | |
| 482 | const path = url.pathname; | |
| 483 | const noStore = "no-store"; | |
| 484 | const message = (title: string, text: string, status = 200, f?: { action: string; fields: Record<string, string>; button: string }) => | |
| 485 | html(renderMessage({ ...options, selfUrl: `${url.origin}${path}` }, { title, text, form: f }), noStore, status); | |
| 486 | const post = request.method === "POST"; | |
| 487 | ||
| 488 | if (path === "/subscribe" && post) { | |
| 489 | if (!emailOn(env)) return message("Email updates are not available", "Follow the Atom or JSON feed instead.", 503); | |
| 490 | const data = await form(request); | |
| 491 | if (String(data.get("website") ?? "")) return message("Check your inbox", "If the address is right, a confirmation link is on its way."); | |
| 492 | const email = normalizeEmail(data.get("email")); | |
| 493 | if (!email) return message("That is not an email address", "Go back and check it.", 400); | |
| 494 | const chosen = chosenParts(data.getAll("components").map(String), parts(env).map((p) => p.key)); | |
| 495 | const token = newToken(); | |
| 496 | const { send } = await requestSubscription(env.DB, email, chosen, await hashToken(token), new Date(), CONFIRM_TTL_MS, RESEND_AFTER_MS); | |
| 497 | if (send) { | |
| 498 | const link = `${url.origin}/subscribe/confirm?token=${encodeURIComponent(token)}`; | |
| 499 | const { text, html: body } = renderMail(confirmLetter(link, chosen ? chosen.map((k) => names(env).get(k) ?? k) : null)); | |
| 500 | ctx.waitUntil(sender(env)!.send({ to: email, subject: "Confirm your subscription to g1t status", text, html: body }).catch((e) => console.error(JSON.stringify({ event: "status.confirm_failed", error: String(e) })))); | |
| 501 | } | |
| 502 | return message("Check your inbox", "If the address is right, a confirmation link is on its way. It works for 24 hours."); | |
| 503 | } | |
| 504 | if (path === "/subscribe/confirm") { | |
| 505 | const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : ""); | |
| 506 | if (!token) return message("That link is incomplete", "Copy the whole link from the email.", 400); | |
| 507 | if (!post) return message("Confirm your subscription", "One more step: confirm to start getting emails about incidents and maintenance.", 200, { action: "/subscribe/confirm", fields: { token }, button: "Confirm subscription" }); | |
| 508 | const done = await confirmSubscription(env.DB, await hashToken(token), new Date()); | |
| 509 | if (!done) return message("That link has expired", "Confirmation links work for 24 hours and once. Subscribe again for a new one.", 410); | |
| 510 | return message("You are subscribed", `${done.email} will get an email when g1t posts an incident or maintenance${done.parts ? ` affecting ${done.parts.map((k) => names(env).get(k) ?? k).join(", ")}` : ""}. Every email has a link to unsubscribe.`); | |
| 511 | } | |
| 512 | if (path === "/unsubscribe") { | |
| 513 | const token = url.searchParams.get("token") ?? (post ? String((await form(request)).get("token") ?? "") : ""); | |
| 514 | const id = env.STATUS_SECRET && token ? await readUnsubscribeToken(env.STATUS_SECRET, token) : null; | |
| 515 | if (!id) return message("That link is not valid", "Use the unsubscribe link at the bottom of any email from g1t status.", 400); | |
| 516 | if (!post) return message("Unsubscribe", "Stop getting emails from g1t status?", 200, { action: "/unsubscribe", fields: { token }, button: "Unsubscribe" }); | |
| 517 | await unsubscribe(env.DB, id); | |
| 518 | return message("You are unsubscribed", "You will not get any more emails from g1t status. You can subscribe again at any time."); | |
| 519 | } | |
| 520 | return null; | |
| 521 | } | |
| 522 | ||
| 523 | async function handle(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> { | |
| 524 | const url = new URL(request.url); | |
| 525 | const path = url.pathname.length > 1 ? url.pathname.replace(/\/+$/, "") : url.pathname; | |
| 526 | if (request.method === "OPTIONS" && (path === "/status.json" || path === "/feed.json")) { | |
| 527 | return new Response(null, { | |
| 528 | status: 204, | |
| 529 | headers: { "access-control-allow-origin": "*", "access-control-allow-methods": "GET, HEAD", "access-control-max-age": "86400" }, | |
| 530 | }); | |
| 531 | } | |
| 532 | const now = new Date(); | |
| 533 | const origin = originOf(env, url); | |
| 534 | const site = env.PUBLIC_SITE_URL || env.SITE_URL || url.origin; | |
| 535 | const options: PageOptions = { | |
| 536 | siteUrl: site, | |
| 537 | supportUrl: env.SUPPORT_URL || `${site}/support`, | |
| 538 | ogImage: env.OG_IMAGE ?? "", | |
| 539 | selfUrl: `${url.origin}${path === "/" ? "/" : path}`, | |
| 540 | now, | |
| 541 | email: emailOn(env), | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 542 | zone: readZone(request.headers.get("cookie"), (request as { cf?: { timezone?: unknown } }).cf?.timezone), |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 543 | }; |
| 544 | ||
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 545 | if (request.method === "POST" && path === "/deploys") return deployHook(request, env); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 546 | if (request.method === "POST") { |
| 547 | const answer = await subscriptions(request, env, ctx, url, options); | |
| 548 | return answer ?? new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD", ...COMMON } }); | |
| 549 | } | |
| 550 | if (request.method !== "GET" && request.method !== "HEAD") { | |
| 551 | return new Response("Method not allowed", { status: 405, headers: { allow: "GET, HEAD, POST", ...COMMON } }); | |
| 552 | } | |
| 553 | const fresh = (cacheSeconds = CACHE_SECONDS) => `public, max-age=${cacheSeconds}`; | |
| 554 | const notFound = () => html(renderMessage(options, { title: "Not found", text: "There is nothing at this address." }), fresh(), 404); | |
| 555 | ||
| 556 | switch (path) { | |
| 557 | case "/": | |
| 558 | return cached(request, ctx, async () => { | |
| 559 | const page = await model(env, now, origin); | |
| 560 | catchUp(env, ctx, page, now); | |
| 561 | return html(renderPage(page, options), fresh()); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 562 | }, options.zone); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 563 | case "/status.json": |
| 564 | return cached(request, ctx, async () => { | |
| 565 | const page = await model(env, now, origin); | |
| 566 | catchUp(env, ctx, page, now); | |
| 567 | return Response.json(page.report, { headers: { "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON } }); | |
| 568 | }); | |
| 569 | case "/badge.svg": | |
| 570 | return cached(request, ctx, async () => { | |
| 571 | const page = await model(env, now, origin); | |
| 572 | return new Response(renderBadge(page.report.overall.state, page.report.overall.title), { | |
| 573 | headers: { "content-type": "image/svg+xml", "cache-control": fresh(), "access-control-allow-origin": "*", ...COMMON }, | |
| 574 | }); | |
| 575 | }); | |
| 576 | case "/history": | |
| 577 | return cached(request, ctx, async () => { | |
| 578 | const since = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 11, 1)); | |
| 579 | const { incidents, maintenance } = await loadHistory(env.DB, since, origin); | |
| 580 | return html(renderHistory(incidents, maintenance, options), fresh()); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 581 | }, options.zone); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 582 | case "/feed.xml": |
| 583 | case "/feed.json": | |
| 584 | return cached(request, ctx, async () => { | |
| 585 | const { incidents, maintenance } = await loadHistory(env.DB, new Date(now.getTime() - 365 * 86_400_000), origin); | |
| 586 | const items = feedItems(incidents, maintenance); | |
| 587 | const feed = { origin, title: "g1t status", updated: now.toISOString() }; | |
| 588 | const headers = { "cache-control": fresh(60), "access-control-allow-origin": "*", ...COMMON }; | |
| 589 | return path === "/feed.xml" | |
| 590 | ? new Response(atom(items, feed), { headers: { "content-type": "application/atom+xml; charset=utf-8", ...headers } }) | |
| 591 | : Response.json(jsonFeed(items, feed), { headers: { "content-type": "application/feed+json; charset=utf-8", ...headers } }); | |
| 592 | }); | |
| 593 | case "/subscribe": | |
| 594 | return html(renderSubscribe(options, parts(env).map(({ key, name }) => ({ key, name }))), fresh(300)); | |
| 595 | case "/subscribe/confirm": | |
| 596 | case "/unsubscribe": | |
| 597 | return (await subscriptions(request, env, ctx, url, options))!; | |
| 598 | case "/status.js": | |
| 599 | return new Response(SCRIPT, { headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": fresh(3600), ...COMMON } }); | |
| 600 | case "/favicon.svg": | |
| 601 | case "/favicon.ico": | |
| 602 | return new Response(FAVICON, { headers: { "content-type": "image/svg+xml", "cache-control": fresh(86400), ...COMMON } }); | |
| 603 | case "/robots.txt": | |
| 604 | return new Response("User-agent: *\nAllow: /\nDisallow: /subscribe/confirm\nDisallow: /unsubscribe\n", { | |
| 605 | headers: { "content-type": "text/plain", "cache-control": fresh(86400) }, | |
| 606 | }); | |
| 607 | } | |
| 608 | const incident = /^\/incidents\/([a-z0-9-]{1,64})$/.exec(path); | |
| 609 | if (incident) { | |
| 610 | return cached(request, ctx, async () => { | |
| 611 | const found = await loadPublicIncident(env.DB, incident[1]!, origin); | |
| 612 | return found ? html(renderIncident(found.incident, found.postmortem, names(env), options), fresh()) : notFound(); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 613 | }, options.zone); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 614 | } |
| 615 | const maintenance = /^\/maintenance\/([a-z0-9-]{1,64})$/.exec(path); | |
| 616 | if (maintenance) { | |
| 617 | return cached(request, ctx, async () => { | |
| 618 | const found = await loadPublicMaintenance(env.DB, maintenance[1]!, origin); | |
| 619 | return found ? html(renderMaintenance(found, names(env), options), fresh()) : notFound(); | |
| Fast pages, required checks on the branch, self-hosted runners, honest incidents | 620 | }, options.zone); |
| status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas | 621 | } |
| 622 | const font = FONTS[path]; | |
| 623 | if (font) { | |
| 624 | return new Response(font, { headers: { "content-type": "font/woff2", "cache-control": "public, max-age=31536000, immutable", ...COMMON } }); | |
| 625 | } | |
| 626 | return notFound(); | |
| 627 | } | |
| 628 | ||
| 629 | export default { | |
| 630 | async fetch(request, env, ctx) { | |
| 631 | try { | |
| 632 | return await handle(request, env, ctx); | |
| 633 | } catch (error) { | |
| 634 | console.error(JSON.stringify({ event: "status.failed", path: new URL(request.url).pathname, error: String(error) })); | |
| 635 | return new Response("The status page could not be drawn. Try again in a minute.", { | |
| 636 | status: 503, | |
| 637 | headers: { "content-type": "text/plain; charset=utf-8", "retry-after": "60", ...COMMON }, | |
| 638 | }); | |
| 639 | } | |
| 640 | }, | |
| 641 | async scheduled(_controller, env, ctx) { | |
| 642 | const now = new Date(); | |
| 643 | ctx.waitUntil( | |
| 644 | checkAll(env, now) | |
| 645 | .then(async (observations) => { | |
| 646 | const failing = observations.filter((o) => o.state === "down" || o.state === "degraded"); | |
| 647 | if (failing.length) console.warn(JSON.stringify({ event: "status.trouble", parts: failing })); | |
| 648 | await afterChecks(env, ctx, observations, now); | |
| 649 | }) | |
| 650 | .catch((error) => console.error(JSON.stringify({ event: "status.cron_failed", error: String(error) }))), | |
| 651 | ); | |
| 652 | }, | |
| 653 | } satisfies ExportedHandler<Env>; | |
| 654 | ||
| 655 | // --- Staff --------------------------------------------------------------------------------- | |
| 656 | ||
| 657 | /** | |
| 658 | * Staff only: running incidents and maintenance. Reached only through a | |
| 659 | * service binding (sudo's `STATUS`); status.g1t.sh's own address cannot. | |
| 660 | */ | |
| 661 | export class StatusAdmin extends WorkerEntrypoint<Env> implements StatusAdminApi { | |
| 662 | private known() { | |
| 663 | return parts(this.env).map((p) => p.key); | |
| 664 | } | |
| 665 | ||
| 666 | private origin() { | |
| 667 | return originOf(this.env); | |
| 668 | } | |
| 669 | ||
| 670 | private async detail(id: string): Promise<AdminIncidentDetail | null> { | |
| 671 | return incidentDetail(this.env.DB, String(id), this.origin(), names(this.env)); | |
| 672 | } | |
| 673 | ||
| 674 | private async summary(id: string): Promise<AdminIncident> { | |
| 675 | const { timeline: _t, followups: _f, postmortem: _p, postmortem_draft: _d, url: _u, ...incident } = (await this.detail(id))!; | |
| 676 | return incident; | |
| 677 | } | |
| 678 | ||
| 679 | /** Emails a public update to subscribers when asked; the count to keep with it. */ | |
| 680 | private async announce(incident: { id: string; title: string; components: { key: string; impact: string }[] }, status: keyof typeof INCIDENT_STATUS, text: string, wanted: boolean) { | |
| 681 | if (!wanted) return null; | |
| 682 | const about = incident.components.filter((c) => c.impact !== "operational").map((c) => c.key); | |
| 683 | return notify(this.env, this.ctx, about, { | |
| 684 | heading: `${INCIDENT_STATUS[status]}: ${incident.title}`, | |
| 685 | text, | |
| 686 | url: `${this.origin()}/incidents/${incident.id}`, | |
| 687 | }); | |
| 688 | } | |
| 689 | ||
| 690 | async components(): Promise<{ key: string; name: string }[]> { | |
| 691 | return parts(this.env).map(({ key, name }) => ({ key, name })); | |
| 692 | } | |
| 693 | ||
| 694 | async board(): Promise<StatusBoard> { | |
| 695 | return { ...(await board(this.env.DB, new Date(), this.origin())), email: emailOn(this.env) }; | |
| 696 | } | |
| 697 | ||
| 698 | async incident(id: string): Promise<AdminIncidentDetail | null> { | |
| 699 | return this.detail(id); | |
| 700 | } | |
| 701 | ||
| 702 | async openCount(): Promise<number> { | |
| 703 | return openCount(this.env.DB); | |
| 704 | } | |
| 705 | ||
| 706 | async declare(input: DeclareIncident): Promise<Result<AdminIncident>> { | |
| 707 | const checked = checkDeclare(input, this.known()); | |
| 708 | if (!checked.ok) return fail("invalid", checked.error); | |
| 709 | const v = checked.value; | |
| 710 | const now = new Date(); | |
| 711 | const entries: (Entry & { notified?: number | null })[] = [ | |
| 712 | { kind: "declared", public: false, status: null, text: `Declared ${SEVERITY_LABEL[v.severity]}.` }, | |
| 713 | ]; | |
| 714 | if (v.commander) entries.push({ kind: "role", public: false, status: null, text: `Incident commander: ${v.commander}.` }); | |
| 715 | if (v.communications) entries.push({ kind: "role", public: false, status: null, text: `Communications: ${v.communications}.` }); | |
| 716 | const id = shortId(); | |
| 717 | const status = v.status ?? "investigating"; | |
| 718 | const notified = await this.announce({ id, title: v.title, components: v.components }, status, v.message, v.notify); | |
| 719 | entries.push({ kind: "update", public: true, status, text: v.message, notified }); | |
| 720 | await createIncident( | |
| 721 | this.env.DB, | |
| 722 | { | |
| 723 | title: v.title, | |
| 724 | severity: v.severity, | |
| 725 | status, | |
| 726 | visibility: "public", | |
| 727 | source: "declared", | |
| 728 | components: v.components, | |
| 729 | started_at: v.started_at ?? now.toISOString(), | |
| 730 | acknowledged_at: now.toISOString(), | |
| 731 | commander: v.commander ?? null, | |
| 732 | communications: v.communications ?? null, | |
| 733 | by: v.by, | |
| 734 | }, | |
| 735 | entries, | |
| 736 | now, | |
| 737 | { action: "incident_declared", detail: `${SEVERITY_LABEL[v.severity]}: ${v.title}` }, | |
| 738 | id, | |
| 739 | ); | |
| 740 | console.log(JSON.stringify({ event: "status.incident_declared", id, by: v.by })); | |
| 741 | return ok(await this.summary(id)); | |
| 742 | } | |
| 743 | ||
| 744 | async update(id: string, change: IncidentChange): Promise<Result<AdminIncident>> { | |
| 745 | const checked = checkChange(change, this.known()); | |
| 746 | if (!checked.ok) return fail("invalid", checked.error); | |
| 747 | const existing = await this.detail(id); | |
| 748 | if (!existing) return fail("not_found", "No such incident."); | |
| 749 | const now = new Date(); | |
| 750 | const applied = applyChange(facts(existing), checked.value, now, names(this.env)); | |
| 751 | if (!applied.ok) return fail("invalid", applied.error); | |
| 752 | const { next, entries } = applied.value; | |
| 753 | const update = entries.find((e) => e.kind === "update"); | |
| 754 | const notified = update | |
| 755 | ? await this.announce({ id: existing.id, title: existing.title, components: next.components }, next.status, update.text, checked.value.notify === true) | |
| 756 | : null; | |
| 757 | const lines = entries.map((e) => (e === update ? { ...e, notified } : e)); | |
| 758 | const action = next.status === "resolved" && existing.status !== "resolved" ? "incident_resolved" : update ? "incident_update" : "incident_note"; | |
| 759 | await saveIncident(this.env.DB, existing.id, next, lines, now, checked.value.by, { | |
| 760 | action, | |
| 761 | detail: entries.map((e) => (e.kind === "update" || e.kind === "note" ? `${e.kind === "update" ? "Public" : "Note"}: ${e.text}` : e.text)).join(" ").slice(0, 500), | |
| 762 | }); | |
| 763 | return ok(await this.summary(existing.id)); | |
| 764 | } | |
| 765 | ||
| 766 | async roles(id: string, change: RolesChange): Promise<Result<AdminIncident>> { | |
| 767 | const checked = checkRoles(change); | |
| 768 | if (!checked.ok) return fail("invalid", checked.error); | |
| 769 | const existing = await this.detail(id); | |
| 770 | if (!existing) return fail("not_found", "No such incident."); | |
| 771 | const now = new Date(); | |
| 772 | const { next, entries } = applyRoles(facts(existing), checked.value, now); | |
| 773 | if (!entries.length) return ok(await this.summary(existing.id)); | |
| 774 | await saveIncident(this.env.DB, existing.id, next, entries, now, checked.value.by, { action: "incident_roles", detail: entries.map((e) => e.text).join(" ") }); | |
| 775 | return ok(await this.summary(existing.id)); | |
| 776 | } | |
| 777 | ||
| 778 | async publish(id: string, input: PublishIncident): Promise<Result<AdminIncident>> { | |
| 779 | const checked = checkPublish(input); | |
| 780 | if (!checked.ok) return fail("invalid", checked.error); | |
| 781 | const existing = await this.detail(id); | |
| 782 | if (!existing) return fail("not_found", "No such incident."); | |
| 783 | if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be published."); | |
| 784 | const now = new Date(); | |
| 785 | const at = now.toISOString(); | |
| 786 | const title = checked.value.title ?? existing.title; | |
| 787 | const next = { ...facts(existing), visibility: "public" as const, acknowledged_at: existing.acknowledged_at ?? at, title, published_at: at }; | |
| 788 | const notified = await this.announce({ id: existing.id, title, components: existing.components }, existing.status, checked.value.message, checked.value.notify); | |
| 789 | await saveIncident( | |
| 790 | this.env.DB, | |
| 791 | existing.id, | |
| 792 | next, | |
| 793 | [ | |
| 794 | ...(existing.acknowledged_at ? [] : [{ kind: "acknowledged" as const, public: false, status: null, text: "Acknowledged." }]), | |
| 795 | { kind: "published", public: false, status: null, text: title !== existing.title ? `Published as “${title}”.` : "Published to the status page." }, | |
| 796 | { kind: "update", public: true, status: existing.status, text: checked.value.message, notified }, | |
| 797 | ], | |
| 798 | now, | |
| 799 | checked.value.by, | |
| 800 | { action: "incident_published", detail: title }, | |
| 801 | ); | |
| 802 | return ok(await this.summary(existing.id)); | |
| 803 | } | |
| 804 | ||
| 805 | async dismiss(id: string, input: { reason: string; by: string }): Promise<Result<AdminIncident>> { | |
| 806 | const existing = await this.detail(id); | |
| 807 | if (!existing) return fail("not_found", "No such incident."); | |
| 808 | if (existing.visibility !== "draft") return fail("conflict", "Only a draft can be dismissed; resolve a published incident instead."); | |
| 809 | const by = String(input?.by ?? "").trim(); | |
| 810 | if (!by) return fail("invalid", "Who is making the change is missing."); | |
| 811 | const reason = String(input?.reason ?? "").trim().slice(0, 500) || "No reason given."; | |
| 812 | const now = new Date(); | |
| 813 | const at = now.toISOString(); | |
| 814 | const next = { ...facts(existing), visibility: "dismissed" as const, status: "resolved" as const, acknowledged_at: existing.acknowledged_at ?? at, resolved_at: at }; | |
| 815 | await saveIncident(this.env.DB, existing.id, next, [{ kind: "dismissed", public: false, status: null, text: `Dismissed: ${reason}` }], now, by, { | |
| 816 | action: "incident_dismissed", | |
| 817 | detail: `${existing.title}: ${reason}`, | |
| 818 | }); | |
| 819 | return ok(await this.summary(existing.id)); | |
| 820 | } | |
| 821 | ||
| 822 | async addFollowUp(id: string, input: { title: string; owner: string | null; by: string }): Promise<Result<FollowUp>> { | |
| 823 | const checked = checkFollowUp(input); | |
| 824 | if (!checked.ok) return fail("invalid", checked.error); | |
| 825 | if (!(await this.detail(id))) return fail("not_found", "No such incident."); | |
| 826 | return ok(await addFollowUp(this.env.DB, String(id), checked.value, new Date())); | |
| 827 | } | |
| 828 | ||
| 829 | async setFollowUp(id: string, followUp: string, input: { done: boolean; by: string }): Promise<Result<FollowUp>> { | |
| 830 | const by = String(input?.by ?? "").trim(); | |
| 831 | if (!by) return fail("invalid", "Who is making the change is missing."); | |
| 832 | const done = await setFollowUp(this.env.DB, String(id), String(followUp), input.done === true, by, new Date()); | |
| 833 | return done ? ok(done) : fail("not_found", "No such follow-up."); | |
| 834 | } | |
| 835 | ||
| 836 | async savePostmortem(id: string, input: PostmortemFields & { by: string }): Promise<Result<Postmortem>> { | |
| 837 | const checked = checkPostmortem(input); | |
| 838 | if (!checked.ok) return fail("invalid", checked.error); | |
| 839 | const existing = await this.detail(id); | |
| 840 | if (!existing) return fail("not_found", "No such incident."); | |
| 841 | if (existing.visibility !== "public") return fail("conflict", "Only a published incident has a postmortem."); | |
| 842 | const { by, ...fields } = checked.value; | |
| 843 | await savePostmortem(this.env.DB, existing.id, fields, by, new Date()); | |
| 844 | return ok((await this.detail(existing.id))!.postmortem!); | |
| 845 | } | |
| 846 | ||
| 847 | async publishPostmortem(id: string, input: { publish: boolean; by: string }): Promise<Result<Postmortem>> { | |
| 848 | const by = String(input?.by ?? "").trim(); | |
| 849 | if (!by) return fail("invalid", "Who is making the change is missing."); | |
| 850 | const existing = await this.detail(id); | |
| 851 | if (!existing) return fail("not_found", "No such incident."); | |
| 852 | if (!existing.postmortem) return fail("conflict", "Save the postmortem before publishing it."); | |
| 853 | if (input.publish) { | |
| 854 | if (!existing.resolved_at) return fail("conflict", "Resolve the incident before publishing its postmortem."); | |
| 855 | const missing = postmortemReady(existing.postmortem); | |
| 856 | if (missing) return fail("invalid", missing); | |
| 857 | } | |
| 858 | await publishPostmortem(this.env.DB, existing.id, input.publish === true, by, new Date()); | |
| 859 | return ok((await this.detail(existing.id))!.postmortem!); | |
| 860 | } | |
| 861 | ||
| 862 | async scheduleMaintenance(input: NewMaintenance): Promise<Result<AdminMaintenance>> { | |
| 863 | const checked = checkMaintenance(input, this.known()); | |
| 864 | if (!checked.ok) return fail("invalid", checked.error); | |
| 865 | const v = checked.value; | |
| 866 | const now = new Date(); | |
| 867 | const id = shortId(); | |
| 868 | const notified = v.notify | |
| 869 | ? await notify(this.env, this.ctx, v.components, { | |
| 870 | heading: `Planned maintenance: ${v.title}`, | |
| 871 | text: `${v.message}\n\nWhen: ${stamp(v.starts_at)} to ${stamp(v.ends_at)}.`, | |
| 872 | url: maintenanceUrl(this.origin(), id), | |
| 873 | }) | |
| 874 | : null; | |
| 875 | const made = await scheduleMaintenance(this.env.DB, v, notified, now, id); | |
| 876 | return ok((await maintenanceById(this.env.DB, made, this.origin()))!); | |
| 877 | } | |
| 878 | ||
| 879 | async changeMaintenance(id: string, change: MaintenanceChange): Promise<Result<AdminMaintenance>> { | |
| 880 | const checked = checkMaintenanceChange(change); | |
| 881 | if (!checked.ok) return fail("invalid", checked.error); | |
| 882 | const existing = await maintenanceById(this.env.DB, String(id), this.origin()); | |
| 883 | if (!existing) return fail("not_found", "No such maintenance."); | |
| 884 | const v = checked.value; | |
| 885 | if (existing.state === "completed" || existing.state === "cancelled") return fail("conflict", `This maintenance is ${existing.state}.`); | |
| 886 | if (v.action === "start" && existing.state !== "scheduled") return fail("conflict", "It has already started."); | |
| 887 | const state = v.action === "start" ? "in_progress" : v.action === "complete" ? "completed" : v.action === "cancel" ? "cancelled" : null; | |
| 888 | const text = | |
| 889 | v.message || | |
| 890 | (v.action === "start" ? "The maintenance has begun." : v.action === "complete" ? "The maintenance is complete." : "This maintenance is cancelled."); | |
| 891 | const word = { update: "Update", start: "In progress", complete: "Completed", cancel: "Cancelled" }[v.action]; | |
| 892 | const notified = v.notify ? await notify(this.env, this.ctx, existing.components, { heading: `${word}: ${existing.title}`, text, url: existing.url }) : null; | |
| 893 | await maintenanceUpdate(this.env.DB, existing.id, state, text, v.by, notified, new Date(), { | |
| 894 | action: `maintenance_${v.action === "update" ? "update" : v.action === "start" ? "started" : v.action === "complete" ? "completed" : "cancelled"}`, | |
| 895 | detail: `${existing.title}: ${text}`, | |
| 896 | }); | |
| 897 | return ok((await maintenanceById(this.env.DB, existing.id, this.origin()))!); | |
| 898 | } | |
| 899 | ||
| 900 | async audit(filter?: { before?: string | null }): Promise<StatusAuditEntry[]> { | |
| 901 | const before = filter?.before && !Number.isNaN(Date.parse(filter.before)) ? filter.before : null; | |
| 902 | return auditLog(this.env.DB, before); | |
| 903 | } | |
| 904 | } | |
| 905 |