| 1 | /** |
| 2 | * What a check's result means, how the parts add up to one line for the |
| 3 | * whole, and the 90 days of history behind each uptime bar. No Workers |
| 4 | * imports, so it is tested under Node. |
| 5 | */ |
| 6 | import type { |
| 7 | ComponentImpact, |
| 8 | IncidentImpact, |
| 9 | IncidentStatus, |
| 10 | MaintenanceState, |
| 11 | StatusComponent, |
| 12 | StatusComponentState, |
| 13 | StatusIncident, |
| 14 | StatusMaintenance, |
| 15 | StatusOverall, |
| 16 | StatusOverallState, |
| 17 | StatusReport, |
| 18 | } from "@g1t/contracts"; |
| 19 | |
| 20 | import type { ComponentInfo } from "./components.ts"; |
| 21 | import type { ProbeResult } from "./probe.ts"; |
| 22 | |
| 23 | /** Slower than this, a part that answered counts as degraded. */ |
| 24 | export const SLOW_MS = 1500; |
| 25 | /** How many days of history the page shows and keeps. */ |
| 26 | export const HISTORY_DAYS = 90; |
| 27 | /** Checks older than this mean the checker has stopped: the page says so. */ |
| 28 | export const STALE_MS = 10 * 60 * 1000; |
| 29 | |
| 30 | /** What a check's result means for its part. `null` is a part with no check. */ |
| 31 | export function classify(result: ProbeResult | null, slowMs = SLOW_MS): { state: StatusComponentState; detail: string } { |
| 32 | if (!result) return { state: "unmonitored", detail: "Not monitored yet" }; |
| 33 | if (!result.ok) return { state: "down", detail: `Failed: ${result.error ?? "no answer"}` }; |
| 34 | const ms = Math.round(result.ms); |
| 35 | if (ms > slowMs) return { state: "degraded", detail: `Slow: answered in ${ms} ms` }; |
| 36 | return { state: "up", detail: `Answered in ${ms} ms` }; |
| 37 | } |
| 38 | |
| 39 | /** A part's state at its last check, as stored. */ |
| 40 | export type Current = { state: StatusComponentState; detail: string; latency_ms: number | null }; |
| 41 | |
| 42 | /** One part's checks on one UTC day, as stored. */ |
| 43 | export type DayRow = { |
| 44 | component: string; |
| 45 | /** `YYYY-MM-DD`, UTC. */ |
| 46 | day: string; |
| 47 | checks: number; |
| 48 | up: number; |
| 49 | degraded: number; |
| 50 | down: number; |
| 51 | latency_total: number; |
| 52 | latency_count: number; |
| 53 | }; |
| 54 | |
| 55 | /** One day's square on a bar. */ |
| 56 | export type DayBar = { |
| 57 | day: string; |
| 58 | state: "up" | "degraded" | "down" | "none"; |
| 59 | /** Share of checks that answered, 0 to 100; null with none. */ |
| 60 | uptime: number | null; |
| 61 | checks: number; |
| 62 | failed: number; |
| 63 | slow: number; |
| 64 | avg_ms: number | null; |
| 65 | /** Incidents on this part that were open some time that day. */ |
| 66 | incidents: string[]; |
| 67 | }; |
| 68 | |
| 69 | const DAY_MS = 24 * 60 * 60 * 1000; |
| 70 | |
| 71 | /** `YYYY-MM-DD` of a time, in UTC. */ |
| 72 | export function dayOf(at: Date | number): string { |
| 73 | return new Date(at).toISOString().slice(0, 10); |
| 74 | } |
| 75 | |
| 76 | /** The last `count` days, oldest first, ending today. */ |
| 77 | export function lastDays(now: Date, count = HISTORY_DAYS): string[] { |
| 78 | const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); |
| 79 | return Array.from({ length: count }, (_, i) => dayOf(today - (count - 1 - i) * DAY_MS)); |
| 80 | } |
| 81 | |
| 82 | /** |
| 83 | * Share of checks answered, rounded down to hundredths so a single failure |
| 84 | * never shows as 100%. |
| 85 | */ |
| 86 | export function uptimeOf(answered: number, checks: number): number | null { |
| 87 | if (checks <= 0) return null; |
| 88 | return Math.floor((answered / checks) * 10000) / 100; |
| 89 | } |
| 90 | |
| 91 | /** Words for an uptime: "100%", "99.93%". */ |
| 92 | export function percent(uptime: number | null): string { |
| 93 | if (uptime == null) return "No data"; |
| 94 | return uptime >= 100 ? "100%" : `${uptime.toFixed(2)}%`; |
| 95 | } |
| 96 | |
| 97 | const LEVEL = { none: 0, up: 1, degraded: 2, down: 3 } as const; |
| 98 | |
| 99 | /** |
| 100 | * A day's colour: answered at least 99.9% of the time is up (one blip in a |
| 101 | * day of minutely checks), unless over a quarter of answers were slow; at |
| 102 | * least 95% is degraded; less is down. An incident on the part that day |
| 103 | * colours it at least as badly as its impact. |
| 104 | */ |
| 105 | export function dayState(row: Pick<DayRow, "checks" | "up" | "degraded" | "down"> | null, incidentImpact: IncidentImpact | null = null): DayBar["state"] { |
| 106 | let state: DayBar["state"] = "none"; |
| 107 | if (row && row.checks > 0) { |
| 108 | const uptime = ((row.up + row.degraded) / row.checks) * 100; |
| 109 | if (uptime >= 99.9) state = row.degraded / row.checks > 0.25 ? "degraded" : "up"; |
| 110 | else if (uptime >= 95) state = "degraded"; |
| 111 | else state = "down"; |
| 112 | } |
| 113 | if (incidentImpact && LEVEL[incidentImpact] > LEVEL[state]) state = incidentImpact; |
| 114 | return state; |
| 115 | } |
| 116 | |
| 117 | /** Whether an incident was open at some time during `day`. */ |
| 118 | export function openOn(incident: Pick<StatusIncident, "started_at" | "resolved_at">, day: string): boolean { |
| 119 | const start = Date.parse(`${day}T00:00:00Z`); |
| 120 | const end = start + DAY_MS; |
| 121 | const began = Date.parse(incident.started_at); |
| 122 | const ended = incident.resolved_at ? Date.parse(incident.resolved_at) : Number.POSITIVE_INFINITY; |
| 123 | return began < end && ended >= start; |
| 124 | } |
| 125 | |
| 126 | /** A part's bar: one square per day, oldest first. */ |
| 127 | export function barFor( |
| 128 | key: string, |
| 129 | rows: DayRow[], |
| 130 | incidents: Pick<StatusIncident, "title" | "started_at" | "resolved_at" | "component_impacts">[], |
| 131 | days: string[], |
| 132 | ): DayBar[] { |
| 133 | const byDay = new Map(rows.filter((r) => r.component === key).map((r) => [r.day, r])); |
| 134 | const mine = incidents |
| 135 | .map((i) => ({ incident: i, impact: i.component_impacts.find((c) => c.key === key)?.impact ?? null })) |
| 136 | .filter((m) => m.impact != null && m.impact !== "operational"); |
| 137 | return days.map((day) => { |
| 138 | const row = byDay.get(day) ?? null; |
| 139 | const during = mine.filter((m) => openOn(m.incident, day)); |
| 140 | const impact = during.some((m) => m.impact === "major_outage") ? "down" : during.length ? "degraded" : null; |
| 141 | return { |
| 142 | day, |
| 143 | state: dayState(row, impact), |
| 144 | uptime: row ? uptimeOf(row.up + row.degraded, row.checks) : null, |
| 145 | checks: row?.checks ?? 0, |
| 146 | failed: row?.down ?? 0, |
| 147 | slow: row?.degraded ?? 0, |
| 148 | avg_ms: row && row.latency_count > 0 ? Math.round(row.latency_total / row.latency_count) : null, |
| 149 | incidents: during.map((m) => m.incident.title), |
| 150 | }; |
| 151 | }); |
| 152 | } |
| 153 | |
| 154 | /** A part's uptime over the days given. */ |
| 155 | export function uptimeOver(key: string, rows: DayRow[]): number | null { |
| 156 | let checks = 0; |
| 157 | let answered = 0; |
| 158 | for (const row of rows) { |
| 159 | if (row.component !== key) continue; |
| 160 | checks += row.checks; |
| 161 | answered += row.up + row.degraded; |
| 162 | } |
| 163 | return uptimeOf(answered, checks); |
| 164 | } |
| 165 | |
| 166 | |
| 167 | const RANK: Record<StatusOverallState, number> = { unknown: 0, up: 1, maintenance: 1.5, degraded: 2, down: 3 }; |
| 168 | |
| 169 | function worse(a: StatusOverallState, b: StatusOverallState): StatusOverallState { |
| 170 | return RANK[b] > RANK[a] ? b : a; |
| 171 | } |
| 172 | |
| 173 | /** Words for an incident's status. */ |
| 174 | export const INCIDENT_STATUS: Record<IncidentStatus, string> = { |
| 175 | investigating: "Investigating", |
| 176 | identified: "Identified", |
| 177 | monitoring: "Monitoring", |
| 178 | resolved: "Resolved", |
| 179 | }; |
| 180 | |
| 181 | /** Words for what an incident does to a part. */ |
| 182 | export const IMPACT_WORD: Record<ComponentImpact, string> = { |
| 183 | operational: "Operational", |
| 184 | degraded: "Degraded performance", |
| 185 | partial_outage: "Partial outage", |
| 186 | major_outage: "Major outage", |
| 187 | }; |
| 188 | |
| 189 | /** A part's state for an impact on it. */ |
| 190 | export const IMPACT_STATE: Record<ComponentImpact, StatusComponentState> = { |
| 191 | operational: "up", |
| 192 | degraded: "degraded", |
| 193 | partial_outage: "partial", |
| 194 | major_outage: "down", |
| 195 | }; |
| 196 | |
| 197 | /** How bad each state is: the worse of a check and an incident wins. */ |
| 198 | const STATE_RANK: Record<StatusComponentState, number> = { unmonitored: 0, up: 1, maintenance: 1, degraded: 2, partial: 3, down: 4 }; |
| 199 | |
| 200 | /** An incident's impact as a whole: `down` when any part has a major outage. */ |
| 201 | export function overallImpact(impacts: { impact: ComponentImpact }[]): IncidentImpact { |
| 202 | return impacts.some((i) => i.impact === "major_outage") ? "down" : "degraded"; |
| 203 | } |
| 204 | |
| 205 | /** |
| 206 | * What the page shows for a part: the worse of its last check and what |
| 207 | * open incidents say about it. During maintenance it is "Under |
| 208 | * maintenance", unless an incident says worse, since checks failing in a |
| 209 | * window are expected. |
| 210 | */ |
| 211 | export function effectiveState(check: StatusComponentState, impacts: ComponentImpact[], underMaintenance: boolean): StatusComponentState { |
| 212 | let reported: StatusComponentState | null = null; |
| 213 | for (const impact of impacts) { |
| 214 | const state = IMPACT_STATE[impact]; |
| 215 | if (reported == null || STATE_RANK[state] > STATE_RANK[reported]) reported = state; |
| 216 | } |
| 217 | if (underMaintenance && (reported == null || reported === "up")) return "maintenance"; |
| 218 | if (reported == null) return check; |
| 219 | if (check === "unmonitored" || check === "maintenance") return reported; |
| 220 | return STATE_RANK[reported] > STATE_RANK[check] ? reported : check; |
| 221 | } |
| 222 | |
| 223 | /** |
| 224 | * Where maintenance stands at `now`. Cancelled and completed are final; |
| 225 | * otherwise the window decides, so the page is right even if the |
| 226 | * minutely job that moves it along is late. |
| 227 | */ |
| 228 | export function maintenanceState(m: Pick<StatusMaintenance, "state" | "starts_at" | "ends_at">, now: Date): MaintenanceState { |
| 229 | if (m.state === "cancelled" || m.state === "completed") return m.state; |
| 230 | const t = now.getTime(); |
| 231 | if (t >= Date.parse(m.ends_at)) return "completed"; |
| 232 | if (t >= Date.parse(m.starts_at) || m.state === "in_progress") return "in_progress"; |
| 233 | return "scheduled"; |
| 234 | } |
| 235 | |
| 236 | /** The parts under maintenance at `now`. */ |
| 237 | export function underMaintenance( |
| 238 | list: Pick<StatusMaintenance, "state" | "starts_at" | "ends_at" | "components">[], |
| 239 | now: Date, |
| 240 | ): Set<string> { |
| 241 | const keys = new Set<string>(); |
| 242 | for (const m of list) if (maintenanceState(m, now) === "in_progress") for (const k of m.components) keys.add(k); |
| 243 | return keys; |
| 244 | } |
| 245 | |
| 246 | /** |
| 247 | * The whole, from its parts and its open incidents: a major outage when a |
| 248 | * core part is down or an incident says so; a partial outage when any part |
| 249 | * is down or partly down, or an incident is open; degraded performance |
| 250 | * when parts are only slow; under maintenance when work is under way and |
| 251 | * nothing else is wrong. Parts with no check never make it look better or |
| 252 | * worse. |
| 253 | */ |
| 254 | export function summarize( |
| 255 | parts: Pick<StatusComponent, "key" | "name" | "state">[], |
| 256 | core: Set<string>, |
| 257 | open: Pick<StatusIncident, "title" | "impact" | "status">[] = [], |
| 258 | maintenance: Pick<StatusMaintenance, "title">[] = [], |
| 259 | ): StatusOverall { |
| 260 | const monitored = parts.filter((c) => c.state !== "unmonitored"); |
| 261 | if (monitored.length === 0 && open.length === 0) { |
| 262 | return { state: "unknown", title: "Status unavailable", line: "No checks have run yet." }; |
| 263 | } |
| 264 | let state: StatusOverallState = monitored.length > 0 ? "up" : "unknown"; |
| 265 | for (const c of monitored) { |
| 266 | if (c.state === "down") state = worse(state, core.has(c.key) ? "down" : "degraded"); |
| 267 | else if (c.state === "partial" || c.state === "degraded") state = worse(state, "degraded"); |
| 268 | else if (c.state === "maintenance") state = worse(state, "maintenance"); |
| 269 | } |
| 270 | for (const incident of open) state = worse(state, incident.impact); |
| 271 | |
| 272 | const down = monitored.filter((c) => c.state === "down" || c.state === "partial").map((c) => c.name); |
| 273 | const slow = monitored.filter((c) => c.state === "degraded").map((c) => c.name); |
| 274 | const lead = open[0]; |
| 275 | const incidentLine = lead ? `${INCIDENT_STATUS[lead.status]}: ${lead.title}.` : null; |
| 276 | const named = (names: string[]) => names.join(", "); |
| 277 | |
| 278 | if (state === "up") return { state, title: "All systems normal", line: "Every part of g1t answered its last check." }; |
| 279 | if (state === "maintenance") { |
| 280 | const work = maintenance[0]; |
| 281 | return { state, title: "Under maintenance", line: work ? `Planned work: ${work.title}.` : "Planned work is under way." }; |
| 282 | } |
| 283 | if (state === "down") { |
| 284 | return { |
| 285 | state, |
| 286 | title: "Major outage", |
| 287 | line: incidentLine ?? (down.length ? `Not answering: ${named(down)}.` : "g1t is not working right now."), |
| 288 | }; |
| 289 | } |
| 290 | if (state === "degraded") { |
| 291 | if (down.length || open.length) { |
| 292 | return { |
| 293 | state, |
| 294 | title: "Partial outage", |
| 295 | line: incidentLine ?? `Not answering: ${named(down)}${slow.length ? `. Slow: ${named(slow)}` : ""}.`, |
| 296 | }; |
| 297 | } |
| 298 | return { state, title: "Degraded performance", line: `Slow to answer: ${named(slow)}.` }; |
| 299 | } |
| 300 | return { state, title: "Status unavailable", line: "No checks have run yet." }; |
| 301 | } |
| 302 | |
| 303 | /** Open incidents, and those resolved within `days` of `now`, newest first. */ |
| 304 | export function recentIncidents<T extends Pick<StatusIncident, "started_at" | "resolved_at">>( |
| 305 | incidents: T[], |
| 306 | now: Date, |
| 307 | days = HISTORY_DAYS, |
| 308 | ): T[] { |
| 309 | const since = now.getTime() - days * DAY_MS; |
| 310 | return incidents |
| 311 | .filter((i) => i.resolved_at == null || Date.parse(i.resolved_at) >= since) |
| 312 | .sort((a, b) => Date.parse(b.started_at) - Date.parse(a.started_at)); |
| 313 | } |
| 314 | |
| 315 | /** Everything the page and the JSON show. */ |
| 316 | export type PageModel = { |
| 317 | report: StatusReport; |
| 318 | bars: Record<string, DayBar[]>; |
| 319 | /** The checks have not run for a while: the checker itself is in trouble. */ |
| 320 | stale: boolean; |
| 321 | }; |
| 322 | |
| 323 | /** The report, from what is stored. */ |
| 324 | export function buildPage(input: { |
| 325 | parts: ComponentInfo[]; |
| 326 | current: Map<string, Current>; |
| 327 | checkedAt: string | null; |
| 328 | days: DayRow[]; |
| 329 | incidents: StatusIncident[]; |
| 330 | maintenance?: StatusMaintenance[]; |
| 331 | now: Date; |
| 332 | }): PageModel { |
| 333 | const { parts, current, checkedAt, days, now } = input; |
| 334 | const window = lastDays(now); |
| 335 | const since = window[0]!; |
| 336 | const rows = days.filter((r) => r.day >= since); |
| 337 | const incidents = recentIncidents(input.incidents, now); |
| 338 | const open = incidents.filter((i) => i.resolved_at == null); |
| 339 | // Upcoming and under way, soonest first, each with its state as of now. |
| 340 | const maintenance = (input.maintenance ?? []) |
| 341 | .map((m) => ({ ...m, state: maintenanceState(m, now) })) |
| 342 | .filter((m) => m.state === "scheduled" || m.state === "in_progress") |
| 343 | .sort((a, b) => a.starts_at.localeCompare(b.starts_at)); |
| 344 | const working = underMaintenance(maintenance, now); |
| 345 | const components: StatusComponent[] = parts.map((info) => { |
| 346 | const seen = info.check.kind === "none" ? null : current.get(info.key); |
| 347 | const check: StatusComponentState = seen?.state ?? "unmonitored"; |
| 348 | const impacts = open.flatMap((i) => i.component_impacts.filter((c) => c.key === info.key).map((c) => c.impact)); |
| 349 | return { |
| 350 | key: info.key, |
| 351 | name: info.name, |
| 352 | address: info.address, |
| 353 | checks: info.checks, |
| 354 | state: effectiveState(check, impacts, working.has(info.key)), |
| 355 | check_state: check, |
| 356 | detail: seen?.detail ?? (info.check.kind === "none" ? "Not monitored yet" : "Not checked yet"), |
| 357 | latency_ms: seen?.latency_ms ?? null, |
| 358 | uptime_90d: info.check.kind === "none" ? null : uptimeOver(info.key, rows), |
| 359 | }; |
| 360 | }); |
| 361 | const core = new Set(parts.filter((p) => p.core).map((p) => p.key)); |
| 362 | const bars = Object.fromEntries(parts.map((p) => [p.key, barFor(p.key, rows, incidents, window)])); |
| 363 | const inProgress = maintenance.filter((m) => m.state === "in_progress"); |
| 364 | return { |
| 365 | report: { checked_at: checkedAt, overall: summarize(components, core, open, inProgress), components, incidents, maintenance }, |
| 366 | bars, |
| 367 | stale: checkedAt == null || now.getTime() - Date.parse(checkedAt) > STALE_MS, |
| 368 | }; |
| 369 | } |