Sign in

flagon-io/g1t

public

g1t/apps/status/src/model.ts

369 lines14,748 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1/**
2 * What a check's result means, how the parts add up to one line for the
3 * whole, and the 90 days of history behind each uptime bar. No Workers
4 * imports, so it is tested under Node.
5 */
6import type {
7 ComponentImpact,
8 IncidentImpact,
9 IncidentStatus,
10 MaintenanceState,
11 StatusComponent,
12 StatusComponentState,
13 StatusIncident,
14 StatusMaintenance,
15 StatusOverall,
16 StatusOverallState,
17 StatusReport,
18} from "@g1t/contracts";
19
20import type { ComponentInfo } from "./components.ts";
21import type { ProbeResult } from "./probe.ts";
22
23/** Slower than this, a part that answered counts as degraded. */
24export const SLOW_MS = 1500;
25/** How many days of history the page shows and keeps. */
26export const HISTORY_DAYS = 90;
27/** Checks older than this mean the checker has stopped: the page says so. */
28export const STALE_MS = 10 * 60 * 1000;
29
30/** What a check's result means for its part. `null` is a part with no check. */
31export function classify(result: ProbeResult | null, slowMs = SLOW_MS): { state: StatusComponentState; detail: string } {
32 if (!result) return { state: "unmonitored", detail: "Not monitored yet" };
33 if (!result.ok) return { state: "down", detail: `Failed: ${result.error ?? "no answer"}` };
34 const ms = Math.round(result.ms);
35 if (ms > slowMs) return { state: "degraded", detail: `Slow: answered in ${ms} ms` };
36 return { state: "up", detail: `Answered in ${ms} ms` };
37}
38
39/** A part's state at its last check, as stored. */
40export type Current = { state: StatusComponentState; detail: string; latency_ms: number | null };
41
42/** One part's checks on one UTC day, as stored. */
43export type DayRow = {
44 component: string;
45 /** `YYYY-MM-DD`, UTC. */
46 day: string;
47 checks: number;
48 up: number;
49 degraded: number;
50 down: number;
51 latency_total: number;
52 latency_count: number;
53};
54
55/** One day's square on a bar. */
56export type DayBar = {
57 day: string;
58 state: "up" | "degraded" | "down" | "none";
59 /** Share of checks that answered, 0 to 100; null with none. */
60 uptime: number | null;
61 checks: number;
62 failed: number;
63 slow: number;
64 avg_ms: number | null;
65 /** Incidents on this part that were open some time that day. */
66 incidents: string[];
67};
68
69const DAY_MS = 24 * 60 * 60 * 1000;
70
71/** `YYYY-MM-DD` of a time, in UTC. */
72export function dayOf(at: Date | number): string {
73 return new Date(at).toISOString().slice(0, 10);
74}
75
76/** The last `count` days, oldest first, ending today. */
77export function lastDays(now: Date, count = HISTORY_DAYS): string[] {
78 const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
79 return Array.from({ length: count }, (_, i) => dayOf(today - (count - 1 - i) * DAY_MS));
80}
81
82/**
83 * Share of checks answered, rounded down to hundredths so a single failure
84 * never shows as 100%.
85 */
86export function uptimeOf(answered: number, checks: number): number | null {
87 if (checks <= 0) return null;
88 return Math.floor((answered / checks) * 10000) / 100;
89}
90
91/** Words for an uptime: "100%", "99.93%". */
92export function percent(uptime: number | null): string {
93 if (uptime == null) return "No data";
94 return uptime >= 100 ? "100%" : `${uptime.toFixed(2)}%`;
95}
96
97const LEVEL = { none: 0, up: 1, degraded: 2, down: 3 } as const;
98
99/**
100 * A day's colour: answered at least 99.9% of the time is up (one blip in a
101 * day of minutely checks), unless over a quarter of answers were slow; at
102 * least 95% is degraded; less is down. An incident on the part that day
103 * colours it at least as badly as its impact.
104 */
105export function dayState(row: Pick<DayRow, "checks" | "up" | "degraded" | "down"> | null, incidentImpact: IncidentImpact | null = null): DayBar["state"] {
106 let state: DayBar["state"] = "none";
107 if (row && row.checks > 0) {
108 const uptime = ((row.up + row.degraded) / row.checks) * 100;
109 if (uptime >= 99.9) state = row.degraded / row.checks > 0.25 ? "degraded" : "up";
110 else if (uptime >= 95) state = "degraded";
111 else state = "down";
112 }
113 if (incidentImpact && LEVEL[incidentImpact] > LEVEL[state]) state = incidentImpact;
114 return state;
115}
116
117/** Whether an incident was open at some time during `day`. */
118export function openOn(incident: Pick<StatusIncident, "started_at" | "resolved_at">, day: string): boolean {
119 const start = Date.parse(`${day}T00:00:00Z`);
120 const end = start + DAY_MS;
121 const began = Date.parse(incident.started_at);
122 const ended = incident.resolved_at ? Date.parse(incident.resolved_at) : Number.POSITIVE_INFINITY;
123 return began < end && ended >= start;
124}
125
126/** A part's bar: one square per day, oldest first. */
127export function barFor(
128 key: string,
129 rows: DayRow[],
130 incidents: Pick<StatusIncident, "title" | "started_at" | "resolved_at" | "component_impacts">[],
131 days: string[],
132): DayBar[] {
133 const byDay = new Map(rows.filter((r) => r.component === key).map((r) => [r.day, r]));
134 const mine = incidents
135 .map((i) => ({ incident: i, impact: i.component_impacts.find((c) => c.key === key)?.impact ?? null }))
136 .filter((m) => m.impact != null && m.impact !== "operational");
137 return days.map((day) => {
138 const row = byDay.get(day) ?? null;
139 const during = mine.filter((m) => openOn(m.incident, day));
140 const impact = during.some((m) => m.impact === "major_outage") ? "down" : during.length ? "degraded" : null;
141 return {
142 day,
143 state: dayState(row, impact),
144 uptime: row ? uptimeOf(row.up + row.degraded, row.checks) : null,
145 checks: row?.checks ?? 0,
146 failed: row?.down ?? 0,
147 slow: row?.degraded ?? 0,
148 avg_ms: row && row.latency_count > 0 ? Math.round(row.latency_total / row.latency_count) : null,
149 incidents: during.map((m) => m.incident.title),
150 };
151 });
152}
153
154/** A part's uptime over the days given. */
155export function uptimeOver(key: string, rows: DayRow[]): number | null {
156 let checks = 0;
157 let answered = 0;
158 for (const row of rows) {
159 if (row.component !== key) continue;
160 checks += row.checks;
161 answered += row.up + row.degraded;
162 }
163 return uptimeOf(answered, checks);
164}
165
166
167const RANK: Record<StatusOverallState, number> = { unknown: 0, up: 1, maintenance: 1.5, degraded: 2, down: 3 };
168
169function worse(a: StatusOverallState, b: StatusOverallState): StatusOverallState {
170 return RANK[b] > RANK[a] ? b : a;
171}
172
173/** Words for an incident's status. */
174export const INCIDENT_STATUS: Record<IncidentStatus, string> = {
175 investigating: "Investigating",
176 identified: "Identified",
177 monitoring: "Monitoring",
178 resolved: "Resolved",
179};
180
181/** Words for what an incident does to a part. */
182export const IMPACT_WORD: Record<ComponentImpact, string> = {
183 operational: "Operational",
184 degraded: "Degraded performance",
185 partial_outage: "Partial outage",
186 major_outage: "Major outage",
187};
188
189/** A part's state for an impact on it. */
190export const IMPACT_STATE: Record<ComponentImpact, StatusComponentState> = {
191 operational: "up",
192 degraded: "degraded",
193 partial_outage: "partial",
194 major_outage: "down",
195};
196
197/** How bad each state is: the worse of a check and an incident wins. */
198const STATE_RANK: Record<StatusComponentState, number> = { unmonitored: 0, up: 1, maintenance: 1, degraded: 2, partial: 3, down: 4 };
199
200/** An incident's impact as a whole: `down` when any part has a major outage. */
201export function overallImpact(impacts: { impact: ComponentImpact }[]): IncidentImpact {
202 return impacts.some((i) => i.impact === "major_outage") ? "down" : "degraded";
203}
204
205/**
206 * What the page shows for a part: the worse of its last check and what
207 * open incidents say about it. During maintenance it is "Under
208 * maintenance", unless an incident says worse, since checks failing in a
209 * window are expected.
210 */
211export function effectiveState(check: StatusComponentState, impacts: ComponentImpact[], underMaintenance: boolean): StatusComponentState {
212 let reported: StatusComponentState | null = null;
213 for (const impact of impacts) {
214 const state = IMPACT_STATE[impact];
215 if (reported == null || STATE_RANK[state] > STATE_RANK[reported]) reported = state;
216 }
217 if (underMaintenance && (reported == null || reported === "up")) return "maintenance";
218 if (reported == null) return check;
219 if (check === "unmonitored" || check === "maintenance") return reported;
220 return STATE_RANK[reported] > STATE_RANK[check] ? reported : check;
221}
222
223/**
224 * Where maintenance stands at `now`. Cancelled and completed are final;
225 * otherwise the window decides, so the page is right even if the
226 * minutely job that moves it along is late.
227 */
228export function maintenanceState(m: Pick<StatusMaintenance, "state" | "starts_at" | "ends_at">, now: Date): MaintenanceState {
229 if (m.state === "cancelled" || m.state === "completed") return m.state;
230 const t = now.getTime();
231 if (t >= Date.parse(m.ends_at)) return "completed";
232 if (t >= Date.parse(m.starts_at) || m.state === "in_progress") return "in_progress";
233 return "scheduled";
234}
235
236/** The parts under maintenance at `now`. */
237export function underMaintenance(
238 list: Pick<StatusMaintenance, "state" | "starts_at" | "ends_at" | "components">[],
239 now: Date,
240): Set<string> {
241 const keys = new Set<string>();
242 for (const m of list) if (maintenanceState(m, now) === "in_progress") for (const k of m.components) keys.add(k);
243 return keys;
244}
245
246/**
247 * The whole, from its parts and its open incidents: a major outage when a
248 * core part is down or an incident says so; a partial outage when any part
249 * is down or partly down, or an incident is open; degraded performance
250 * when parts are only slow; under maintenance when work is under way and
251 * nothing else is wrong. Parts with no check never make it look better or
252 * worse.
253 */
254export function summarize(
255 parts: Pick<StatusComponent, "key" | "name" | "state">[],
256 core: Set<string>,
257 open: Pick<StatusIncident, "title" | "impact" | "status">[] = [],
258 maintenance: Pick<StatusMaintenance, "title">[] = [],
259): StatusOverall {
260 const monitored = parts.filter((c) => c.state !== "unmonitored");
261 if (monitored.length === 0 && open.length === 0) {
262 return { state: "unknown", title: "Status unavailable", line: "No checks have run yet." };
263 }
264 let state: StatusOverallState = monitored.length > 0 ? "up" : "unknown";
265 for (const c of monitored) {
266 if (c.state === "down") state = worse(state, core.has(c.key) ? "down" : "degraded");
267 else if (c.state === "partial" || c.state === "degraded") state = worse(state, "degraded");
268 else if (c.state === "maintenance") state = worse(state, "maintenance");
269 }
270 for (const incident of open) state = worse(state, incident.impact);
271
272 const down = monitored.filter((c) => c.state === "down" || c.state === "partial").map((c) => c.name);
273 const slow = monitored.filter((c) => c.state === "degraded").map((c) => c.name);
274 const lead = open[0];
275 const incidentLine = lead ? `${INCIDENT_STATUS[lead.status]}: ${lead.title}.` : null;
276 const named = (names: string[]) => names.join(", ");
277
278 if (state === "up") return { state, title: "All systems normal", line: "Every part of g1t answered its last check." };
279 if (state === "maintenance") {
280 const work = maintenance[0];
281 return { state, title: "Under maintenance", line: work ? `Planned work: ${work.title}.` : "Planned work is under way." };
282 }
283 if (state === "down") {
284 return {
285 state,
286 title: "Major outage",
287 line: incidentLine ?? (down.length ? `Not answering: ${named(down)}.` : "g1t is not working right now."),
288 };
289 }
290 if (state === "degraded") {
291 if (down.length || open.length) {
292 return {
293 state,
294 title: "Partial outage",
295 line: incidentLine ?? `Not answering: ${named(down)}${slow.length ? `. Slow: ${named(slow)}` : ""}.`,
296 };
297 }
298 return { state, title: "Degraded performance", line: `Slow to answer: ${named(slow)}.` };
299 }
300 return { state, title: "Status unavailable", line: "No checks have run yet." };
301}
302
303/** Open incidents, and those resolved within `days` of `now`, newest first. */
304export function recentIncidents<T extends Pick<StatusIncident, "started_at" | "resolved_at">>(
305 incidents: T[],
306 now: Date,
307 days = HISTORY_DAYS,
308): T[] {
309 const since = now.getTime() - days * DAY_MS;
310 return incidents
311 .filter((i) => i.resolved_at == null || Date.parse(i.resolved_at) >= since)
312 .sort((a, b) => Date.parse(b.started_at) - Date.parse(a.started_at));
313}
314
315/** Everything the page and the JSON show. */
316export type PageModel = {
317 report: StatusReport;
318 bars: Record<string, DayBar[]>;
319 /** The checks have not run for a while: the checker itself is in trouble. */
320 stale: boolean;
321};
322
323/** The report, from what is stored. */
324export function buildPage(input: {
325 parts: ComponentInfo[];
326 current: Map<string, Current>;
327 checkedAt: string | null;
328 days: DayRow[];
329 incidents: StatusIncident[];
330 maintenance?: StatusMaintenance[];
331 now: Date;
332}): PageModel {
333 const { parts, current, checkedAt, days, now } = input;
334 const window = lastDays(now);
335 const since = window[0]!;
336 const rows = days.filter((r) => r.day >= since);
337 const incidents = recentIncidents(input.incidents, now);
338 const open = incidents.filter((i) => i.resolved_at == null);
339 // Upcoming and under way, soonest first, each with its state as of now.
340 const maintenance = (input.maintenance ?? [])
341 .map((m) => ({ ...m, state: maintenanceState(m, now) }))
342 .filter((m) => m.state === "scheduled" || m.state === "in_progress")
343 .sort((a, b) => a.starts_at.localeCompare(b.starts_at));
344 const working = underMaintenance(maintenance, now);
345 const components: StatusComponent[] = parts.map((info) => {
346 const seen = info.check.kind === "none" ? null : current.get(info.key);
347 const check: StatusComponentState = seen?.state ?? "unmonitored";
348 const impacts = open.flatMap((i) => i.component_impacts.filter((c) => c.key === info.key).map((c) => c.impact));
349 return {
350 key: info.key,
351 name: info.name,
352 address: info.address,
353 checks: info.checks,
354 state: effectiveState(check, impacts, working.has(info.key)),
355 check_state: check,
356 detail: seen?.detail ?? (info.check.kind === "none" ? "Not monitored yet" : "Not checked yet"),
357 latency_ms: seen?.latency_ms ?? null,
358 uptime_90d: info.check.kind === "none" ? null : uptimeOver(info.key, rows),
359 };
360 });
361 const core = new Set(parts.filter((p) => p.core).map((p) => p.key));
362 const bars = Object.fromEntries(parts.map((p) => [p.key, barFor(p.key, rows, incidents, window)]));
363 const inProgress = maintenance.filter((m) => m.state === "in_progress");
364 return {
365 report: { checked_at: checkedAt, overall: summarize(components, core, open, inProgress), components, incidents, maintenance },
366 bars,
367 stale: checkedAt == null || now.getTime() - Date.parse(checkedAt) > STALE_MS,
368 };
369}