| 1 | /** |
| 2 | * Noticing trouble before anyone reports it: when a part fails (or is |
| 3 | * slow) for DETECT_AFTER checks in a row, a draft incident is made for |
| 4 | * staff in sudo, not shown on the status page until someone publishes it. |
| 5 | * When a part that had crossed that line answers again, open incidents on |
| 6 | * it get a note. No Workers imports, so it is tested under Node. |
| 7 | */ |
| 8 | import type { ComponentImpact, StatusComponentState } from "@g1t/contracts"; |
| 9 | |
| 10 | /** Checks in a row, a minute apart, before a draft is made. */ |
| 11 | export const DETECT_AFTER = 3; |
| 12 | |
| 13 | export type Streak = { |
| 14 | component: string; |
| 15 | /** The worst seen in this run of failures. */ |
| 16 | state: "degraded" | "down"; |
| 17 | count: number; |
| 18 | /** When the run began. */ |
| 19 | since: string; |
| 20 | /** Whether it has crossed the line, and been raised. */ |
| 21 | alerted: boolean; |
| 22 | }; |
| 23 | |
| 24 | /** An open incident (draft or public), with the parts it affects. */ |
| 25 | export type OpenRef = { id: string; components: string[] }; |
| 26 | |
| 27 | export type Detection = { |
| 28 | /** Every failing part's run after this round; parts not here have none. */ |
| 29 | streaks: Streak[]; |
| 30 | /** Parts that crossed the line with no open incident on them: one draft for all. */ |
| 31 | draft: { key: string; state: "degraded" | "down"; since: string }[]; |
| 32 | /** Parts that crossed the line while an incident on them was open. */ |
| 33 | failing: { incident: string; key: string; state: "degraded" | "down"; since: string }[]; |
| 34 | /** Parts answering again after crossing the line. */ |
| 35 | recovered: { incident: string; key: string; since: string; checks: number }[]; |
| 36 | }; |
| 37 | |
| 38 | export function detect( |
| 39 | previous: Map<string, Streak>, |
| 40 | observations: { component: string; state: StatusComponentState }[], |
| 41 | open: OpenRef[], |
| 42 | maintenance: Set<string>, |
| 43 | at: Date, |
| 44 | threshold = DETECT_AFTER, |
| 45 | ): Detection { |
| 46 | const out: Detection = { streaks: [], draft: [], failing: [], recovered: [] }; |
| 47 | const covering = (key: string) => open.filter((i) => i.components.includes(key)); |
| 48 | for (const { component: key, state } of observations) { |
| 49 | const prev = previous.get(key); |
| 50 | if (state === "unmonitored" || maintenance.has(key)) continue; |
| 51 | if (state !== "degraded" && state !== "down") { |
| 52 | if (prev?.alerted) for (const i of covering(key)) out.recovered.push({ incident: i.id, key, since: prev.since, checks: prev.count }); |
| 53 | continue; |
| 54 | } |
| 55 | const streak: Streak = prev |
| 56 | ? { ...prev, count: prev.count + 1, state: prev.state === "down" || state === "down" ? "down" : "degraded" } |
| 57 | : { component: key, state, count: 1, since: at.toISOString(), alerted: false }; |
| 58 | if (!streak.alerted && streak.count >= threshold) { |
| 59 | streak.alerted = true; |
| 60 | const incidents = covering(key); |
| 61 | if (incidents.length) for (const i of incidents) out.failing.push({ incident: i.id, key, state: streak.state, since: streak.since }); |
| 62 | else out.draft.push({ key, state: streak.state, since: streak.since }); |
| 63 | } |
| 64 | out.streaks.push(streak); |
| 65 | } |
| 66 | return out; |
| 67 | } |
| 68 | |
| 69 | /** What a detected failure does to its part, for the draft. */ |
| 70 | export function detectedImpact(state: "degraded" | "down"): ComponentImpact { |
| 71 | return state === "down" ? "major_outage" : "degraded"; |
| 72 | } |
| 73 | |
| 74 | /** The draft's title: "Detected: API and Git not answering". */ |
| 75 | export function draftTitle(parts: { name: string; state: "degraded" | "down" }[]): string { |
| 76 | const list = (names: string[]) => (names.length <= 2 ? names.join(" and ") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`); |
| 77 | const down = parts.filter((p) => p.state === "down").map((p) => p.name); |
| 78 | const slow = parts.filter((p) => p.state === "degraded").map((p) => p.name); |
| 79 | const said = [down.length ? `${list(down)} not answering` : "", slow.length ? `${list(slow)} slow` : ""].filter(Boolean).join("; "); |
| 80 | return `Detected: ${said}`.slice(0, 120); |
| 81 | } |