| 1 | /** |
| 2 | * Noticing trouble before anyone reports it: when a part fails (or is |
| 3 | * slow) for DETECT_AFTER checks in a row, a draft incident is made for |
| 4 | * staff in sudo, not shown on the status page until someone publishes it. |
| 5 | * When a part that had crossed that line answers again, open incidents on |
| 6 | * it get a note. A detected draft nobody has picked up is dismissed on its |
| 7 | * own once its parts have stayed healthy for RECOVERED_FOR_MS, and while a |
| 8 | * deploy is running (and for DEPLOY_GRACE_MS after) no new draft is made |
| 9 | * unless the trouble outlasts it. No Workers imports, so it is tested |
| 10 | * under Node. |
| 11 | */ |
| 12 | import type { ComponentImpact, StatusComponentState } from "@g1t/contracts"; |
| 13 | |
| 14 | /** Checks in a row, a minute apart, before a draft is made. */ |
| 15 | export const DETECT_AFTER = 3; |
| 16 | /** How long a detected draft's parts stay healthy before it is dismissed on its own. */ |
| 17 | export const RECOVERED_FOR_MS = 10 * 60_000; |
| 18 | /** How long after a deploy finishes its restarts are still forgiven. */ |
| 19 | export const DEPLOY_GRACE_MS = 3 * 60_000; |
| 20 | /** A deploy that said it started and never said it finished stops counting after this. */ |
| 21 | export const DEPLOY_MAX_MS = 30 * 60_000; |
| 22 | |
| 23 | /** |
| 24 | * One part's current run of failed or slow checks. Only parts that were |
| 25 | * failing or slow on the latest check have one: any good check ends it, |
| 26 | * and the next trouble starts a new run, with its own start. |
| 27 | */ |
| 28 | export type Streak = { |
| 29 | component: string; |
| 30 | /** The worst seen in this run of failures. */ |
| 31 | state: "degraded" | "down"; |
| 32 | count: number; |
| 33 | /** The first failed or slow check of this run. */ |
| 34 | since: string; |
| 35 | /** Whether it has crossed the line, and been raised. */ |
| 36 | alerted: boolean; |
| 37 | }; |
| 38 | |
| 39 | /** An open incident (draft or public), with the parts it affects. */ |
| 40 | export type OpenRef = { id: string; components: string[] }; |
| 41 | |
| 42 | export type Trouble = { key: string; state: "degraded" | "down"; since: string; checks: number }; |
| 43 | |
| 44 | export type Detection = { |
| 45 | /** Every failing part's run after this round; parts not here have none. */ |
| 46 | streaks: Streak[]; |
| 47 | /** Parts that crossed the line with no open incident on them: one draft for all. */ |
| 48 | draft: Trouble[]; |
| 49 | /** Parts that crossed the line while an incident on them was open. */ |
| 50 | failing: (Trouble & { incident: string })[]; |
| 51 | /** Parts answering again after crossing the line. */ |
| 52 | recovered: { incident: string; key: string; state: "degraded" | "down"; since: string; checks: number }[]; |
| 53 | /** Parts that crossed the line during a deploy: kept counting, not raised yet. */ |
| 54 | held: string[]; |
| 55 | }; |
| 56 | |
| 57 | export type DetectOptions = { |
| 58 | threshold?: number; |
| 59 | /** A deploy is running, or just finished: no new drafts, only counting. */ |
| 60 | quiet?: boolean; |
| 61 | }; |
| 62 | |
| 63 | export function detect( |
| 64 | previous: Map<string, Streak>, |
| 65 | observations: { component: string; state: StatusComponentState }[], |
| 66 | open: OpenRef[], |
| 67 | maintenance: Set<string>, |
| 68 | at: Date, |
| 69 | { threshold = DETECT_AFTER, quiet = false }: DetectOptions = {}, |
| 70 | ): Detection { |
| 71 | const out: Detection = { streaks: [], draft: [], failing: [], recovered: [], held: [] }; |
| 72 | const covering = (key: string) => open.filter((i) => i.components.includes(key)); |
| 73 | for (const { component: key, state } of observations) { |
| 74 | const prev = previous.get(key); |
| 75 | if (state === "unmonitored" || maintenance.has(key)) continue; |
| 76 | if (state !== "degraded" && state !== "down") { |
| 77 | // A good check: the run, if any, is over. Leaving it out of `streaks` ends it. |
| 78 | if (prev?.alerted) for (const i of covering(key)) out.recovered.push({ incident: i.id, key, state: prev.state, since: prev.since, checks: prev.count }); |
| 79 | continue; |
| 80 | } |
| 81 | const streak: Streak = prev |
| 82 | ? { ...prev, count: prev.count + 1, state: prev.state === "down" || state === "down" ? "down" : "degraded" } |
| 83 | : { component: key, state, count: 1, since: at.toISOString(), alerted: false }; |
| 84 | if (!streak.alerted && streak.count >= threshold) { |
| 85 | const trouble = { key, state: streak.state, since: streak.since, checks: streak.count }; |
| 86 | const incidents = covering(key); |
| 87 | if (incidents.length) { |
| 88 | streak.alerted = true; |
| 89 | for (const i of incidents) out.failing.push({ incident: i.id, ...trouble }); |
| 90 | } else if (quiet) { |
| 91 | out.held.push(key); |
| 92 | } else { |
| 93 | streak.alerted = true; |
| 94 | out.draft.push(trouble); |
| 95 | } |
| 96 | } |
| 97 | out.streaks.push(streak); |
| 98 | } |
| 99 | return out; |
| 100 | } |
| 101 | |
| 102 | // --- Deploys -------------------------------------------------------------------------- |
| 103 | |
| 104 | /** A deploy as the deploy tool reported it. */ |
| 105 | export type DeployWindow = { id: string | null; started_at: string; finished_at: string | null }; |
| 106 | |
| 107 | /** Whether detection holds off at `at`: during a deploy, and for a grace period after. */ |
| 108 | export function deployQuiet(window: DeployWindow | null, at: Date): boolean { |
| 109 | if (!window) return false; |
| 110 | const start = Date.parse(window.started_at); |
| 111 | const t = at.getTime(); |
| 112 | if (Number.isNaN(start) || t < start - 60_000) return false; |
| 113 | if (window.finished_at) return t < Date.parse(window.finished_at) + DEPLOY_GRACE_MS; |
| 114 | return t < start + DEPLOY_MAX_MS; |
| 115 | } |
| 116 | |
| 117 | /** The deploy tool's start or finish, folded into what is kept. */ |
| 118 | export function deployChange(window: DeployWindow | null, phase: "started" | "finished", id: string | null, at: Date): DeployWindow { |
| 119 | const now = at.toISOString(); |
| 120 | if (phase === "started") { |
| 121 | // A second start while one is running keeps the earlier start. |
| 122 | const running = window && !window.finished_at && deployQuiet(window, at); |
| 123 | return { id, started_at: running ? window.started_at : now, finished_at: null }; |
| 124 | } |
| 125 | const running = window && !window.finished_at && deployQuiet(window, at); |
| 126 | return { id: id ?? window?.id ?? null, started_at: running ? window.started_at : now, finished_at: now }; |
| 127 | } |
| 128 | |
| 129 | // --- Detected drafts that recover --------------------------------------------------------- |
| 130 | |
| 131 | /** A detected draft no one has picked up yet, and since when its parts have been healthy. */ |
| 132 | export type WatchedDraft = { id: string; title: string; components: string[]; started_at: string; healthy_since: string | null }; |
| 133 | |
| 134 | export type Settled = { |
| 135 | /** Every watched draft's healthy-since after this round: null while a part is in trouble. */ |
| 136 | healthy: { id: string; since: string | null }[]; |
| 137 | /** Drafts healthy long enough to dismiss. */ |
| 138 | dismiss: { id: string; title: string; recovered_at: string; lasted_ms: number }[]; |
| 139 | }; |
| 140 | |
| 141 | /** |
| 142 | * Which detected drafts have recovered for good. `troubled` is every part |
| 143 | * with a run of failed or slow checks after this round. |
| 144 | */ |
| 145 | export function settleDrafts(drafts: WatchedDraft[], troubled: Set<string>, at: Date, after = RECOVERED_FOR_MS): Settled { |
| 146 | const out: Settled = { healthy: [], dismiss: [] }; |
| 147 | for (const d of drafts) { |
| 148 | if (d.components.some((k) => troubled.has(k))) { |
| 149 | out.healthy.push({ id: d.id, since: null }); |
| 150 | continue; |
| 151 | } |
| 152 | const since = d.healthy_since ?? at.toISOString(); |
| 153 | if (at.getTime() - Date.parse(since) >= after) { |
| 154 | out.dismiss.push({ id: d.id, title: d.title, recovered_at: since, lasted_ms: Math.max(0, Date.parse(since) - Date.parse(d.started_at)) }); |
| 155 | } else { |
| 156 | out.healthy.push({ id: d.id, since }); |
| 157 | } |
| 158 | } |
| 159 | return out; |
| 160 | } |
| 161 | |
| 162 | // --- Wording -------------------------------------------------------------------------------- |
| 163 | |
| 164 | /** What a detected failure does to its part, for the draft. */ |
| 165 | export function detectedImpact(state: "degraded" | "down"): ComponentImpact { |
| 166 | return state === "down" ? "major_outage" : "degraded"; |
| 167 | } |
| 168 | |
| 169 | /** The draft's title: "Detected: API and Git not answering". */ |
| 170 | export function draftTitle(parts: { name: string; state: "degraded" | "down" }[]): string { |
| 171 | const list = (names: string[]) => (names.length <= 2 ? names.join(" and ") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`); |
| 172 | const down = parts.filter((p) => p.state === "down").map((p) => p.name); |
| 173 | const slow = parts.filter((p) => p.state === "degraded").map((p) => p.name); |
| 174 | const said = [down.length ? `${list(down)} not answering` : "", slow.length ? `${list(slow)} slow` : ""].filter(Boolean).join("; "); |
| 175 | return `Detected: ${said}`.slice(0, 120); |
| 176 | } |
| 177 | |
| 178 | /** A part's slow line in words: "1.5 s", "800 ms". */ |
| 179 | export function limitWords(ms: number): string { |
| 180 | return ms >= 1000 ? `${Number((ms / 1000).toFixed(1))} s` : `${ms} ms`; |
| 181 | } |
| 182 | |
| 183 | /** "3 minutes", "1 minute", "2h 05m". */ |
| 184 | export function minutesWords(ms: number): string { |
| 185 | const minutes = Math.max(1, Math.round(ms / 60_000)); |
| 186 | if (minutes < 60) return `${minutes} minute${minutes === 1 ? "" : "s"}`; |
| 187 | return `${Math.floor(minutes / 60)}h ${String(minutes % 60).padStart(2, "0")}m`; |
| 188 | } |
| 189 | |
| 190 | /** |
| 191 | * One part's trouble in a sentence, slow and down said apart: |
| 192 | * "Git has been slow — over 1.5 s — on 3 checks in a row since 6 Oct 07:25 UTC." |
| 193 | * "API has not answered on 3 checks in a row since 6 Oct 07:25 UTC." |
| 194 | * `since` is already written out, in whatever zone the reader needs. |
| 195 | */ |
| 196 | export function troubleSentence(name: string, t: { state: "degraded" | "down"; checks: number }, since: string, slowMs: number): string { |
| 197 | const checks = `${t.checks} check${t.checks === 1 ? "" : "s"} in a row`; |
| 198 | return t.state === "down" |
| 199 | ? `${name} has not answered on ${checks} since ${since}.` |
| 200 | : `${name} has been slow — over ${limitWords(slowMs)} — on ${checks} since ${since}.`; |
| 201 | } |
| 202 | |
| 203 | /** A part answering again, after a run that crossed the line. */ |
| 204 | export function recoverySentence(name: string, t: { state: "degraded" | "down"; checks: number }, since: string): string { |
| 205 | const checks = `${t.checks} check${t.checks === 1 ? "" : "s"} in a row`; |
| 206 | return t.state === "down" |
| 207 | ? `${name} is answering again, after not answering on ${checks} since ${since}.` |
| 208 | : `${name} is back to normal speed, after being slow on ${checks} since ${since}.`; |
| 209 | } |
| 210 | |
| 211 | /** The timeline line, and the follow-up email's words, when a recovered draft is dismissed on its own. */ |
| 212 | export function autoDismissText(lastedMs: number, recoveredAt: string, healthyFor = RECOVERED_FOR_MS): string { |
| 213 | return `Recovered after ${minutesWords(lastedMs)}, at ${recoveredAt}, and stayed healthy for ${minutesWords(healthyFor)}; dismissed automatically. It never appeared on the status page.`; |
| 214 | } |