| 1 | /** |
| 2 | * One project's telemetry: a Durable Object named by the project's id, |
| 3 | * holding its requests, logs, spans, metrics, error groups, ingest tokens |
| 4 | * and exports in its own SQLite (./schema.ts). One object per project |
| 5 | * means each project's store grows, and is queried, on its own. |
| 6 | * |
| 7 | * Writes arrive from the service's two doors (`ingest`); reads are the |
| 8 | * service's RPC methods, called once the site or the API has checked the |
| 9 | * viewer may read the project. Open live tails are hibernating WebSockets |
| 10 | * here, each handed what arrives as it arrives. |
| 11 | * |
| 12 | * An alarm runs while there is work: each minute it counts requests into |
| 13 | * per-minute rollups and sends to exports what is waiting; each hour it |
| 14 | * removes what is past the project's retention. |
| 15 | */ |
| 16 | import { |
| 17 | DEFAULT_TELEMETRY_SETTINGS, |
| 18 | RETENTION_CHOICES, |
| 19 | TELEMETRY_TOKEN_PREFIX, |
| 20 | type Attributes, |
| 21 | type ErrorGroup, |
| 22 | type IngestToken, |
| 23 | type LiveItem, |
| 24 | type LogQuery, |
| 25 | type MetricInfo, |
| 26 | type MetricSeries, |
| 27 | type TelemetryOverview, |
| 28 | type TelemetryTotals, |
| 29 | type Page, |
| 30 | type RequestQuery, |
| 31 | type LogSeverity, |
| 32 | type TelemetryExport, |
| 33 | type TelemetryFilter, |
| 34 | type TelemetryLog, |
| 35 | type TelemetryRequest, |
| 36 | type TelemetrySettings, |
| 37 | type TelemetrySpan, |
| 38 | type TelemetryUsage, |
| 39 | type TraceDetail, |
| 40 | type TraceQuery, |
| 41 | type TraceSummary, |
| 42 | } from "@g1t/contracts/telemetry"; |
| 43 | |
| 44 | import type { Batch } from "./batch.ts"; |
| 45 | import { fingerprint } from "./errors.ts"; |
| 46 | import { exportEndpointProblem, otlpBodies, send } from "./export.ts"; |
| 47 | import { addInto, bucketOf, decode, emptyHistogram, encode, percentiles } from "./histogram.ts"; |
| 48 | import { FTS, MIGRATIONS, ROLLUP_DAYS } from "./schema.ts"; |
| 49 | import { SEVERITY_FLOOR, randomId, severityOf } from "./shape.ts"; |
| 50 | import type { RequestHalf } from "./tail.ts"; |
| 51 | |
| 52 | const MINUTE_US = 60_000_000; |
| 53 | const DAY_US = 86_400_000_000; |
| 54 | /** Requests are counted once they are this old, so both halves have arrived. */ |
| 55 | const ROLL_AFTER_US = 90_000_000; |
| 56 | /** A store larger than this sheds its oldest day until it is not. */ |
| 57 | const MAX_BYTES = 4 * 1024 ** 3; |
| 58 | /** At most this many items a second go to each live tail; the rest are counted as dropped. */ |
| 59 | const LIVE_PER_BATCH = 200; |
| 60 | const STEPS = [60, 300, 900, 1800, 3600, 3 * 3600, 6 * 3600, 86400]; |
| 61 | |
| 62 | type Row = Record<string, SqlStorageValue>; |
| 63 | |
| 64 | const json = <T>(text: SqlStorageValue, fallback: T): T => { |
| 65 | if (typeof text !== "string" || text === "") return fallback; |
| 66 | try { |
| 67 | return JSON.parse(text) as T; |
| 68 | } catch { |
| 69 | return fallback; |
| 70 | } |
| 71 | }; |
| 72 | const strOrNull = (v: SqlStorageValue) => (v === null || v === undefined || v === "" ? null : String(v)); |
| 73 | const numOrNull = (v: SqlStorageValue) => (v === null || v === undefined ? null : Number(v)); |
| 74 | |
| 75 | async function sha256(text: string): Promise<string> { |
| 76 | const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); |
| 77 | return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); |
| 78 | } |
| 79 | |
| 80 | function month(us = Date.now() * 1000): string { |
| 81 | return new Date(us / 1000).toISOString().slice(0, 7); |
| 82 | } |
| 83 | |
| 84 | /** A page's cursor: the last row's time and id, so the next page starts after it. */ |
| 85 | function cursorOf(time: number, id: string): string { |
| 86 | return btoa(`${time}:${id}`); |
| 87 | } |
| 88 | function readCursor(cursor: string | null | undefined): { time: number; id: string } | null { |
| 89 | if (!cursor) return null; |
| 90 | try { |
| 91 | const [time, id] = atob(cursor).split(":"); |
| 92 | return Number.isFinite(Number(time)) && id ? { time: Number(time), id } : null; |
| 93 | } catch { |
| 94 | return null; |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | /** A limit within reason. */ |
| 99 | const limitOf = (n: number | null | undefined, fallback = 100, max = 500) => Math.min(max, Math.max(1, Math.trunc(n ?? fallback))); |
| 100 | |
| 101 | /** |
| 102 | * Search text split into what it says: `key=value` and `key:value` pairs |
| 103 | * filter by attribute (or by a column the row has), the rest are words to find. |
| 104 | */ |
| 105 | export function parseSearch(text: string | null | undefined): { words: string[]; pairs: [string, string][] } { |
| 106 | const words: string[] = []; |
| 107 | const pairs: [string, string][] = []; |
| 108 | for (const m of (text ?? "").matchAll(/"([^"]+)"|(\S+)/g)) { |
| 109 | const token = m[1] ?? m[2]; |
| 110 | const pair = m[1] ? null : /^([\w.@/-]+)[=:](.+)$/.exec(token); |
| 111 | if (pair && !/^https?$/.test(pair[1])) pairs.push([pair[1], pair[2]]); |
| 112 | else words.push(token); |
| 113 | } |
| 114 | return { words, pairs }; |
| 115 | } |
| 116 | |
| 117 | /** An FTS5 query in which every word must appear (as a prefix, for the last). */ |
| 118 | export function ftsQuery(words: string[]): string { |
| 119 | return words.map((w, i) => `"${w.replace(/"/g, '""')}"${i === words.length - 1 && !/\s/.test(w) ? "*" : ""}`).join(" AND "); |
| 120 | } |
| 121 | |
| 122 | /** |
| 123 | * The store's logic, apart from the Durable Object it runs in |
| 124 | * (./durable.ts), so tests run it on Node's SQLite. |
| 125 | */ |
| 126 | export class Store { |
| 127 | state!: DurableObjectState; |
| 128 | sql!: SqlStorage; |
| 129 | fts = false; |
| 130 | |
| 131 | constructor(state: DurableObjectState) { |
| 132 | this.open(state); |
| 133 | } |
| 134 | |
| 135 | open(state: DurableObjectState) { |
| 136 | this.state = state; |
| 137 | this.sql = state.storage.sql; |
| 138 | this.migrate(); |
| 139 | } |
| 140 | |
| 141 | migrate() { |
| 142 | this.sql.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"); |
| 143 | const done = Number(this.sql.exec("SELECT value FROM meta WHERE key = 'migrations'").toArray()[0]?.value ?? 0); |
| 144 | for (let i = done; i < MIGRATIONS.length; i++) { |
| 145 | this.sql.exec(MIGRATIONS[i]); |
| 146 | this.sql.exec("INSERT INTO meta (key, value) VALUES ('migrations', ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", String(i + 1)); |
| 147 | } |
| 148 | try { |
| 149 | this.sql.exec(FTS); |
| 150 | this.fts = true; |
| 151 | } catch { |
| 152 | this.fts = false; |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | rows(query: string, ...args: SqlStorageValue[]): Row[] { |
| 157 | return this.sql.exec(query, ...args).toArray() as Row[]; |
| 158 | } |
| 159 | |
| 160 | // ---- Settings ----------------------------------------------------------- |
| 161 | |
| 162 | settings(): TelemetrySettings { |
| 163 | const out: TelemetrySettings = { ...DEFAULT_TELEMETRY_SETTINGS }; |
| 164 | for (const row of this.rows("SELECT key, value FROM settings")) { |
| 165 | if (row.key === "retention_days") out.retention_days = Number(row.value); |
| 166 | if (row.key === "deployments") out.deployments = row.value === "1"; |
| 167 | } |
| 168 | return out; |
| 169 | } |
| 170 | |
| 171 | setSettings(change: Partial<TelemetrySettings>): TelemetrySettings { |
| 172 | const put = (key: string, value: string) => |
| 173 | this.sql.exec("INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", key, value); |
| 174 | if (change.retention_days !== undefined) { |
| 175 | if (!(RETENTION_CHOICES as readonly number[]).includes(change.retention_days)) throw new Error(`Retention is one of ${RETENTION_CHOICES.join(", ")} days.`); |
| 176 | put("retention_days", String(change.retention_days)); |
| 177 | } |
| 178 | if (change.deployments !== undefined) put("deployments", change.deployments ? "1" : "0"); |
| 179 | return this.settings(); |
| 180 | } |
| 181 | |
| 182 | // ---- Tokens ------------------------------------------------------------- |
| 183 | |
| 184 | tokenRow(row: Row): IngestToken { |
| 185 | return { |
| 186 | id: String(row.id), |
| 187 | name: String(row.name), |
| 188 | hint: String(row.hint), |
| 189 | created_by: strOrNull(row.created_by), |
| 190 | created_us: Number(row.created_us), |
| 191 | last_used_us: numOrNull(row.last_used_us), |
| 192 | managed: row.managed === 1, |
| 193 | }; |
| 194 | } |
| 195 | |
| 196 | /** |
| 197 | * The tokens people made. The one g1t's deployments send with is kept |
| 198 | * out of sight: it is g1t's, given to the project's apps on their own. |
| 199 | */ |
| 200 | tokens(): IngestToken[] { |
| 201 | return this.rows("SELECT * FROM tokens WHERE managed = 0 ORDER BY created_us DESC").map((r) => this.tokenRow(r)); |
| 202 | } |
| 203 | |
| 204 | /** A new token: `g1to_<project>_<secret>`, so the ingest door knows whose store to ask. */ |
| 205 | async createToken(projectId: string, name: string, createdBy: string | null, managed = false): Promise<{ token: IngestToken; secret: string }> { |
| 206 | const secret = `${TELEMETRY_TOKEN_PREFIX}${projectId}_${randomId(24)}`; |
| 207 | const id = randomId(8); |
| 208 | const created = Date.now() * 1000; |
| 209 | this.sql.exec( |
| 210 | "INSERT INTO tokens (id, name, hash, hint, created_by, created_us, managed, secret) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", |
| 211 | id, |
| 212 | name.trim().slice(0, 80) || "Ingest token", |
| 213 | await sha256(secret), |
| 214 | `${secret.slice(0, TELEMETRY_TOKEN_PREFIX.length + 4)}…${secret.slice(-4)}`, |
| 215 | createdBy, |
| 216 | created, |
| 217 | managed ? 1 : 0, |
| 218 | managed ? secret : null, |
| 219 | ); |
| 220 | return { token: this.tokenRow(this.rows("SELECT * FROM tokens WHERE id = ?", id)[0]), secret }; |
| 221 | } |
| 222 | |
| 223 | revokeToken(id: string): boolean { |
| 224 | return this.sql.exec("DELETE FROM tokens WHERE id = ? AND managed = 0", id).rowsWritten > 0; |
| 225 | } |
| 226 | |
| 227 | async deploymentToken(projectId: string): Promise<{ secret: string }> { |
| 228 | const row = this.rows("SELECT secret FROM tokens WHERE managed = 1 LIMIT 1")[0]; |
| 229 | if (row?.secret) return { secret: String(row.secret) }; |
| 230 | const made = await this.createToken(projectId, "Deployments on g1t.page", null, true); |
| 231 | return { secret: made.secret }; |
| 232 | } |
| 233 | |
| 234 | /** Whether `secret` is one of this project's tokens; marks it used. */ |
| 235 | async checkToken(secret: string): Promise<boolean> { |
| 236 | const hash = await sha256(secret); |
| 237 | const row = this.rows("SELECT id, last_used_us FROM tokens WHERE hash = ?", hash)[0]; |
| 238 | if (!row) return false; |
| 239 | const now = Date.now() * 1000; |
| 240 | if (!row.last_used_us || now - Number(row.last_used_us) > MINUTE_US) { |
| 241 | this.sql.exec("UPDATE tokens SET last_used_us = ? WHERE id = ?", now, row.id); |
| 242 | } |
| 243 | return true; |
| 244 | } |
| 245 | |
| 246 | // ---- Ingest ------------------------------------------------------------- |
| 247 | |
| 248 | /** |
| 249 | * Adds what arrived. `token` is checked first when given (the OTLP door); |
| 250 | * tail events come from Cloudflare and need none. Returns false for a |
| 251 | * token that is not this project's, or when this kind of telemetry is off. |
| 252 | */ |
| 253 | async ingest(batch: Batch, halves: RequestHalf[] = [], token?: string): Promise<{ accepted: boolean; reason?: string }> { |
| 254 | if (token !== undefined && !(await this.checkToken(token))) return { accepted: false, reason: "token" }; |
| 255 | if (token === undefined && !this.settings().deployments) return { accepted: false, reason: "off" }; |
| 256 | |
| 257 | const live: LiveItem[] = []; |
| 258 | for (const h of halves) this.putHalf(h); |
| 259 | for (const r of batch.requests) this.putHalf({ ...r, half: "app" }); |
| 260 | for (const log of batch.logs) { |
| 261 | this.sql.exec( |
| 262 | `INSERT OR IGNORE INTO logs (id, time_us, severity_number, body, trace_id, span_id, request_id, attributes, service, environment, deployment_id, commit_sha, source) |
| 263 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, |
| 264 | log.id, log.time_us, log.severity_number, log.body, log.trace_id, log.span_id, log.request_id, JSON.stringify(log.attributes), |
| 265 | log.service, log.environment, log.deployment_id, log.commit_sha, log.source, |
| 266 | ); |
| 267 | live.push({ type: "log", log }); |
| 268 | } |
| 269 | for (const s of batch.spans) { |
| 270 | this.sql.exec( |
| 271 | `INSERT OR REPLACE INTO spans (trace_id, span_id, parent_span_id, name, kind, start_us, duration_us, status, status_message, attributes, resource, events, links, |
| 272 | service, environment, deployment_id, commit_sha, source) |
| 273 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, |
| 274 | s.trace_id, s.span_id, s.parent_span_id, s.name, s.kind, s.start_us, s.duration_us, s.status, s.status_message, |
| 275 | JSON.stringify(s.attributes), JSON.stringify(s.resource), JSON.stringify(s.events), JSON.stringify(s.links), |
| 276 | s.service, s.environment, s.deployment_id, s.commit_sha, s.source, |
| 277 | ); |
| 278 | live.push({ type: "span", span: s }); |
| 279 | } |
| 280 | for (const m of batch.metrics) { |
| 281 | this.sql.exec( |
| 282 | `INSERT INTO metric_points (name, kind, time_us, value, count, sum, min, max, bounds, counts, quantiles, monotonic, attributes, service, environment, deployment_id, source) |
| 283 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, |
| 284 | m.name, m.kind, m.time_us, m.value, m.count, m.sum, m.min, m.max, |
| 285 | m.bounds ? JSON.stringify(m.bounds) : null, m.counts ? JSON.stringify(m.counts) : null, m.quantiles ? JSON.stringify(m.quantiles) : null, |
| 286 | m.monotonic ? 1 : 0, JSON.stringify(m.attributes), m.service, m.environment, m.deployment_id, m.source, |
| 287 | ); |
| 288 | const known = this.rows("SELECT services FROM metric_names WHERE name = ?", m.name)[0]; |
| 289 | const services = new Set<string>(json<string[]>(known?.services ?? null, [])); |
| 290 | services.add(m.service); |
| 291 | this.sql.exec( |
| 292 | `INSERT INTO metric_names (name, kind, unit, description, services, last_us) VALUES (?, ?, ?, ?, ?, ?) |
| 293 | ON CONFLICT (name) DO UPDATE SET kind = excluded.kind, unit = excluded.unit, |
| 294 | description = CASE WHEN excluded.description = '' THEN metric_names.description ELSE excluded.description END, |
| 295 | services = excluded.services, last_us = MAX(metric_names.last_us, excluded.last_us)`, |
| 296 | m.name, m.kind, m.unit, m.description, JSON.stringify([...services].slice(0, 50)), m.time_us, |
| 297 | ); |
| 298 | } |
| 299 | for (const e of batch.errors) await this.seeError(e); |
| 300 | |
| 301 | const requestCount = halves.filter((h) => h.half === "edge").length + batch.requests.length + halves.filter((h) => h.half === "app" && h.trigger !== "fetch").length; |
| 302 | this.sql.exec( |
| 303 | `INSERT INTO usage (month, ingested_bytes, requests, logs, spans, metric_points) VALUES (?, ?, ?, ?, ?, ?) |
| 304 | ON CONFLICT (month) DO UPDATE SET ingested_bytes = ingested_bytes + excluded.ingested_bytes, requests = requests + excluded.requests, |
| 305 | logs = logs + excluded.logs, spans = spans + excluded.spans, metric_points = metric_points + excluded.metric_points`, |
| 306 | month(), batch.bytes, requestCount, batch.logs.length, batch.spans.length, batch.metrics.length, |
| 307 | ); |
| 308 | |
| 309 | // Requests go live (and to exports) whole: once both halves are in, or |
| 310 | // for OTLP, at once. |
| 311 | const ids = [...new Set([...halves.map((h) => h.id), ...batch.requests.map((r) => r.id)])]; |
| 312 | const requests = ids.length ? this.requestsByIds(ids) : []; |
| 313 | for (const r of requests) live.unshift({ type: "request", request: r }); |
| 314 | this.broadcast(live); |
| 315 | // Requests are sent on when they are counted (`roll`), once both halves are in. |
| 316 | this.state.waitUntil(this.forward({ ...batch, requests: [] }, [])); |
| 317 | await this.ensureAlarm(); |
| 318 | return { accepted: true }; |
| 319 | } |
| 320 | |
| 321 | /** One half of a request (./tail.ts), merged with the other. */ |
| 322 | putHalf(h: RequestHalf) { |
| 323 | const edge = h.half === "edge"; |
| 324 | this.sql.exec( |
| 325 | `INSERT INTO requests (id, time_us, duration_us, trigger, method, url, route, status, outcome, cpu_ms, region, country, trace_id, log_count, |
| 326 | error_name, error_message, service, environment, deployment_id, commit_sha, source, edge, app) |
| 327 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 328 | ON CONFLICT (id) DO UPDATE SET |
| 329 | time_us = MIN(requests.time_us, excluded.time_us), |
| 330 | -- The dispatcher's half has the whole request: its time and status win. |
| 331 | duration_us = CASE WHEN excluded.edge = 1 OR requests.edge = 0 THEN excluded.duration_us ELSE requests.duration_us END, |
| 332 | status = CASE WHEN excluded.edge = 1 OR requests.status IS NULL THEN COALESCE(excluded.status, requests.status) ELSE requests.status END, |
| 333 | trigger = COALESCE(excluded.trigger, requests.trigger), |
| 334 | method = COALESCE(requests.method, excluded.method), |
| 335 | url = COALESCE(requests.url, excluded.url), |
| 336 | route = COALESCE(requests.route, excluded.route), |
| 337 | region = COALESCE(requests.region, excluded.region), |
| 338 | country = COALESCE(requests.country, excluded.country), |
| 339 | trace_id = COALESCE(requests.trace_id, excluded.trace_id), |
| 340 | outcome = CASE WHEN excluded.app = 1 THEN excluded.outcome ELSE requests.outcome END, |
| 341 | cpu_ms = COALESCE(excluded.cpu_ms, requests.cpu_ms), |
| 342 | log_count = requests.log_count + excluded.log_count, |
| 343 | error_name = COALESCE(requests.error_name, excluded.error_name), |
| 344 | error_message = COALESCE(requests.error_message, excluded.error_message), |
| 345 | service = CASE WHEN excluded.service != '' THEN excluded.service ELSE requests.service END, |
| 346 | environment = COALESCE(excluded.environment, requests.environment), |
| 347 | deployment_id = COALESCE(excluded.deployment_id, requests.deployment_id), |
| 348 | commit_sha = COALESCE(excluded.commit_sha, requests.commit_sha), |
| 349 | edge = MAX(requests.edge, excluded.edge), |
| 350 | app = MAX(requests.app, excluded.app)`, |
| 351 | h.id, h.time_us ?? Date.now() * 1000, h.duration_us ?? 0, h.trigger ?? "fetch", h.method ?? null, h.url ?? null, h.route ?? null, |
| 352 | h.status ?? null, h.outcome ?? "ok", h.cpu_ms ?? null, h.region ?? null, h.country ?? null, h.trace_id ?? null, h.log_count ?? 0, |
| 353 | h.error?.name ?? null, h.error?.message ?? null, h.service ?? "", h.environment ?? null, h.deployment_id ?? null, h.commit_sha ?? null, |
| 354 | h.source ?? "cloudflare", edge ? 1 : 0, edge ? 0 : 1, |
| 355 | ); |
| 356 | } |
| 357 | |
| 358 | async seeError(e: Batch["errors"][number]) { |
| 359 | const fp = await fingerprint(e.name, e.message, e.stack, e.service); |
| 360 | const known = this.rows("SELECT state FROM error_groups WHERE fingerprint = ?", fp)[0]; |
| 361 | if (!known) { |
| 362 | this.sql.exec( |
| 363 | `INSERT INTO error_groups (fingerprint, name, message, stack, service, environment, first_us, last_us, count, first_deployment_id, last_request_id, last_trace_id) |
| 364 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)`, |
| 365 | fp, e.name.slice(0, 200), e.message.slice(0, 4096), e.stack?.slice(0, 16384) ?? null, e.service, e.environment, e.time_us, e.time_us, |
| 366 | e.deployment_id, e.request_id, e.trace_id, |
| 367 | ); |
| 368 | return; |
| 369 | } |
| 370 | // A resolved error that happens again is open again; an ignored one stays ignored. |
| 371 | this.sql.exec( |
| 372 | `UPDATE error_groups SET count = count + 1, last_us = MAX(last_us, ?), last_request_id = COALESCE(?, last_request_id), |
| 373 | last_trace_id = COALESCE(?, last_trace_id), stack = COALESCE(stack, ?), state = CASE WHEN state = 'resolved' THEN 'open' ELSE state END |
| 374 | WHERE fingerprint = ?`, |
| 375 | e.time_us, e.request_id, e.trace_id, e.stack ?? null, fp, |
| 376 | ); |
| 377 | } |
| 378 | |
| 379 | // ---- Live --------------------------------------------------------------- |
| 380 | |
| 381 | async fetch(request: Request): Promise<Response> { |
| 382 | if (request.headers.get("upgrade") !== "websocket") return new Response("Expected a WebSocket.\n", { status: 426 }); |
| 383 | const pair = new WebSocketPair(); |
| 384 | this.state.acceptWebSocket(pair[1]); |
| 385 | pair[1].send(JSON.stringify({ type: "hello" })); |
| 386 | return new Response(null, { status: 101, webSocket: pair[0] }); |
| 387 | } |
| 388 | |
| 389 | async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer) { |
| 390 | if (message === "ping") ws.send("pong"); |
| 391 | } |
| 392 | |
| 393 | async webSocketClose(ws: WebSocket, code: number) { |
| 394 | try { |
| 395 | ws.close(code === 1005 ? 1000 : code, "closing"); |
| 396 | } catch {} |
| 397 | } |
| 398 | |
| 399 | broadcast(items: LiveItem[]) { |
| 400 | const sockets = this.state.getWebSockets(); |
| 401 | if (!sockets.length || !items.length) return; |
| 402 | const shown = items.slice(0, LIVE_PER_BATCH); |
| 403 | const dropped = items.length - shown.length; |
| 404 | const message = JSON.stringify({ type: "items", items: shown, dropped }); |
| 405 | for (const ws of sockets) { |
| 406 | try { |
| 407 | ws.send(message); |
| 408 | } catch {} |
| 409 | } |
| 410 | } |
| 411 | |
| 412 | // ---- Exports ------------------------------------------------------------ |
| 413 | |
| 414 | exportRows(): (TelemetryExport & { headers: Record<string, string> })[] { |
| 415 | return this.rows("SELECT * FROM exports ORDER BY name").map((r) => { |
| 416 | const headers = json<Record<string, string>>(r.headers, {}); |
| 417 | return { |
| 418 | id: String(r.id), |
| 419 | name: String(r.name), |
| 420 | endpoint: String(r.endpoint), |
| 421 | signals: json(r.signals, ["traces", "logs", "metrics"]), |
| 422 | header_names: Object.keys(headers), |
| 423 | headers, |
| 424 | enabled: r.enabled === 1, |
| 425 | last_error: strOrNull(r.last_error), |
| 426 | last_sent_us: numOrNull(r.last_sent_us), |
| 427 | }; |
| 428 | }); |
| 429 | } |
| 430 | |
| 431 | exports(): TelemetryExport[] { |
| 432 | return this.exportRows().map(({ headers: _, ...rest }) => rest); |
| 433 | } |
| 434 | |
| 435 | putExport(input: { id?: string | null; name: string; endpoint: string; signals: TelemetryExport["signals"]; headers?: Record<string, string> | null; enabled: boolean }): TelemetryExport { |
| 436 | const problem = exportEndpointProblem(input.endpoint); |
| 437 | if (problem) throw new Error(problem); |
| 438 | const signals = input.signals.filter((s) => s === "traces" || s === "logs" || s === "metrics"); |
| 439 | if (!signals.length) throw new Error("Choose at least one signal to send."); |
| 440 | const id = input.id || randomId(8); |
| 441 | const before = this.rows("SELECT headers FROM exports WHERE id = ?", id)[0]; |
| 442 | // Header values are written, never read back: keep those not given again. |
| 443 | const headers = { ...json<Record<string, string>>(before?.headers ?? null, {}) }; |
| 444 | for (const [k, v] of Object.entries(input.headers ?? {})) { |
| 445 | if (v === "") delete headers[k]; |
| 446 | else headers[k] = v; |
| 447 | } |
| 448 | this.sql.exec( |
| 449 | `INSERT INTO exports (id, name, endpoint, signals, headers, enabled) VALUES (?, ?, ?, ?, ?, ?) |
| 450 | ON CONFLICT (id) DO UPDATE SET name = excluded.name, endpoint = excluded.endpoint, signals = excluded.signals, |
| 451 | headers = excluded.headers, enabled = excluded.enabled, last_error = NULL`, |
| 452 | id, input.name.trim().slice(0, 80) || "Export", input.endpoint.trim(), JSON.stringify(signals), JSON.stringify(headers), input.enabled ? 1 : 0, |
| 453 | ); |
| 454 | return this.exports().find((e) => e.id === id)!; |
| 455 | } |
| 456 | |
| 457 | deleteExport(id: string): boolean { |
| 458 | return this.sql.exec("DELETE FROM exports WHERE id = ?", id).rowsWritten > 0; |
| 459 | } |
| 460 | |
| 461 | async forward(batch: Batch, requests: TelemetryRequest[]) { |
| 462 | const targets = this.exportRows().filter((e) => e.enabled); |
| 463 | if (!targets.length || (!requests.length && !batch.logs.length && !batch.spans.length && !batch.metrics.length)) return; |
| 464 | const bodies = otlpBodies(batch, requests); |
| 465 | for (const target of targets) { |
| 466 | let error: string | null = null; |
| 467 | for (const signal of target.signals) { |
| 468 | const body = bodies[signal]; |
| 469 | if (body) error = (await send(target, signal, body)) ?? error; |
| 470 | } |
| 471 | this.sql.exec("UPDATE exports SET last_error = ?, last_sent_us = ? WHERE id = ?", error, Date.now() * 1000, target.id); |
| 472 | } |
| 473 | } |
| 474 | |
| 475 | // ---- Upkeep ------------------------------------------------------------- |
| 476 | |
| 477 | async ensureAlarm() { |
| 478 | if ((await this.state.storage.getAlarm()) === null) await this.state.storage.setAlarm(Date.now() + 60_000); |
| 479 | } |
| 480 | |
| 481 | async alarm() { |
| 482 | const rolled = this.roll(); |
| 483 | if (rolled.length) await this.forward({ requests: [], logs: [], spans: [], metrics: [], errors: [], bytes: 0 }, rolled); |
| 484 | const last = Number(this.rows("SELECT value FROM meta WHERE key = 'pruned'")[0]?.value ?? 0); |
| 485 | if (Date.now() - last > 3_600_000) { |
| 486 | this.prune(); |
| 487 | this.sql.exec("INSERT INTO meta (key, value) VALUES ('pruned', ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", String(Date.now())); |
| 488 | } |
| 489 | // Keep going while there is anything left to count or keep. |
| 490 | const pending = this.rows("SELECT 1 FROM requests LIMIT 1").length || this.rows("SELECT 1 FROM logs LIMIT 1").length || this.rows("SELECT 1 FROM spans LIMIT 1").length; |
| 491 | if (pending) await this.state.storage.setAlarm(Date.now() + 60_000); |
| 492 | } |
| 493 | |
| 494 | /** Counts settled requests into per-minute rollups, and sends them to exports. */ |
| 495 | roll(now = Date.now() * 1000): TelemetryRequest[] { |
| 496 | const rolled: TelemetryRequest[] = []; |
| 497 | for (;;) { |
| 498 | const rows = this.rows("SELECT * FROM requests WHERE rolled = 0 AND time_us < ? ORDER BY time_us, id LIMIT 2000", now - ROLL_AFTER_US); |
| 499 | if (!rows.length) break; |
| 500 | for (const a of aggregate(rows)) { |
| 501 | const before = this.rows( |
| 502 | "SELECT latency, cpu FROM rollups WHERE minute = ? AND service = ? AND environment = ? AND deployment_id = ? AND method = ? AND route = ?", |
| 503 | a.minute, a.service, a.environment, a.deployment_id, a.method, a.route, |
| 504 | )[0]; |
| 505 | const latency = encode(before ? addInto(decode(String(before.latency)), decode(a.latency)) : decode(a.latency)); |
| 506 | const cpu = encode(before ? addInto(decode(String(before.cpu)), decode(a.cpu)) : decode(a.cpu)); |
| 507 | this.sql.exec( |
| 508 | `INSERT INTO rollups (minute, service, environment, deployment_id, commit_sha, source, method, route, requests, errors, s2, s3, s4, s5, latency, cpu) |
| 509 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 510 | ON CONFLICT (minute, service, environment, deployment_id, method, route) DO UPDATE SET |
| 511 | requests = requests + excluded.requests, errors = errors + excluded.errors, s2 = s2 + excluded.s2, s3 = s3 + excluded.s3, |
| 512 | s4 = s4 + excluded.s4, s5 = s5 + excluded.s5, latency = excluded.latency, cpu = excluded.cpu, |
| 513 | commit_sha = COALESCE(excluded.commit_sha, rollups.commit_sha)`, |
| 514 | a.minute, a.service, a.environment, a.deployment_id, a.commit_sha, a.source, a.method, a.route, a.requests, a.errors, |
| 515 | a.s2, a.s3, a.s4, a.s5, latency, cpu, |
| 516 | ); |
| 517 | } |
| 518 | const last = rows[rows.length - 1]; |
| 519 | this.sql.exec("UPDATE requests SET rolled = 1 WHERE rolled = 0 AND (time_us < ? OR (time_us = ? AND id <= ?))", last.time_us, last.time_us, last.id); |
| 520 | rolled.push(...rows.map((r) => this.requestOf(r))); |
| 521 | if (rows.length < 2000) break; |
| 522 | } |
| 523 | return rolled; |
| 524 | } |
| 525 | |
| 526 | prune(now = Date.now() * 1000) { |
| 527 | const keep = now - this.settings().retention_days * DAY_US; |
| 528 | // Requests are counted before they go. |
| 529 | this.roll(now); |
| 530 | this.sql.exec("DELETE FROM requests WHERE time_us < ?", keep); |
| 531 | this.sql.exec("DELETE FROM logs WHERE time_us < ?", keep); |
| 532 | this.sql.exec("DELETE FROM spans WHERE start_us < ?", keep); |
| 533 | this.sql.exec("DELETE FROM metric_points WHERE time_us < ?", keep); |
| 534 | this.sql.exec("DELETE FROM rollups WHERE minute < ?", Math.floor((now - ROLLUP_DAYS * DAY_US) / MINUTE_US)); |
| 535 | this.sql.exec("DELETE FROM error_groups WHERE last_us < ? AND issue_number IS NULL", now - ROLLUP_DAYS * DAY_US); |
| 536 | // Over its size, the store sheds a day at a time from the oldest. |
| 537 | for (let i = 0; i < 30 && this.state.storage.sql.databaseSize > MAX_BYTES; i++) { |
| 538 | const oldest = Number(this.rows("SELECT MIN(time_us) AS t FROM logs")[0]?.t ?? now); |
| 539 | const cut = oldest + DAY_US; |
| 540 | this.sql.exec("DELETE FROM logs WHERE time_us < ?", cut); |
| 541 | this.sql.exec("DELETE FROM spans WHERE start_us < ?", cut); |
| 542 | this.sql.exec("DELETE FROM requests WHERE time_us < ?", cut); |
| 543 | this.sql.exec("DELETE FROM metric_points WHERE time_us < ?", cut); |
| 544 | } |
| 545 | } |
| 546 | |
| 547 | async purge(): Promise<boolean> { |
| 548 | for (const ws of this.state.getWebSockets()) { |
| 549 | try { |
| 550 | ws.close(1000, "gone"); |
| 551 | } catch {} |
| 552 | } |
| 553 | await this.state.storage.deleteAlarm(); |
| 554 | await this.state.storage.deleteAll(); |
| 555 | return true; |
| 556 | } |
| 557 | |
| 558 | // ---- Reading ------------------------------------------------------------ |
| 559 | |
| 560 | /** WHERE clauses for what every list narrows by. */ |
| 561 | filter(f: TelemetryFilter, time: string, prefix = ""): { clauses: string[]; args: SqlStorageValue[] } { |
| 562 | const clauses: string[] = []; |
| 563 | const args: SqlStorageValue[] = []; |
| 564 | const col = (c: string) => `${prefix}${c}`; |
| 565 | if (f.from_us) clauses.push(`${col(time)} >= ?`), args.push(f.from_us); |
| 566 | if (f.to_us) clauses.push(`${col(time)} <= ?`), args.push(f.to_us); |
| 567 | if (f.services?.length) clauses.push(`${col("service")} IN (${f.services.map(() => "?").join(",")})`), args.push(...f.services); |
| 568 | if (f.environments?.length) clauses.push(`${col("environment")} IN (${f.environments.map(() => "?").join(",")})`), args.push(...f.environments); |
| 569 | if (f.deployment_id) clauses.push(`${col("deployment_id")} = ?`), args.push(f.deployment_id); |
| 570 | if (f.source) clauses.push(`${col("source")} = ?`), args.push(f.source); |
| 571 | return { clauses, args }; |
| 572 | } |
| 573 | |
| 574 | /** `key=value` pairs: a column the table has, or an attribute. */ |
| 575 | pairs(pairs: [string, string][], columns: string[], clauses: string[], args: SqlStorageValue[], attributes: string) { |
| 576 | for (const [key, value] of pairs) { |
| 577 | if (columns.includes(key)) { |
| 578 | clauses.push(`CAST(${attributes.includes(".") ? `${attributes.split(".")[0]}.` : ""}${key} AS TEXT) = ?`); |
| 579 | args.push(value); |
| 580 | } else if (attributes) { |
| 581 | clauses.push(`CAST(json_extract(${attributes}, ?) AS TEXT) = ?`); |
| 582 | args.push(`$."${key.replace(/"/g, "")}"`, value); |
| 583 | } |
| 584 | } |
| 585 | } |
| 586 | |
| 587 | requestOf(r: Row): TelemetryRequest { |
| 588 | return { |
| 589 | id: String(r.id), |
| 590 | time_us: Number(r.time_us), |
| 591 | duration_us: Number(r.duration_us), |
| 592 | trigger: String(r.trigger), |
| 593 | method: strOrNull(r.method), |
| 594 | url: strOrNull(r.url), |
| 595 | route: strOrNull(r.route), |
| 596 | status: numOrNull(r.status), |
| 597 | outcome: String(r.outcome), |
| 598 | cpu_ms: numOrNull(r.cpu_ms), |
| 599 | region: strOrNull(r.region), |
| 600 | country: strOrNull(r.country), |
| 601 | trace_id: strOrNull(r.trace_id), |
| 602 | log_count: Number(r.log_count ?? 0), |
| 603 | error: r.error_name ? { name: String(r.error_name), message: String(r.error_message ?? "") } : null, |
| 604 | service: String(r.service ?? ""), |
| 605 | environment: strOrNull(r.environment), |
| 606 | deployment_id: strOrNull(r.deployment_id), |
| 607 | commit_sha: strOrNull(r.commit_sha), |
| 608 | source: r.source === "otlp" ? "otlp" : "cloudflare", |
| 609 | }; |
| 610 | } |
| 611 | |
| 612 | requestsByIds(ids: string[]): TelemetryRequest[] { |
| 613 | return this.rows(`SELECT * FROM requests WHERE id IN (${ids.map(() => "?").join(",")})`, ...ids).map((r) => this.requestOf(r)); |
| 614 | } |
| 615 | |
| 616 | logOf(r: Row): TelemetryLog { |
| 617 | const n = Number(r.severity_number); |
| 618 | return { |
| 619 | id: String(r.id), |
| 620 | time_us: Number(r.time_us), |
| 621 | severity: severityOf(n), |
| 622 | severity_number: n, |
| 623 | body: String(r.body), |
| 624 | trace_id: strOrNull(r.trace_id), |
| 625 | span_id: strOrNull(r.span_id), |
| 626 | request_id: strOrNull(r.request_id), |
| 627 | attributes: json<Attributes>(r.attributes, {}), |
| 628 | service: String(r.service ?? ""), |
| 629 | environment: strOrNull(r.environment), |
| 630 | deployment_id: strOrNull(r.deployment_id), |
| 631 | commit_sha: strOrNull(r.commit_sha), |
| 632 | source: r.source === "otlp" ? "otlp" : "cloudflare", |
| 633 | }; |
| 634 | } |
| 635 | |
| 636 | spanOf(r: Row): TelemetrySpan { |
| 637 | return { |
| 638 | trace_id: String(r.trace_id), |
| 639 | span_id: String(r.span_id), |
| 640 | parent_span_id: strOrNull(r.parent_span_id), |
| 641 | name: String(r.name), |
| 642 | kind: String(r.kind) as TelemetrySpan["kind"], |
| 643 | start_us: Number(r.start_us), |
| 644 | duration_us: Number(r.duration_us), |
| 645 | status: String(r.status) as TelemetrySpan["status"], |
| 646 | status_message: strOrNull(r.status_message), |
| 647 | attributes: json(r.attributes, {}), |
| 648 | resource: json(r.resource, {}), |
| 649 | events: json(r.events, []), |
| 650 | links: json(r.links, []), |
| 651 | service: String(r.service ?? ""), |
| 652 | environment: strOrNull(r.environment), |
| 653 | deployment_id: strOrNull(r.deployment_id), |
| 654 | commit_sha: strOrNull(r.commit_sha), |
| 655 | source: r.source === "otlp" ? "otlp" : "cloudflare", |
| 656 | }; |
| 657 | } |
| 658 | |
| 659 | requests(q: RequestQuery): Page<TelemetryRequest> { |
| 660 | const { clauses, args } = this.filter(q, "time_us"); |
| 661 | const { words, pairs } = parseSearch(q.text); |
| 662 | this.pairs(pairs, ["method", "route", "status", "outcome", "region", "country", "trace_id", "service", "environment", "deployment_id", "commit_sha", "trigger"], clauses, args, ""); |
| 663 | for (const w of words) clauses.push("(url LIKE ? OR error_message LIKE ? OR error_name LIKE ?)"), args.push(`%${w}%`, `%${w}%`, `%${w}%`); |
| 664 | if (q.status?.length) { |
| 665 | const parts: string[] = []; |
| 666 | for (const s of q.status) { |
| 667 | const cls = /^([1-5])xx$/i.exec(s); |
| 668 | if (cls) parts.push("(status >= ? AND status < ?)"), args.push(Number(cls[1]) * 100, Number(cls[1]) * 100 + 100); |
| 669 | else if (/^\d{3}$/.test(s)) parts.push("status = ?"), args.push(Number(s)); |
| 670 | } |
| 671 | if (parts.length) clauses.push(`(${parts.join(" OR ")})`); |
| 672 | } |
| 673 | if (q.errors_only) clauses.push(ERROR_SQL); |
| 674 | if (q.route) clauses.push("route = ?"), args.push(q.route); |
| 675 | if (q.min_duration_ms) clauses.push("duration_us >= ?"), args.push(q.min_duration_ms * 1000); |
| 676 | const after = readCursor(q.cursor); |
| 677 | if (after) clauses.push("(time_us < ? OR (time_us = ? AND id < ?))"), args.push(after.time, after.time, after.id); |
| 678 | const limit = limitOf(q.limit); |
| 679 | const rows = this.rows( |
| 680 | `SELECT * FROM requests ${clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""} ORDER BY time_us DESC, id DESC LIMIT ?`, |
| 681 | ...args, |
| 682 | limit + 1, |
| 683 | ); |
| 684 | const items = rows.slice(0, limit).map((r) => this.requestOf(r)); |
| 685 | const last = items[items.length - 1]; |
| 686 | return { items, next: rows.length > limit && last ? cursorOf(last.time_us, last.id) : null }; |
| 687 | } |
| 688 | |
| 689 | request(id: string): { request: TelemetryRequest; logs: TelemetryLog[] } | null { |
| 690 | const row = this.rows("SELECT * FROM requests WHERE id = ?", id)[0]; |
| 691 | if (!row) return null; |
| 692 | const logs = this.rows("SELECT * FROM logs WHERE request_id = ? ORDER BY time_us, rowid LIMIT 1000", id).map((r) => this.logOf(r)); |
| 693 | return { request: this.requestOf(row), logs }; |
| 694 | } |
| 695 | |
| 696 | logs(q: LogQuery): Page<TelemetryLog> { |
| 697 | const { clauses, args } = this.filter(q, "time_us", "logs."); |
| 698 | const { words, pairs } = parseSearch(q.text); |
| 699 | const isLevel = ([k]: [string, string]) => k === "level" || k === "severity"; |
| 700 | this.pairs(pairs.filter((p) => !isLevel(p)), ["trace_id", "span_id", "request_id", "service", "environment", "deployment_id", "commit_sha"], clauses, args, "logs.attributes"); |
| 701 | const levels = [...(q.severities ?? []), ...pairs.filter(isLevel).map(([, v]) => v.toLowerCase() as LogSeverity)]; |
| 702 | const ranges = levels.filter((l) => SEVERITY_FLOOR[l]).map((l) => SEVERITY_FLOOR[l]); |
| 703 | if (ranges.length) { |
| 704 | clauses.push(`(${ranges.map(() => "logs.severity_number BETWEEN ? AND ?").join(" OR ")})`); |
| 705 | for (const [lo, hi] of ranges) args.push(lo, hi); |
| 706 | } |
| 707 | if (q.trace_id) clauses.push("logs.trace_id = ?"), args.push(q.trace_id); |
| 708 | if (q.request_id) clauses.push("logs.request_id = ?"), args.push(q.request_id); |
| 709 | let from = "logs"; |
| 710 | if (words.length && this.fts) { |
| 711 | from = "logs JOIN logs_fts ON logs_fts.rowid = logs.rowid"; |
| 712 | clauses.push("logs_fts MATCH ?"); |
| 713 | args.push(ftsQuery(words)); |
| 714 | } else { |
| 715 | for (const w of words) clauses.push("logs.body LIKE ?"), args.push(`%${w}%`); |
| 716 | } |
| 717 | const after = readCursor(q.cursor); |
| 718 | if (after) clauses.push("(logs.time_us < ? OR (logs.time_us = ? AND logs.id < ?))"), args.push(after.time, after.time, after.id); |
| 719 | const limit = limitOf(q.limit, 200, 1000); |
| 720 | const rows = this.rows( |
| 721 | `SELECT logs.* FROM ${from} ${clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""} ORDER BY logs.time_us DESC, logs.id DESC LIMIT ?`, |
| 722 | ...args, |
| 723 | limit + 1, |
| 724 | ); |
| 725 | const items = rows.slice(0, limit).map((r) => this.logOf(r)); |
| 726 | const last = items[items.length - 1]; |
| 727 | return { items, next: rows.length > limit && last ? cursorOf(last.time_us, last.id) : null }; |
| 728 | } |
| 729 | |
| 730 | traces(q: TraceQuery): Page<TraceSummary> { |
| 731 | const { clauses, args } = this.filter(q, "start_us"); |
| 732 | const { words, pairs } = parseSearch(q.text); |
| 733 | this.pairs(pairs, ["name", "service", "environment", "deployment_id", "kind", "status"], clauses, args, "attributes"); |
| 734 | for (const w of words) clauses.push("name LIKE ?"), args.push(`%${w}%`); |
| 735 | const after = readCursor(q.cursor); |
| 736 | const having: string[] = []; |
| 737 | const havingArgs: SqlStorageValue[] = []; |
| 738 | if (q.errors_only) having.push("SUM(status = 'error') > 0"); |
| 739 | if (q.min_duration_ms) having.push("MAX(start_us + duration_us) - MIN(start_us) >= ?"), havingArgs.push(q.min_duration_ms * 1000); |
| 740 | if (after) having.push("(MIN(start_us) < ? OR (MIN(start_us) = ? AND trace_id < ?))"), havingArgs.push(after.time, after.time, after.id); |
| 741 | const limit = limitOf(q.limit, 50, 200); |
| 742 | const rows = this.rows( |
| 743 | `SELECT trace_id, MIN(start_us) AS start_us, MAX(start_us + duration_us) - MIN(start_us) AS duration_us, COUNT(*) AS span_count, |
| 744 | SUM(status = 'error') AS error_count, GROUP_CONCAT(DISTINCT service) AS services |
| 745 | FROM spans ${clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""} |
| 746 | GROUP BY trace_id ${having.length ? `HAVING ${having.join(" AND ")}` : ""} |
| 747 | ORDER BY start_us DESC, trace_id DESC LIMIT ?`, |
| 748 | ...args, |
| 749 | ...havingArgs, |
| 750 | limit + 1, |
| 751 | ); |
| 752 | const items: TraceSummary[] = rows.slice(0, limit).map((r) => { |
| 753 | const traceId = String(r.trace_id); |
| 754 | // The trace's request on g1t.page (its true root), else its earliest span. |
| 755 | const request = this.rows("SELECT * FROM requests WHERE trace_id = ? ORDER BY time_us LIMIT 1", traceId)[0]; |
| 756 | const first = this.rows("SELECT * FROM spans WHERE trace_id = ? ORDER BY parent_span_id IS NOT NULL, start_us LIMIT 1", traceId)[0]; |
| 757 | const req = request ? this.requestOf(request) : null; |
| 758 | const start = Math.min(Number(r.start_us), req?.time_us ?? Infinity); |
| 759 | const end = Math.max(Number(r.start_us) + Number(r.duration_us), req ? req.time_us + req.duration_us : 0); |
| 760 | return { |
| 761 | trace_id: traceId, |
| 762 | name: req ? [req.method, req.route].filter(Boolean).join(" ") : String(first?.name ?? ""), |
| 763 | start_us: start, |
| 764 | duration_us: end - start, |
| 765 | span_count: Number(r.span_count) + (req ? 1 : 0), |
| 766 | error_count: Number(r.error_count) + (req?.error ? 1 : 0), |
| 767 | services: [...new Set([...(req ? [req.service] : []), ...String(r.services ?? "").split(",").filter(Boolean)])], |
| 768 | status: req?.status ?? numOrNull(json<Attributes>(first?.attributes ?? null, {})["http.response.status_code"] as SqlStorageValue), |
| 769 | service: req?.service ?? String(first?.service ?? ""), |
| 770 | environment: req?.environment ?? strOrNull(first?.environment ?? null), |
| 771 | deployment_id: req?.deployment_id ?? strOrNull(first?.deployment_id ?? null), |
| 772 | commit_sha: req?.commit_sha ?? strOrNull(first?.commit_sha ?? null), |
| 773 | source: req?.source ?? (first?.source === "cloudflare" ? "cloudflare" : "otlp"), |
| 774 | }; |
| 775 | }); |
| 776 | const last = rows[Math.min(rows.length, limit) - 1]; |
| 777 | return { items, next: rows.length > limit && last ? cursorOf(Number(last.start_us), String(last.trace_id)) : null }; |
| 778 | } |
| 779 | |
| 780 | /** |
| 781 | * A trace: its spans, with each request on g1t.page in it as the server |
| 782 | * span it was (the parent the app's own spans name), and every log |
| 783 | * written under it. |
| 784 | */ |
| 785 | trace(traceId: string): TraceDetail | null { |
| 786 | const spans = this.rows("SELECT * FROM spans WHERE trace_id = ? ORDER BY start_us LIMIT 5000", traceId).map((r) => this.spanOf(r)); |
| 787 | const requests = this.rows("SELECT * FROM requests WHERE trace_id = ? ORDER BY time_us LIMIT 100", traceId).map((r) => this.requestOf(r)); |
| 788 | if (!spans.length && !requests.length) return null; |
| 789 | const have = new Set(spans.map((s) => s.span_id)); |
| 790 | for (const r of requests) { |
| 791 | if (have.has(r.id)) continue; |
| 792 | spans.unshift({ |
| 793 | trace_id: traceId, |
| 794 | span_id: r.id, |
| 795 | parent_span_id: null, |
| 796 | name: [r.method, r.route].filter(Boolean).join(" ") || r.trigger, |
| 797 | kind: "server", |
| 798 | start_us: r.time_us, |
| 799 | duration_us: r.duration_us, |
| 800 | status: r.error || (r.status ?? 0) >= 500 ? "error" : "unset", |
| 801 | status_message: r.error?.message ?? null, |
| 802 | attributes: { |
| 803 | ...(r.method ? { "http.request.method": r.method } : {}), |
| 804 | ...(r.url ? { "url.full": `https://${r.url}` } : {}), |
| 805 | ...(r.status !== null ? { "http.response.status_code": r.status } : {}), |
| 806 | ...(r.cpu_ms !== null ? { "cloudflare.cpu_time_ms": r.cpu_ms } : {}), |
| 807 | ...(r.region ? { "cloudflare.colo": r.region } : {}), |
| 808 | "cloudflare.outcome": r.outcome, |
| 809 | "g1t.request.id": r.id, |
| 810 | }, |
| 811 | resource: { "service.name": r.service }, |
| 812 | events: [], |
| 813 | links: [], |
| 814 | service: r.service, |
| 815 | environment: r.environment, |
| 816 | deployment_id: r.deployment_id, |
| 817 | commit_sha: r.commit_sha, |
| 818 | source: r.source, |
| 819 | }); |
| 820 | } |
| 821 | const ids = requests.map((r) => r.id); |
| 822 | const logs = this.rows( |
| 823 | `SELECT * FROM logs WHERE trace_id = ? ${ids.length ? `OR request_id IN (${ids.map(() => "?").join(",")})` : ""} ORDER BY time_us LIMIT 2000`, |
| 824 | traceId, |
| 825 | ...ids, |
| 826 | ).map((r) => this.logOf(r)); |
| 827 | return { trace_id: traceId, spans, logs }; |
| 828 | } |
| 829 | |
| 830 | errors(f: TelemetryFilter & { state?: ErrorGroup["state"] | null }): ErrorGroup[] { |
| 831 | const { clauses, args } = this.filter({ ...f, deployment_id: null, source: null }, "last_us"); |
| 832 | if (f.state) clauses.push("state = ?"), args.push(f.state); |
| 833 | return this.rows(`SELECT * FROM error_groups ${clauses.length ? `WHERE ${clauses.join(" AND ")}` : ""} ORDER BY last_us DESC LIMIT 200`, ...args).map(errorOf); |
| 834 | } |
| 835 | |
| 836 | setError(fp: string, change: { state?: ErrorGroup["state"]; issue_number?: number | null }): ErrorGroup | null { |
| 837 | if (change.state && !["open", "resolved", "ignored"].includes(change.state)) throw new Error("State is open, resolved or ignored."); |
| 838 | if (change.state) this.sql.exec("UPDATE error_groups SET state = ? WHERE fingerprint = ?", change.state, fp); |
| 839 | if (change.issue_number !== undefined) this.sql.exec("UPDATE error_groups SET issue_number = ? WHERE fingerprint = ?", change.issue_number, fp); |
| 840 | const row = this.rows("SELECT * FROM error_groups WHERE fingerprint = ?", fp)[0]; |
| 841 | return row ? errorOf(row) : null; |
| 842 | } |
| 843 | |
| 844 | metrics(): MetricInfo[] { |
| 845 | return this.rows("SELECT * FROM metric_names ORDER BY name").map((r) => ({ |
| 846 | name: String(r.name), |
| 847 | kind: String(r.kind) as MetricInfo["kind"], |
| 848 | unit: String(r.unit), |
| 849 | description: String(r.description), |
| 850 | services: json(r.services, []), |
| 851 | last_us: Number(r.last_us), |
| 852 | })); |
| 853 | } |
| 854 | |
| 855 | /** |
| 856 | * A metric over time. Gauges average (or take the min or max) in each |
| 857 | * step; sums add (a cumulative sum becomes the increase in the step); |
| 858 | * histograms give their mean, or a percentile from their buckets. |
| 859 | */ |
| 860 | metric(q: TelemetryFilter & { name: string; aggregate?: string | null; group_by?: string | null; step_s?: number | null }): MetricSeries { |
| 861 | const now = Date.now() * 1000; |
| 862 | const from = q.from_us ?? now - DAY_US; |
| 863 | const to = q.to_us ?? now; |
| 864 | const step = q.step_s ?? stepFor(to - from); |
| 865 | const { clauses, args } = this.filter({ ...q, from_us: from, to_us: to }, "time_us"); |
| 866 | clauses.push("name = ?"); |
| 867 | args.push(q.name); |
| 868 | const info = this.rows("SELECT unit, kind FROM metric_names WHERE name = ?", q.name)[0]; |
| 869 | const rows = this.rows(`SELECT * FROM metric_points WHERE ${clauses.join(" AND ")} ORDER BY time_us LIMIT 200000`, ...args); |
| 870 | const stepUs = step * 1_000_000; |
| 871 | const groups = new Map<string | null, Map<number, number[]>>(); |
| 872 | const lastCumulative = new Map<string, number>(); |
| 873 | const aggregate = q.aggregate ?? (info?.kind === "sum" ? "sum" : "avg"); |
| 874 | for (const r of rows) { |
| 875 | const attrs = json<Attributes>(r.attributes, {}); |
| 876 | const group = q.group_by ? String(q.group_by === "service" ? r.service : (attrs[q.group_by] ?? "")) : null; |
| 877 | const bucket = Math.floor(Number(r.time_us) / stepUs) * stepUs; |
| 878 | let value: number | null = numOrNull(r.value); |
| 879 | if (r.kind === "sum" && r.monotonic === 1) { |
| 880 | // Each series of a cumulative sum is told apart by its attributes. |
| 881 | const key = `${r.service}\u0000${String(r.attributes)}`; |
| 882 | const before = lastCumulative.get(key); |
| 883 | lastCumulative.set(key, value ?? 0); |
| 884 | value = before === undefined || (value ?? 0) < before ? null : (value ?? 0) - before; |
| 885 | } |
| 886 | if (r.kind === "histogram" && /^p\d+$/.test(aggregate) && r.counts && r.bounds) { |
| 887 | value = histogramQuantile(json(r.bounds, []), json(r.counts, []), Number(aggregate.slice(1)) / 100); |
| 888 | } |
| 889 | if (value === null) continue; |
| 890 | let g = groups.get(group); |
| 891 | if (!g) groups.set(group, (g = new Map())); |
| 892 | g.set(bucket, [...(g.get(bucket) ?? []), value]); |
| 893 | } |
| 894 | const reduce = (values: number[]): number => { |
| 895 | switch (aggregate) { |
| 896 | case "sum": |
| 897 | return values.reduce((a, b) => a + b, 0); |
| 898 | case "min": |
| 899 | return Math.min(...values); |
| 900 | case "max": |
| 901 | return Math.max(...values); |
| 902 | default: |
| 903 | return values.reduce((a, b) => a + b, 0) / values.length; |
| 904 | } |
| 905 | }; |
| 906 | const start = Math.floor(from / stepUs) * stepUs; |
| 907 | return { |
| 908 | name: q.name, |
| 909 | unit: String(info?.unit ?? ""), |
| 910 | step_s: step, |
| 911 | series: [...groups.entries()].slice(0, 20).map(([group, points]) => { |
| 912 | const out: { time_us: number; value: number | null }[] = []; |
| 913 | for (let t = start; t <= to; t += stepUs) { |
| 914 | const values = points.get(t); |
| 915 | out.push({ time_us: t, value: values ? reduce(values) : null }); |
| 916 | } |
| 917 | return { group, points: out }; |
| 918 | }), |
| 919 | }; |
| 920 | } |
| 921 | |
| 922 | facets(): { services: string[]; environments: string[]; attribute_keys: string[] } { |
| 923 | const since = Date.now() * 1000 - 7 * DAY_US; |
| 924 | const services = this.rows( |
| 925 | `SELECT service FROM rollups WHERE minute >= ? AND service != '' GROUP BY service |
| 926 | UNION SELECT service FROM spans WHERE start_us >= ? AND service != '' GROUP BY service |
| 927 | UNION SELECT service FROM logs WHERE time_us >= ? AND service != '' GROUP BY service`, |
| 928 | Math.floor(since / MINUTE_US), since, since, |
| 929 | ).map((r) => String(r.service)); |
| 930 | const environments = this.rows( |
| 931 | `SELECT environment FROM rollups WHERE minute >= ? AND environment != '' GROUP BY environment |
| 932 | UNION SELECT environment FROM logs WHERE time_us >= ? AND environment IS NOT NULL GROUP BY environment`, |
| 933 | Math.floor(since / MINUTE_US), since, |
| 934 | ).map((r) => String(r.environment)); |
| 935 | const keys = new Set<string>(); |
| 936 | for (const r of this.rows("SELECT attributes FROM logs ORDER BY time_us DESC LIMIT 500")) for (const k of Object.keys(json(r.attributes, {}))) keys.add(k); |
| 937 | for (const r of this.rows("SELECT attributes FROM spans ORDER BY start_us DESC LIMIT 500")) for (const k of Object.keys(json(r.attributes, {}))) keys.add(k); |
| 938 | return { services: [...new Set(services)].sort(), environments: [...new Set(environments)].sort(), attribute_keys: [...keys].sort().slice(0, 200) }; |
| 939 | } |
| 940 | |
| 941 | usage(): TelemetryUsage { |
| 942 | const m = month(); |
| 943 | const row = this.rows("SELECT * FROM usage WHERE month = ?", m)[0]; |
| 944 | return { |
| 945 | month: m, |
| 946 | ingested_bytes: Number(row?.ingested_bytes ?? 0), |
| 947 | requests: Number(row?.requests ?? 0), |
| 948 | logs: Number(row?.logs ?? 0), |
| 949 | spans: Number(row?.spans ?? 0), |
| 950 | metric_points: Number(row?.metric_points ?? 0), |
| 951 | stored_bytes: this.state.storage.sql.databaseSize, |
| 952 | }; |
| 953 | } |
| 954 | |
| 955 | overview(f: TelemetryFilter & { step_s?: number | null }): TelemetryOverview { |
| 956 | const now = Date.now() * 1000; |
| 957 | const to = f.to_us ?? now; |
| 958 | const from = f.from_us ?? to - DAY_US; |
| 959 | const step = f.step_s && STEPS.includes(f.step_s) ? f.step_s : stepFor(to - from); |
| 960 | const read = (lo: number, hi: number, detail: boolean) => { |
| 961 | const { clauses, args } = this.filter({ ...f, from_us: null, to_us: null }, "minute"); |
| 962 | clauses.push("minute >= ?", "minute <= ?", detail ? "route != '*'" : "route = '*'"); |
| 963 | args.push(Math.floor(lo / MINUTE_US), Math.floor(hi / MINUTE_US)); |
| 964 | const counted = this.rows(`SELECT * FROM rollups WHERE ${clauses.join(" AND ")}`, ...args); |
| 965 | // The newest requests, not counted yet, are counted here as they stand. |
| 966 | const { clauses: pc, args: pa } = this.filter({ ...f, from_us: lo, to_us: hi }, "time_us"); |
| 967 | pc.push("rolled = 0"); |
| 968 | const pending = aggregate(this.rows(`SELECT * FROM requests WHERE ${pc.join(" AND ")} LIMIT 20000`, ...pa)) |
| 969 | .filter((a) => (detail ? a.route !== "*" : a.route === "*")) as unknown as Row[]; |
| 970 | return [...counted, ...pending]; |
| 971 | }; |
| 972 | const totalsOf = (rows: Row[]): TelemetryTotals => { |
| 973 | let requests = 0; |
| 974 | let errors = 0; |
| 975 | const latency = emptyHistogram(); |
| 976 | const cpu = emptyHistogram(); |
| 977 | for (const r of rows) { |
| 978 | requests += Number(r.requests); |
| 979 | errors += Number(r.errors); |
| 980 | addInto(latency, decode(String(r.latency))); |
| 981 | addInto(cpu, decode(String(r.cpu))); |
| 982 | } |
| 983 | return { requests, errors, error_rate: requests ? errors / requests : 0, latency: percentiles(latency), cpu: percentiles(cpu) }; |
| 984 | }; |
| 985 | |
| 986 | const rows = read(from, to, false); |
| 987 | const previous = read(from - (to - from), from, false); |
| 988 | const stepMinutes = step / 60; |
| 989 | const points = new Map<number, { rows: Row[] }>(); |
| 990 | for (const r of rows) { |
| 991 | const bucket = Math.floor(Number(r.minute) / stepMinutes) * stepMinutes; |
| 992 | const p = points.get(bucket) ?? { rows: [] }; |
| 993 | p.rows.push(r); |
| 994 | points.set(bucket, p); |
| 995 | } |
| 996 | const series = []; |
| 997 | for (let m = Math.floor(from / MINUTE_US / stepMinutes) * stepMinutes; m * MINUTE_US <= to; m += stepMinutes) { |
| 998 | const p = points.get(m)?.rows ?? []; |
| 999 | const t = totalsOf(p); |
| 1000 | const s = (k: string) => p.reduce((n, r) => n + Number(r[k]), 0); |
| 1001 | series.push({ time_us: m * MINUTE_US, requests: t.requests, errors: t.errors, status_2xx: s("s2"), status_3xx: s("s3"), status_4xx: s("s4"), status_5xx: s("s5"), latency: t.latency, cpu: t.cpu }); |
| 1002 | } |
| 1003 | |
| 1004 | const routes = new Map<string, Row[]>(); |
| 1005 | for (const r of read(from, to, true)) { |
| 1006 | const key = `${r.method}\u0000${r.route}`; |
| 1007 | routes.set(key, [...(routes.get(key) ?? []), r]); |
| 1008 | } |
| 1009 | const deployments = new Map<string, Row[]>(); |
| 1010 | const services = new Map<string, Row[]>(); |
| 1011 | for (const r of rows) { |
| 1012 | if (r.deployment_id) deployments.set(String(r.deployment_id), [...(deployments.get(String(r.deployment_id)) ?? []), r]); |
| 1013 | const sk = `${r.service}\u0000${r.source}`; |
| 1014 | services.set(sk, [...(services.get(sk) ?? []), r]); |
| 1015 | } |
| 1016 | const { clauses: rc, args: ra } = this.filter({ ...f, from_us: from, to_us: to }, "time_us"); |
| 1017 | const regions = this.rows( |
| 1018 | `SELECT region, COUNT(*) AS n FROM requests WHERE region IS NOT NULL ${rc.length ? `AND ${rc.join(" AND ")}` : ""} GROUP BY region ORDER BY n DESC LIMIT 20`, |
| 1019 | ...ra, |
| 1020 | ); |
| 1021 | |
| 1022 | return { |
| 1023 | from_us: from, |
| 1024 | to_us: to, |
| 1025 | step_s: step, |
| 1026 | totals: totalsOf(rows), |
| 1027 | previous: totalsOf(previous), |
| 1028 | points: series, |
| 1029 | routes: [...routes.values()] |
| 1030 | .map((list) => { |
| 1031 | const t = totalsOf(list); |
| 1032 | return { route: String(list[0].route), method: strOrNull(list[0].method), requests: t.requests, errors: t.errors, latency: t.latency }; |
| 1033 | }) |
| 1034 | .sort((a, b) => b.requests - a.requests) |
| 1035 | .slice(0, 50), |
| 1036 | deployments: [...deployments.entries()] |
| 1037 | .map(([id, list]) => { |
| 1038 | const t = totalsOf(list); |
| 1039 | return { |
| 1040 | deployment_id: id, |
| 1041 | environment: strOrNull(list[0].environment), |
| 1042 | commit_sha: strOrNull(list.find((r) => r.commit_sha)?.commit_sha ?? null), |
| 1043 | first_us: Math.min(...list.map((r) => Number(r.minute))) * MINUTE_US, |
| 1044 | requests: t.requests, |
| 1045 | errors: t.errors, |
| 1046 | latency: t.latency, |
| 1047 | }; |
| 1048 | }) |
| 1049 | .sort((a, b) => b.first_us - a.first_us) |
| 1050 | .slice(0, 20), |
| 1051 | services: [...services.values()] |
| 1052 | .map((list) => ({ |
| 1053 | service: String(list[0].service), |
| 1054 | source: list[0].source === "otlp" ? ("otlp" as const) : ("cloudflare" as const), |
| 1055 | requests: list.reduce((n, r) => n + Number(r.requests), 0), |
| 1056 | errors: list.reduce((n, r) => n + Number(r.errors), 0), |
| 1057 | last_us: Math.max(...list.map((r) => Number(r.minute))) * MINUTE_US, |
| 1058 | })) |
| 1059 | .sort((a, b) => b.requests - a.requests), |
| 1060 | regions: regions.map((r) => ({ region: String(r.region), requests: Number(r.n) })), |
| 1061 | }; |
| 1062 | } |
| 1063 | } |
| 1064 | |
| 1065 | /** A request that failed: a 5xx, an outcome other than ok, or an error the app recorded. */ |
| 1066 | const ERROR_SQL = "(status >= 500 OR outcome NOT IN ('ok', 'canceled', 'responseStreamDisconnected') OR error_name IS NOT NULL)"; |
| 1067 | |
| 1068 | function isError(status: number | null, outcome: string, errorName: SqlStorageValue): boolean { |
| 1069 | return (status !== null && status >= 500) || !["ok", "canceled", "responseStreamDisconnected"].includes(outcome) || (errorName !== null && errorName !== undefined && errorName !== ""); |
| 1070 | } |
| 1071 | |
| 1072 | function errorOf(r: Row): ErrorGroup { |
| 1073 | return { |
| 1074 | fingerprint: String(r.fingerprint), |
| 1075 | name: String(r.name), |
| 1076 | message: String(r.message), |
| 1077 | stack: strOrNull(r.stack), |
| 1078 | service: String(r.service), |
| 1079 | environment: strOrNull(r.environment), |
| 1080 | first_us: Number(r.first_us), |
| 1081 | last_us: Number(r.last_us), |
| 1082 | count: Number(r.count), |
| 1083 | first_deployment_id: strOrNull(r.first_deployment_id), |
| 1084 | last_request_id: strOrNull(r.last_request_id), |
| 1085 | last_trace_id: strOrNull(r.last_trace_id), |
| 1086 | state: String(r.state) as ErrorGroup["state"], |
| 1087 | issue_number: numOrNull(r.issue_number), |
| 1088 | }; |
| 1089 | } |
| 1090 | |
| 1091 | /** The step that gives a chart about a hundred points over `rangeUs`. */ |
| 1092 | export function stepFor(rangeUs: number): number { |
| 1093 | const seconds = rangeUs / 1_000_000; |
| 1094 | return STEPS.find((s) => seconds / s <= 150) ?? STEPS[STEPS.length - 1]; |
| 1095 | } |
| 1096 | |
| 1097 | /** A percentile from an OTLP histogram's explicit buckets. */ |
| 1098 | export function histogramQuantile(bounds: number[], counts: number[], q: number): number | null { |
| 1099 | const total = counts.reduce((a, b) => a + b, 0); |
| 1100 | if (!total) return null; |
| 1101 | const rank = q * total; |
| 1102 | let seen = 0; |
| 1103 | for (let i = 0; i < counts.length; i++) { |
| 1104 | if (seen + counts[i] >= rank) { |
| 1105 | const low = i === 0 ? 0 : bounds[i - 1]; |
| 1106 | const high = i < bounds.length ? bounds[i] : bounds[bounds.length - 1] ?? low; |
| 1107 | return low + (high - low) * (counts[i] ? (rank - seen) / counts[i] : 0); |
| 1108 | } |
| 1109 | seen += counts[i]; |
| 1110 | } |
| 1111 | return bounds[bounds.length - 1] ?? null; |
| 1112 | } |
| 1113 | |
| 1114 | /** Requests' counts by minute, service, environment, deployment and route, with all routes together as `*`. */ |
| 1115 | export type Rollup = { |
| 1116 | minute: number; |
| 1117 | service: string; |
| 1118 | environment: string; |
| 1119 | deployment_id: string; |
| 1120 | commit_sha: string | null; |
| 1121 | source: string; |
| 1122 | method: string; |
| 1123 | route: string; |
| 1124 | requests: number; |
| 1125 | errors: number; |
| 1126 | s2: number; |
| 1127 | s3: number; |
| 1128 | s4: number; |
| 1129 | s5: number; |
| 1130 | latency: string; |
| 1131 | cpu: string; |
| 1132 | }; |
| 1133 | |
| 1134 | export function aggregate(rows: Row[]): Rollup[] { |
| 1135 | const acc = new Map<string, Omit<Rollup, "latency" | "cpu"> & { latency: number[]; cpu: number[] }>(); |
| 1136 | const add = (key: Pick<Rollup, "minute" | "service" | "environment" | "deployment_id" | "commit_sha" | "source" | "method" | "route">, r: Row) => { |
| 1137 | const k = [key.minute, key.service, key.environment, key.deployment_id, key.method, key.route].join("\u0000"); |
| 1138 | let a = acc.get(k); |
| 1139 | if (!a) acc.set(k, (a = { ...key, requests: 0, errors: 0, s2: 0, s3: 0, s4: 0, s5: 0, latency: emptyHistogram(), cpu: emptyHistogram() })); |
| 1140 | a.requests++; |
| 1141 | const status = numOrNull(r.status); |
| 1142 | if (isError(status, String(r.outcome), r.error_name)) a.errors++; |
| 1143 | if (status !== null && status >= 200 && status < 600) a[`s${Math.floor(status / 100)}` as "s2" | "s3" | "s4" | "s5"]++; |
| 1144 | a.latency[bucketOf(Number(r.duration_us) / 1000)]++; |
| 1145 | if (r.cpu_ms !== null && r.cpu_ms !== undefined) a.cpu[bucketOf(Number(r.cpu_ms))]++; |
| 1146 | }; |
| 1147 | for (const r of rows) { |
| 1148 | const base = { |
| 1149 | minute: Math.floor(Number(r.time_us) / MINUTE_US), |
| 1150 | service: String(r.service ?? ""), |
| 1151 | environment: String(r.environment ?? ""), |
| 1152 | deployment_id: String(r.deployment_id ?? ""), |
| 1153 | commit_sha: strOrNull(r.commit_sha), |
| 1154 | source: String(r.source), |
| 1155 | }; |
| 1156 | add({ ...base, method: "*", route: "*" }, r); |
| 1157 | add({ ...base, method: String(r.method ?? ""), route: String(r.route ?? "") }, r); |
| 1158 | } |
| 1159 | return [...acc.values()].map((a) => ({ ...a, latency: encode(a.latency), cpu: encode(a.cpu) })); |
| 1160 | } |