| 1 | #!/usr/bin/env node |
| 2 | // What are the runner's Durable Object errors? The sandboxes that run |
| 3 | // checks, workflow jobs and merge-queue builds are Durable Objects of the |
| 4 | // g1t-runner Worker (services/runner: AttemptSandbox, Sandbox2Core, |
| 5 | // Sandbox4Core). This asks Cloudflare, over the last N days: |
| 6 | // |
| 7 | // 1. GraphQL Analytics (durableObjectsInvocationsAdaptiveGroups): requests |
| 8 | // and errors by namespace and status, and by day and status. |
| 9 | // 2. Workers Observability (the telemetry query API): the runner's failed |
| 10 | // invocations by class, event type (alarm, rpc, fetch) and outcome; the |
| 11 | // exceptions they threw, by message; the runner's own error-level logs |
| 12 | // (`sandbox stop not reported`, `sandbox alarm failed`, `sandbox not |
| 13 | // started`, `sandbox container error`), by message; and a few recent |
| 14 | // failed invocations in full. |
| 15 | // |
| 16 | // Each message is put in a bucket (`classify`): a deploy resetting the |
| 17 | // object, no container free, a container that exited, and so on, so what is |
| 18 | // expected and what is a bug can be told apart at a glance. |
| 19 | // |
| 20 | // Read-only: GraphQL queries, one Durable Objects listing for names, and |
| 21 | // telemetry queries. Never prints the token. |
| 22 | // |
| 23 | // CLOUDFLARE_API_TOKEN=<token> node scripts/ops/runner-errors.mjs [--days 7] [--json] |
| 24 | // |
| 25 | // The token needs Account Analytics: Read (GraphQL) and Workers Observability: |
| 26 | // Read (logs); Workers Scripts: Read adds namespace names. A part the token |
| 27 | // cannot read is a note in the report, never the end of it. |
| 28 | |
| 29 | import { ACCOUNT_ID, cloudflareAuth } from "../deploy/cloudflare.mjs"; |
| 30 | |
| 31 | const API = "https://api.cloudflare.com/client/v4"; |
| 32 | |
| 33 | const HELP = `node scripts/ops/runner-errors.mjs [--days N] [--script NAME] [--samples N] [--json] [--keys] [--account <id>] |
| 34 | |
| 35 | The runner's Durable Object errors: requests and errors by namespace and |
| 36 | status (GraphQL Analytics), and what failed and why (Workers Observability). |
| 37 | |
| 38 | --days N how far back, in days (default 7, at most 31) |
| 39 | --script NAME the Worker (default g1t-runner) |
| 40 | --samples N recent failed invocations shown in full (default 10) |
| 41 | --json one JSON object, snake_case keys |
| 42 | --keys also list the telemetry keys the runner's events have |
| 43 | --account <id> the Cloudflare account (default CLOUDFLARE_ACCOUNT_ID, else g1t's) |
| 44 | --help this text |
| 45 | |
| 46 | Needs CLOUDFLARE_API_TOKEN (Account Analytics: Read, Workers Observability: |
| 47 | Read), or CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL. Exits 1 when nothing |
| 48 | could be read.`; |
| 49 | |
| 50 | /** The command line. */ |
| 51 | export function parseArgs(argv, env = process.env) { |
| 52 | const option = (name) => { |
| 53 | const at = argv.indexOf(name); |
| 54 | return at >= 0 && argv[at + 1] && !argv[at + 1].startsWith("--") ? argv[at + 1] : null; |
| 55 | }; |
| 56 | const days = Number(option("--days") ?? NaN); |
| 57 | const samples = Number(option("--samples") ?? NaN); |
| 58 | return { |
| 59 | help: argv.includes("--help") || argv.includes("-h"), |
| 60 | json: argv.includes("--json"), |
| 61 | keys: argv.includes("--keys"), |
| 62 | days: Number.isFinite(days) && days > 0 ? Math.min(31, days) : 7, |
| 63 | samples: Number.isFinite(samples) && samples >= 0 ? Math.min(100, Math.floor(samples)) : 10, |
| 64 | script: option("--script") || "g1t-runner", |
| 65 | account: option("--account") || env.CLOUDFLARE_ACCOUNT_ID || ACCOUNT_ID, |
| 66 | }; |
| 67 | } |
| 68 | |
| 69 | /** The window: the last `days` days up to `now`. */ |
| 70 | export function windowOf(now, days) { |
| 71 | return { start: new Date(now.getTime() - days * 86_400_000).toISOString(), end: now.toISOString() }; |
| 72 | } |
| 73 | |
| 74 | /** |
| 75 | * The GraphQL queries, each with variants tried in order when Cloudflare |
| 76 | * refuses a field. Rows come back under `rows`. |
| 77 | */ |
| 78 | export const DO_QUERIES = [ |
| 79 | { |
| 80 | key: "by_namespace", |
| 81 | label: "Durable Object requests by namespace and status", |
| 82 | variants: [ |
| 83 | { sum: ["requests", "errors", "wallTime"], dims: ["namespaceId", "status"] }, |
| 84 | { sum: ["requests", "errors"], dims: ["namespaceId", "status"] }, |
| 85 | { sum: ["requests"], dims: ["namespaceId", "status"] }, |
| 86 | { sum: ["requests", "errors"], dims: ["namespaceId"] }, |
| 87 | ], |
| 88 | }, |
| 89 | { |
| 90 | key: "by_day", |
| 91 | label: "Durable Object requests by day and status", |
| 92 | variants: [ |
| 93 | { sum: ["requests", "errors"], dims: ["date", "status"] }, |
| 94 | { sum: ["requests"], dims: ["date", "status"] }, |
| 95 | ], |
| 96 | }, |
| 97 | ]; |
| 98 | |
| 99 | /** One DO invocations query for one script, for one variant. */ |
| 100 | export function doQuery(variant) { |
| 101 | return `query RunnerErrors($accountTag: String!, $start: Time!, $end: Time!, $script: String!) { |
| 102 | viewer { |
| 103 | accounts(filter: { accountTag: $accountTag }) { |
| 104 | rows: durableObjectsInvocationsAdaptiveGroups(limit: 10000, filter: { datetime_geq: $start, datetime_leq: $end, scriptName: $script }) { |
| 105 | sum { ${variant.sum.join(" ")} } |
| 106 | dimensions { ${variant.dims.join(" ")} } |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | }`; |
| 111 | } |
| 112 | |
| 113 | const snake = (name) => name.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); |
| 114 | |
| 115 | /** |
| 116 | * A GraphQL answer as rows: one per dimension combination, the first |
| 117 | * dimension named through `names` (namespace ids to names), with each sum |
| 118 | * and the error rate, largest first by requests. Throws with Cloudflare's |
| 119 | * message when the answer has errors. |
| 120 | */ |
| 121 | export function parseDoGroups(body, variant, names = {}) { |
| 122 | if (body?.errors?.length) |
| 123 | throw new Error( |
| 124 | body.errors |
| 125 | .map((error) => error.message) |
| 126 | .join("; ") |
| 127 | .slice(0, 400), |
| 128 | ); |
| 129 | const groups = body?.data?.viewer?.accounts?.[0]?.rows; |
| 130 | if (!Array.isArray(groups)) throw new Error("no rows in the answer"); |
| 131 | const rows = groups.map((group) => { |
| 132 | const row = {}; |
| 133 | variant.dims.forEach((dim, at) => { |
| 134 | const value = group.dimensions?.[dim]; |
| 135 | const text = value == null || value === "" ? "(none)" : String(value); |
| 136 | row[snake(dim)] = at === 0 && names[text] ? names[text] : text; |
| 137 | }); |
| 138 | for (const field of variant.sum) row[snake(field)] = Number(group.sum?.[field] ?? 0); |
| 139 | return row; |
| 140 | }); |
| 141 | const sortKey = variant.dims[0] === "date" ? null : "requests"; |
| 142 | rows.sort((a, b) => (sortKey ? b.requests - a.requests : 0) || String(a[snake(variant.dims[0])]).localeCompare(String(b[snake(variant.dims[0])]))); |
| 143 | const totals = Object.fromEntries(variant.sum.map((field) => [snake(field), rows.reduce((sum, row) => sum + row[snake(field)], 0)])); |
| 144 | return { dims: variant.dims.map(snake), metrics: variant.sum.map(snake), rows, totals }; |
| 145 | } |
| 146 | |
| 147 | /** |
| 148 | * Requests and errors per status, summed over namespaces: which statuses |
| 149 | * the errors are (`scriptThrewException`, `internalError`, |
| 150 | * `clientDisconnected`, `exceededResources`, ...). |
| 151 | */ |
| 152 | export function byStatus(parsed) { |
| 153 | if (!parsed.dims.includes("status")) return []; |
| 154 | const out = new Map(); |
| 155 | for (const row of parsed.rows) { |
| 156 | const entry = out.get(row.status) ?? { status: row.status, requests: 0, errors: 0 }; |
| 157 | entry.requests += row.requests ?? 0; |
| 158 | entry.errors += row.errors ?? 0; |
| 159 | out.set(row.status, entry); |
| 160 | } |
| 161 | return [...out.values()].sort((a, b) => b.requests - a.requests); |
| 162 | } |
| 163 | |
| 164 | /** The telemetry keys the questions below group by and filter on. */ |
| 165 | export const KEYS = { |
| 166 | outcome: "$workers.outcome", |
| 167 | eventType: "$workers.eventType", |
| 168 | entrypoint: "$workers.entrypoint", |
| 169 | error: "$metadata.error", |
| 170 | message: "$metadata.message", |
| 171 | level: "$metadata.level", |
| 172 | }; |
| 173 | |
| 174 | /** Which key names the Worker: tried in order, the next when one finds nothing. */ |
| 175 | export const SERVICE_KEYS = ["$workers.scriptName", "$metadata.service"]; |
| 176 | |
| 177 | const filter = (key, operation, value) => (value === undefined ? { key, operation, type: "string" } : { key, operation, type: "string", value }); |
| 178 | |
| 179 | /** |
| 180 | * The telemetry questions: each a body for `POST .../workers/observability/ |
| 181 | * telemetry/query`, for the Worker named under `serviceKey`. |
| 182 | */ |
| 183 | export function telemetryQuestions({ script, from, to, samples, serviceKey = SERVICE_KEYS[0] }) { |
| 184 | const service = filter(serviceKey, "eq", script); |
| 185 | const failed = filter(KEYS.outcome, "neq", "ok"); |
| 186 | const base = { timeframe: { from, to }, limit: 100 }; |
| 187 | const count = [{ operator: "count", alias: "events" }]; |
| 188 | const group = (...keys) => keys.map((value) => ({ type: "string", value })); |
| 189 | return [ |
| 190 | { |
| 191 | key: "failed_invocations", |
| 192 | label: "Failed invocations by class, event and outcome", |
| 193 | body: { |
| 194 | ...base, |
| 195 | queryId: "g1t-runner-failed-invocations", |
| 196 | view: "calculations", |
| 197 | parameters: { |
| 198 | datasets: ["cloudflare-workers"], |
| 199 | filters: [service, failed], |
| 200 | calculations: count, |
| 201 | groupBys: group(KEYS.entrypoint, KEYS.eventType, KEYS.outcome), |
| 202 | }, |
| 203 | }, |
| 204 | }, |
| 205 | { |
| 206 | key: "exceptions", |
| 207 | label: "Exceptions by message", |
| 208 | body: { |
| 209 | ...base, |
| 210 | queryId: "g1t-runner-exceptions", |
| 211 | view: "calculations", |
| 212 | parameters: { datasets: ["cloudflare-workers"], filters: [service, filter(KEYS.error, "exists")], calculations: count, groupBys: group(KEYS.error) }, |
| 213 | }, |
| 214 | }, |
| 215 | { |
| 216 | key: "error_logs", |
| 217 | label: "The runner's error-level logs by message", |
| 218 | body: { |
| 219 | ...base, |
| 220 | queryId: "g1t-runner-error-logs", |
| 221 | view: "calculations", |
| 222 | parameters: { |
| 223 | datasets: ["cloudflare-workers"], |
| 224 | filters: [service, filter(KEYS.level, "eq", "error")], |
| 225 | calculations: count, |
| 226 | groupBys: group(KEYS.message), |
| 227 | }, |
| 228 | }, |
| 229 | }, |
| 230 | { |
| 231 | key: "samples", |
| 232 | label: "Recent failed invocations", |
| 233 | body: { |
| 234 | ...base, |
| 235 | limit: Math.max(1, samples), |
| 236 | queryId: "g1t-runner-failed-samples", |
| 237 | view: "events", |
| 238 | parameters: { datasets: ["cloudflare-workers"], filters: [service, failed] }, |
| 239 | }, |
| 240 | }, |
| 241 | ]; |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * A telemetry calculations answer as rows: each group's key values and its |
| 246 | * count, largest first. Throws with Cloudflare's message when it failed. |
| 247 | */ |
| 248 | export function parseCalculations(body) { |
| 249 | if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body)); |
| 250 | const calculation = body?.result?.calculations?.[0]; |
| 251 | if (!calculation) throw new Error("no calculations in the answer"); |
| 252 | const rows = (calculation.aggregates ?? []).map((aggregate) => { |
| 253 | const row = {}; |
| 254 | for (const group of aggregate.groups ?? []) row[group.key] = group.value == null || group.value === "" ? "(none)" : String(group.value); |
| 255 | row.count = Number(aggregate.value ?? aggregate.count ?? 0); |
| 256 | return row; |
| 257 | }); |
| 258 | return rows.sort((a, b) => b.count - a.count); |
| 259 | } |
| 260 | |
| 261 | /** A telemetry events answer as plain records: when, which class, event, outcome, and what went wrong. */ |
| 262 | export function parseEvents(body) { |
| 263 | if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body)); |
| 264 | const events = body?.result?.events?.events ?? body?.result?.events ?? []; |
| 265 | if (!Array.isArray(events)) throw new Error("no events in the answer"); |
| 266 | return events.map((event) => { |
| 267 | const workers = event.$workers ?? {}; |
| 268 | const metadata = event.$metadata ?? {}; |
| 269 | const at = event.timestamp ?? metadata.startTime ?? workers.timestamp; |
| 270 | return { |
| 271 | at: typeof at === "number" ? new Date(at).toISOString() : (at ?? null), |
| 272 | entrypoint: workers.entrypoint ?? null, |
| 273 | event_type: workers.eventType ?? null, |
| 274 | outcome: workers.outcome ?? null, |
| 275 | object: typeof workers.durableObjectId === "string" ? workers.durableObjectId.slice(0, 12) : null, |
| 276 | error: metadata.error ?? null, |
| 277 | message: metadata.message ?? (typeof event.source === "string" ? event.source : (event.source?.message ?? null)), |
| 278 | }; |
| 279 | }); |
| 280 | } |
| 281 | |
| 282 | function messagesOf(body) { |
| 283 | const errors = body?.errors ?? []; |
| 284 | const text = errors.map((error) => error.message ?? JSON.stringify(error)).join("; "); |
| 285 | return (text || `request failed${body?.status ? ` with ${body.status}` : ""}`).slice(0, 400); |
| 286 | } |
| 287 | |
| 288 | /** |
| 289 | * What a failure most likely is, from its message or outcome: a bucket and |
| 290 | * whether it is expected. Order matters: the first match wins. |
| 291 | */ |
| 292 | export const BUCKETS = [ |
| 293 | { |
| 294 | bucket: "deploy_reset", |
| 295 | expected: true, |
| 296 | why: "a runner deploy reset the object mid-invocation", |
| 297 | test: /code (was|has been) updated|reset because its code|new version of the (script|worker)|durable object reset/i, |
| 298 | }, |
| 299 | { |
| 300 | bucket: "stop_not_reported", |
| 301 | expected: false, |
| 302 | why: "onStop could not tell a service the sandbox stopped (now logged, not thrown)", |
| 303 | test: /sandbox stop not reported/i, |
| 304 | }, |
| 305 | { bucket: "alarm_failed", expected: false, why: "the sandbox's alarm threw (retried by Cloudflare)", test: /sandbox alarm failed/i }, |
| 306 | { |
| 307 | bucket: "no_capacity", |
| 308 | expected: true, |
| 309 | why: "no container instance free (max_instances, or provisioning)", |
| 310 | test: /no container instance|max(imum)? concurrent instance|too many containers per second/i, |
| 311 | }, |
| 312 | { |
| 313 | bucket: "not_started", |
| 314 | expected: false, |
| 315 | why: "a sandbox could not start (guardrails unreadable, or the container would not start)", |
| 316 | test: /sandbox not started|could not read this project's guardrails|did not start after|failed to start container/i, |
| 317 | }, |
| 318 | { |
| 319 | bucket: "container_exited", |
| 320 | expected: true, |
| 321 | why: "the container exited or was stopped (a finished run, a time cap, a stop)", |
| 322 | test: /container exited|runtime signalled|exited before we could determine|crashed while checking for ports|exit code/i, |
| 323 | }, |
| 324 | { bucket: "connection_lost", expected: false, why: "the connection to the container was lost", test: /network connection lost|disconnected/i }, |
| 325 | { bucket: "storage", expected: false, why: "Durable Object storage failed or was overloaded", test: /storage|sqlite|overloaded/i }, |
| 326 | { bucket: "limits", expected: false, why: "a CPU, memory or subrequest limit", test: /exceeded|too many subrequests|memory limit/i }, |
| 327 | { bucket: "container_error", expected: false, why: "the containers library reported an error", test: /sandbox container error|container error/i }, |
| 328 | ]; |
| 329 | |
| 330 | /** The bucket of one failure's message (or, failing that, its outcome). */ |
| 331 | export function classify(message, outcome = null) { |
| 332 | const text = String(message ?? ""); |
| 333 | for (const bucket of BUCKETS) if (text && bucket.test.test(text)) return { bucket: bucket.bucket, expected: bucket.expected, why: bucket.why }; |
| 334 | if (/canceled|cancelled|clientdisconnected|responsestreamdisconnected/i.test(String(outcome ?? ""))) { |
| 335 | return { bucket: "caller_gone", expected: true, why: "the caller went away before the object answered" }; |
| 336 | } |
| 337 | return { bucket: "other", expected: false, why: "not recognised: read the message" }; |
| 338 | } |
| 339 | |
| 340 | /** Rows of messages with counts, summed into buckets, largest first. */ |
| 341 | export function bucketsOf(rows, messageKey) { |
| 342 | const out = new Map(); |
| 343 | for (const row of rows) { |
| 344 | const found = classify(row[messageKey], row[KEYS.outcome]); |
| 345 | const entry = out.get(found.bucket) ?? { ...found, count: 0 }; |
| 346 | entry.count += row.count; |
| 347 | out.set(found.bucket, entry); |
| 348 | } |
| 349 | return [...out.values()].sort((a, b) => b.count - a.count); |
| 350 | } |
| 351 | |
| 352 | const number = (value) => (Number.isInteger(value) ? value.toLocaleString("en-US") : value.toLocaleString("en-US", { maximumFractionDigits: 2 })); |
| 353 | const percent = (part, whole) => (whole > 0 ? `${((100 * part) / whole).toFixed(1)}%` : "-"); |
| 354 | |
| 355 | function table(rows, columns) { |
| 356 | if (!rows.length) return [" (nothing in this window)"]; |
| 357 | const cells = [columns.map((c) => c.title), ...rows.map((row) => columns.map((c) => c.value(row)))]; |
| 358 | const widths = columns.map((_, at) => Math.max(...cells.map((row) => String(row[at]).length))); |
| 359 | return cells.map((row) => " " + row.map((cell, at) => (columns[at].right ? String(cell).padStart(widths[at]) : String(cell).padEnd(widths[at]))).join(" ")); |
| 360 | } |
| 361 | |
| 362 | /** The report as plain text. */ |
| 363 | export function format(report, top = 25) { |
| 364 | const lines = [`Durable Object errors of ${report.script} on account ${report.account}, ${report.start} to ${report.end}`]; |
| 365 | for (const query of report.analytics) { |
| 366 | lines.push("", query.label); |
| 367 | if (query.error) { |
| 368 | lines.push(` could not read: ${query.error}`); |
| 369 | continue; |
| 370 | } |
| 371 | const columns = [ |
| 372 | ...query.dims.map((dim) => ({ title: dim, value: (row) => row[dim] })), |
| 373 | ...query.metrics.map((metric) => ({ title: metric, right: true, value: (row) => number(row[metric]) })), |
| 374 | ]; |
| 375 | if (query.metrics.includes("errors")) columns.push({ title: "error_rate", right: true, value: (row) => percent(row.errors, row.requests) }); |
| 376 | lines.push(...table(query.rows.slice(0, top), columns)); |
| 377 | lines.push( |
| 378 | ` total: ${Object.entries(query.totals) |
| 379 | .map(([metric, value]) => `${metric} ${number(value)}`) |
| 380 | .join(", ")}`, |
| 381 | ); |
| 382 | if (query.by_status?.length) { |
| 383 | lines.push(" by status:"); |
| 384 | lines.push( |
| 385 | ...table(query.by_status, [ |
| 386 | { title: "status", value: (row) => row.status }, |
| 387 | { title: "requests", right: true, value: (row) => number(row.requests) }, |
| 388 | { title: "errors", right: true, value: (row) => number(row.errors) }, |
| 389 | ]).map((line) => ` ${line}`), |
| 390 | ); |
| 391 | } |
| 392 | } |
| 393 | for (const question of report.telemetry) { |
| 394 | lines.push("", question.label); |
| 395 | if (question.error) { |
| 396 | lines.push(` could not read: ${question.error}`); |
| 397 | continue; |
| 398 | } |
| 399 | if (question.key === "samples") { |
| 400 | if (!question.rows.length) lines.push(" (none)"); |
| 401 | for (const row of question.rows) { |
| 402 | lines.push(` ${row.at ?? "?"} ${row.entrypoint ?? "?"} ${row.event_type ?? "?"} ${row.outcome ?? "?"}${row.object ? ` ${row.object}` : ""}`); |
| 403 | if (row.error || row.message) lines.push(` ${String(row.error ?? row.message).slice(0, 300)}`); |
| 404 | } |
| 405 | continue; |
| 406 | } |
| 407 | const keys = Object.keys(question.rows[0] ?? {}).filter((key) => key !== "count"); |
| 408 | lines.push( |
| 409 | ...table(question.rows.slice(0, top), [ |
| 410 | ...keys.map((key) => ({ title: key, value: (row) => String(row[key] ?? "").slice(0, 120) })), |
| 411 | { title: "count", right: true, value: (row) => number(row.count) }, |
| 412 | ]), |
| 413 | ); |
| 414 | if (question.buckets?.length) { |
| 415 | lines.push(" most likely:"); |
| 416 | for (const bucket of question.buckets) |
| 417 | lines.push(` ${String(number(bucket.count)).padStart(6)} ${bucket.bucket}${bucket.expected ? " (expected)" : ""}: ${bucket.why}`); |
| 418 | } |
| 419 | } |
| 420 | if (report.keys) lines.push("", "Telemetry keys:", ...report.keys.map((key) => ` ${key}`)); |
| 421 | if (report.notes.length) lines.push("", "Notes:", ...report.notes.map((note) => ` ${note}`)); |
| 422 | return lines.join("\n"); |
| 423 | } |
| 424 | |
| 425 | async function post(auth, path, body) { |
| 426 | const response = await fetch(`${API}${path}`, { |
| 427 | method: "POST", |
| 428 | headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" }, |
| 429 | body: JSON.stringify(body), |
| 430 | }); |
| 431 | const answer = await response.json().catch(() => ({ success: false, errors: [{ message: `HTTP ${response.status}, not JSON` }] })); |
| 432 | if (!response.ok && !answer.errors?.length) answer.errors = [{ message: `HTTP ${response.status}` }]; |
| 433 | return answer; |
| 434 | } |
| 435 | |
| 436 | async function durableNames(auth, account) { |
| 437 | const out = {}; |
| 438 | try { |
| 439 | for (let page = 1; page <= 10; page++) { |
| 440 | const response = await fetch(`${API}/accounts/${account}/workers/durable_objects/namespaces?per_page=100&page=${page}`, { |
| 441 | headers: { ...auth, "user-agent": "g1t-ops" }, |
| 442 | }); |
| 443 | const body = await response.json(); |
| 444 | if (!response.ok || !Array.isArray(body.result)) break; |
| 445 | for (const item of body.result) if (item.id) out[item.id] = item.name ?? item.id; |
| 446 | if (body.result.length < 100) break; |
| 447 | } |
| 448 | } catch { |
| 449 | // Ids stand in for names. |
| 450 | } |
| 451 | return out; |
| 452 | } |
| 453 | |
| 454 | async function analytics(auth, options, window, names) { |
| 455 | return Promise.all( |
| 456 | DO_QUERIES.map(async (query) => { |
| 457 | let error = null; |
| 458 | for (const variant of query.variants) { |
| 459 | try { |
| 460 | const body = await post(auth, "/graphql", { |
| 461 | query: doQuery(variant), |
| 462 | variables: { accountTag: options.account, start: window.start, end: window.end, script: options.script }, |
| 463 | }); |
| 464 | const parsed = parseDoGroups(body, variant, names); |
| 465 | return { key: query.key, label: query.label, ...parsed, by_status: query.key === "by_namespace" ? byStatus(parsed) : undefined }; |
| 466 | } catch (thrown) { |
| 467 | error ??= String(thrown.message ?? thrown); |
| 468 | } |
| 469 | } |
| 470 | return { key: query.key, label: query.label, error }; |
| 471 | }), |
| 472 | ); |
| 473 | } |
| 474 | |
| 475 | async function telemetry(auth, options, window) { |
| 476 | const path = `/accounts/${options.account}/workers/observability/telemetry/query`; |
| 477 | const from = Date.parse(window.start); |
| 478 | const to = Date.parse(window.end); |
| 479 | let results = null; |
| 480 | for (const serviceKey of SERVICE_KEYS) { |
| 481 | results = await Promise.all( |
| 482 | telemetryQuestions({ script: options.script, from, to, samples: options.samples, serviceKey }).map(async (question) => { |
| 483 | try { |
| 484 | const body = await post(auth, path, question.body); |
| 485 | if (question.key === "samples") return { key: question.key, label: question.label, rows: parseEvents(body) }; |
| 486 | const rows = parseCalculations(body); |
| 487 | const messageKey = question.key === "exceptions" ? KEYS.error : question.key === "error_logs" ? KEYS.message : null; |
| 488 | return { key: question.key, label: question.label, rows, buckets: messageKey ? bucketsOf(rows, messageKey) : undefined }; |
| 489 | } catch (error) { |
| 490 | return { key: question.key, label: question.label, error: String(error.message ?? error) }; |
| 491 | } |
| 492 | }), |
| 493 | ); |
| 494 | // Another key names the Worker when this one found nothing at all. |
| 495 | if (results.some((result) => result.rows?.length)) return { serviceKey, results }; |
| 496 | } |
| 497 | return { serviceKey: SERVICE_KEYS.at(-1), results }; |
| 498 | } |
| 499 | |
| 500 | async function telemetryKeys(auth, options, window) { |
| 501 | const body = await post(auth, `/accounts/${options.account}/workers/observability/telemetry/keys`, { |
| 502 | timeframe: { from: Date.parse(window.start), to: Date.parse(window.end) }, |
| 503 | datasets: ["cloudflare-workers"], |
| 504 | filters: [filter(SERVICE_KEYS[0], "eq", options.script)], |
| 505 | limit: 500, |
| 506 | }); |
| 507 | if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body)); |
| 508 | return (body.result ?? []).map((key) => (typeof key === "string" ? key : `${key.key} (${key.type})`)).sort(); |
| 509 | } |
| 510 | |
| 511 | async function main() { |
| 512 | const options = parseArgs(process.argv.slice(2)); |
| 513 | if (options.help) { |
| 514 | console.log(HELP); |
| 515 | return 0; |
| 516 | } |
| 517 | const auth = cloudflareAuth(); |
| 518 | if (!auth) { |
| 519 | console.error(`Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read and Workers Observability: Read on account ${options.account}.`); |
| 520 | return 2; |
| 521 | } |
| 522 | const window = windowOf(new Date(), options.days); |
| 523 | const names = await durableNames(auth, options.account); |
| 524 | const [graph, logs, keys] = await Promise.all([ |
| 525 | analytics(auth, options, window, names), |
| 526 | telemetry(auth, options, window), |
| 527 | options.keys ? telemetryKeys(auth, options, window).catch((error) => [`could not list keys: ${error.message}`]) : Promise.resolve(null), |
| 528 | ]); |
| 529 | const notes = []; |
| 530 | if (!Object.keys(names).length) notes.push("Namespace ids are not named: the token cannot read the Durable Objects listing (Workers Scripts: Read)."); |
| 531 | notes.push(`Telemetry is filtered on ${logs.serviceKey}.`); |
| 532 | const report = { |
| 533 | account: options.account, |
| 534 | script: options.script, |
| 535 | start: window.start, |
| 536 | end: window.end, |
| 537 | analytics: graph, |
| 538 | telemetry: logs.results, |
| 539 | keys, |
| 540 | notes, |
| 541 | }; |
| 542 | console.log(options.json ? JSON.stringify(report, null, 2) : format(report)); |
| 543 | const read = [...graph, ...logs.results].some((part) => !part.error); |
| 544 | return read ? 0 : 1; |
| 545 | } |
| 546 | |
| 547 | if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/runner-errors.mjs")) { |
| 548 | main().then( |
| 549 | (code) => process.exit(code), |
| 550 | (error) => { |
| 551 | console.error(`runner-errors: ${error.message}`); |
| 552 | process.exit(1); |
| 553 | }, |
| 554 | ); |
| 555 | } |