Skip to content
Sign in

flagon-io/g1t

public
0
491 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge remote-tracking branch 'origin/main' into workspace-chat1#!/usr/bin/env node
2// What are the runner's Durable Object errors? The sandboxes that run agent
3// attempts, workflow jobs and merge-queue builds are Durable Objects of the
4// g1t-runner Worker (services/runner: AttemptSandbox, Sandbox2Core,
5// Sandbox4Core). This asks Cloudflare, over the last N days:
6//
7// 1. GraphQL Analytics (durableObjectsInvocationsAdaptiveGroups): requests
8// and errors by namespace and status, and by day and status.
9// 2. Workers Observability (the telemetry query API): the runner's failed
10// invocations by class, event type (alarm, rpc, fetch) and outcome; the
11// exceptions they threw, by message; the runner's own error-level logs
12// (`sandbox stop not reported`, `sandbox alarm failed`, `sandbox not
13// started`, `sandbox container error`), by message; and a few recent
14// failed invocations in full.
15//
16// Each message is put in a bucket (`classify`): a deploy resetting the
17// object, no container free, a container that exited, and so on, so what is
18// expected and what is a bug can be told apart at a glance.
19//
20// Read-only: GraphQL queries, one Durable Objects listing for names, and
21// telemetry queries. Never prints the token.
22//
23// CLOUDFLARE_API_TOKEN=<token> node scripts/ops/runner-errors.mjs [--days 7] [--json]
24//
25// The token needs Account Analytics: Read (GraphQL) and Workers Observability:
26// Read (logs); Workers Scripts: Read adds namespace names. A part the token
27// cannot read is a note in the report, never the end of it.
28
29import { ACCOUNT_ID, cloudflareAuth } from "../deploy/cloudflare.mjs";
30
31const API = "https://api.cloudflare.com/client/v4";
32
33const HELP = `node scripts/ops/runner-errors.mjs [--days N] [--script NAME] [--samples N] [--json] [--keys] [--account <id>]
34
35The runner's Durable Object errors: requests and errors by namespace and
36status (GraphQL Analytics), and what failed and why (Workers Observability).
37
38 --days N how far back, in days (default 7, at most 31)
39 --script NAME the Worker (default g1t-runner)
40 --samples N recent failed invocations shown in full (default 10)
41 --json one JSON object, snake_case keys
42 --keys also list the telemetry keys the runner's events have
43 --account <id> the Cloudflare account (default CLOUDFLARE_ACCOUNT_ID, else g1t's)
44 --help this text
45
46Needs CLOUDFLARE_API_TOKEN (Account Analytics: Read, Workers Observability:
47Read), or CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL. Exits 1 when nothing
48could be read.`;
49
50/** The command line. */
51export function parseArgs(argv, env = process.env) {
52 const option = (name) => {
53 const at = argv.indexOf(name);
54 return at >= 0 && argv[at + 1] && !argv[at + 1].startsWith("--") ? argv[at + 1] : null;
55 };
56 const days = Number(option("--days") ?? NaN);
57 const samples = Number(option("--samples") ?? NaN);
58 return {
59 help: argv.includes("--help") || argv.includes("-h"),
60 json: argv.includes("--json"),
61 keys: argv.includes("--keys"),
62 days: Number.isFinite(days) && days > 0 ? Math.min(31, days) : 7,
63 samples: Number.isFinite(samples) && samples >= 0 ? Math.min(100, Math.floor(samples)) : 10,
64 script: option("--script") || "g1t-runner",
65 account: option("--account") || env.CLOUDFLARE_ACCOUNT_ID || ACCOUNT_ID,
66 };
67}
68
69/** The window: the last `days` days up to `now`. */
70export function windowOf(now, days) {
71 return { start: new Date(now.getTime() - days * 86_400_000).toISOString(), end: now.toISOString() };
72}
73
74/**
75 * The GraphQL queries, each with variants tried in order when Cloudflare
76 * refuses a field. Rows come back under `rows`.
77 */
78export const DO_QUERIES = [
79 {
80 key: "by_namespace",
81 label: "Durable Object requests by namespace and status",
82 variants: [
83 { sum: ["requests", "errors", "wallTime"], dims: ["namespaceId", "status"] },
84 { sum: ["requests", "errors"], dims: ["namespaceId", "status"] },
85 { sum: ["requests"], dims: ["namespaceId", "status"] },
86 { sum: ["requests", "errors"], dims: ["namespaceId"] },
87 ],
88 },
89 {
90 key: "by_day",
91 label: "Durable Object requests by day and status",
92 variants: [
93 { sum: ["requests", "errors"], dims: ["date", "status"] },
94 { sum: ["requests"], dims: ["date", "status"] },
95 ],
96 },
97];
98
99/** One DO invocations query for one script, for one variant. */
100export function doQuery(variant) {
101 return `query RunnerErrors($accountTag: String!, $start: Time!, $end: Time!, $script: String!) {
102 viewer {
103 accounts(filter: { accountTag: $accountTag }) {
104 rows: durableObjectsInvocationsAdaptiveGroups(limit: 10000, filter: { datetime_geq: $start, datetime_leq: $end, scriptName: $script }) {
105 sum { ${variant.sum.join(" ")} }
106 dimensions { ${variant.dims.join(" ")} }
107 }
108 }
109 }
110}`;
111}
112
113const snake = (name) => name.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
114
115/**
116 * A GraphQL answer as rows: one per dimension combination, the first
117 * dimension named through `names` (namespace ids to names), with each sum
118 * and the error rate, largest first by requests. Throws with Cloudflare's
119 * message when the answer has errors.
120 */
121export function parseDoGroups(body, variant, names = {}) {
122 if (body?.errors?.length) throw new Error(body.errors.map((error) => error.message).join("; ").slice(0, 400));
123 const groups = body?.data?.viewer?.accounts?.[0]?.rows;
124 if (!Array.isArray(groups)) throw new Error("no rows in the answer");
125 const rows = groups.map((group) => {
126 const row = {};
127 variant.dims.forEach((dim, at) => {
128 const value = group.dimensions?.[dim];
129 const text = value == null || value === "" ? "(none)" : String(value);
130 row[snake(dim)] = at === 0 && names[text] ? names[text] : text;
131 });
132 for (const field of variant.sum) row[snake(field)] = Number(group.sum?.[field] ?? 0);
133 return row;
134 });
135 const sortKey = variant.dims[0] === "date" ? null : "requests";
136 rows.sort((a, b) => (sortKey ? b.requests - a.requests : 0) || String(a[snake(variant.dims[0])]).localeCompare(String(b[snake(variant.dims[0])])));
137 const totals = Object.fromEntries(variant.sum.map((field) => [snake(field), rows.reduce((sum, row) => sum + row[snake(field)], 0)]));
138 return { dims: variant.dims.map(snake), metrics: variant.sum.map(snake), rows, totals };
139}
140
141/**
142 * Requests and errors per status, summed over namespaces: which statuses
143 * the errors are (`scriptThrewException`, `internalError`,
144 * `clientDisconnected`, `exceededResources`, ...).
145 */
146export function byStatus(parsed) {
147 if (!parsed.dims.includes("status")) return [];
148 const out = new Map();
149 for (const row of parsed.rows) {
150 const entry = out.get(row.status) ?? { status: row.status, requests: 0, errors: 0 };
151 entry.requests += row.requests ?? 0;
152 entry.errors += row.errors ?? 0;
153 out.set(row.status, entry);
154 }
155 return [...out.values()].sort((a, b) => b.requests - a.requests);
156}
157
158/** The telemetry keys the questions below group by and filter on. */
159export const KEYS = {
160 outcome: "$workers.outcome",
161 eventType: "$workers.eventType",
162 entrypoint: "$workers.entrypoint",
163 error: "$metadata.error",
164 message: "$metadata.message",
165 level: "$metadata.level",
166};
167
168/** Which key names the Worker: tried in order, the next when one finds nothing. */
169export const SERVICE_KEYS = ["$workers.scriptName", "$metadata.service"];
170
171const filter = (key, operation, value) => (value === undefined ? { key, operation, type: "string" } : { key, operation, type: "string", value });
172
173/**
174 * The telemetry questions: each a body for `POST .../workers/observability/
175 * telemetry/query`, for the Worker named under `serviceKey`.
176 */
177export function telemetryQuestions({ script, from, to, samples, serviceKey = SERVICE_KEYS[0] }) {
178 const service = filter(serviceKey, "eq", script);
179 const failed = filter(KEYS.outcome, "neq", "ok");
180 const base = { timeframe: { from, to }, limit: 100 };
181 const count = [{ operator: "count", alias: "events" }];
182 const group = (...keys) => keys.map((value) => ({ type: "string", value }));
183 return [
184 {
185 key: "failed_invocations",
186 label: "Failed invocations by class, event and outcome",
187 body: {
188 ...base,
189 queryId: "g1t-runner-failed-invocations",
190 view: "calculations",
191 parameters: { datasets: ["cloudflare-workers"], filters: [service, failed], calculations: count, groupBys: group(KEYS.entrypoint, KEYS.eventType, KEYS.outcome) },
192 },
193 },
194 {
195 key: "exceptions",
196 label: "Exceptions by message",
197 body: {
198 ...base,
199 queryId: "g1t-runner-exceptions",
200 view: "calculations",
201 parameters: { datasets: ["cloudflare-workers"], filters: [service, filter(KEYS.error, "exists")], calculations: count, groupBys: group(KEYS.error) },
202 },
203 },
204 {
205 key: "error_logs",
206 label: "The runner's error-level logs by message",
207 body: {
208 ...base,
209 queryId: "g1t-runner-error-logs",
210 view: "calculations",
211 parameters: { datasets: ["cloudflare-workers"], filters: [service, filter(KEYS.level, "eq", "error")], calculations: count, groupBys: group(KEYS.message) },
212 },
213 },
214 {
215 key: "samples",
216 label: "Recent failed invocations",
217 body: {
218 ...base,
219 limit: Math.max(1, samples),
220 queryId: "g1t-runner-failed-samples",
221 view: "events",
222 parameters: { datasets: ["cloudflare-workers"], filters: [service, failed] },
223 },
224 },
225 ];
226}
227
228/**
229 * A telemetry calculations answer as rows: each group's key values and its
230 * count, largest first. Throws with Cloudflare's message when it failed.
231 */
232export function parseCalculations(body) {
233 if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body));
234 const calculation = body?.result?.calculations?.[0];
235 if (!calculation) throw new Error("no calculations in the answer");
236 const rows = (calculation.aggregates ?? []).map((aggregate) => {
237 const row = {};
238 for (const group of aggregate.groups ?? []) row[group.key] = group.value == null || group.value === "" ? "(none)" : String(group.value);
239 row.count = Number(aggregate.value ?? aggregate.count ?? 0);
240 return row;
241 });
242 return rows.sort((a, b) => b.count - a.count);
243}
244
245/** A telemetry events answer as plain records: when, which class, event, outcome, and what went wrong. */
246export function parseEvents(body) {
247 if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body));
248 const events = body?.result?.events?.events ?? body?.result?.events ?? [];
249 if (!Array.isArray(events)) throw new Error("no events in the answer");
250 return events.map((event) => {
251 const workers = event.$workers ?? {};
252 const metadata = event.$metadata ?? {};
253 const at = event.timestamp ?? metadata.startTime ?? workers.timestamp;
254 return {
255 at: typeof at === "number" ? new Date(at).toISOString() : (at ?? null),
256 entrypoint: workers.entrypoint ?? null,
257 event_type: workers.eventType ?? null,
258 outcome: workers.outcome ?? null,
259 object: typeof workers.durableObjectId === "string" ? workers.durableObjectId.slice(0, 12) : null,
260 error: metadata.error ?? null,
261 message: metadata.message ?? (typeof event.source === "string" ? event.source : (event.source?.message ?? null)),
262 };
263 });
264}
265
266function messagesOf(body) {
267 const errors = body?.errors ?? [];
268 const text = errors.map((error) => error.message ?? JSON.stringify(error)).join("; ");
269 return (text || `request failed${body?.status ? ` with ${body.status}` : ""}`).slice(0, 400);
270}
271
272/**
273 * What a failure most likely is, from its message or outcome: a bucket and
274 * whether it is expected. Order matters: the first match wins.
275 */
276export const BUCKETS = [
277 { bucket: "deploy_reset", expected: true, why: "a runner deploy reset the object mid-invocation", test: /code (was|has been) updated|reset because its code|new version of the (script|worker)|durable object reset/i },
278 { bucket: "stop_not_reported", expected: false, why: "onStop could not tell a service the sandbox stopped (now logged, not thrown)", test: /sandbox stop not reported/i },
279 { bucket: "alarm_failed", expected: false, why: "the sandbox's alarm threw (retried by Cloudflare)", test: /sandbox alarm failed/i },
280 { bucket: "no_capacity", expected: true, why: "no container instance free (max_instances, or provisioning)", test: /no container instance|max(imum)? concurrent instance|too many containers per second/i },
281 { bucket: "not_started", expected: false, why: "a sandbox could not start (guardrails unreadable, or the container would not start)", test: /sandbox not started|could not read this project's guardrails|did not start after|failed to start container/i },
282 { bucket: "container_exited", expected: true, why: "the container exited or was stopped (a finished run, a time cap, a stop)", test: /container exited|runtime signalled|exited before we could determine|crashed while checking for ports|exit code/i },
283 { bucket: "connection_lost", expected: false, why: "the connection to the container was lost", test: /network connection lost|disconnected/i },
284 { bucket: "storage", expected: false, why: "Durable Object storage failed or was overloaded", test: /storage|sqlite|overloaded/i },
285 { bucket: "limits", expected: false, why: "a CPU, memory or subrequest limit", test: /exceeded|too many subrequests|memory limit/i },
286 { bucket: "container_error", expected: false, why: "the containers library reported an error", test: /sandbox container error|container error/i },
287];
288
289/** The bucket of one failure's message (or, failing that, its outcome). */
290export function classify(message, outcome = null) {
291 const text = String(message ?? "");
292 for (const bucket of BUCKETS) if (text && bucket.test.test(text)) return { bucket: bucket.bucket, expected: bucket.expected, why: bucket.why };
293 if (/canceled|cancelled|clientdisconnected|responsestreamdisconnected/i.test(String(outcome ?? ""))) {
294 return { bucket: "caller_gone", expected: true, why: "the caller went away before the object answered" };
295 }
296 return { bucket: "other", expected: false, why: "not recognised: read the message" };
297}
298
299/** Rows of messages with counts, summed into buckets, largest first. */
300export function bucketsOf(rows, messageKey) {
301 const out = new Map();
302 for (const row of rows) {
303 const found = classify(row[messageKey], row[KEYS.outcome]);
304 const entry = out.get(found.bucket) ?? { ...found, count: 0 };
305 entry.count += row.count;
306 out.set(found.bucket, entry);
307 }
308 return [...out.values()].sort((a, b) => b.count - a.count);
309}
310
311const number = (value) => (Number.isInteger(value) ? value.toLocaleString("en-US") : value.toLocaleString("en-US", { maximumFractionDigits: 2 }));
312const percent = (part, whole) => (whole > 0 ? `${((100 * part) / whole).toFixed(1)}%` : "-");
313
314function table(rows, columns) {
315 if (!rows.length) return [" (nothing in this window)"];
316 const cells = [columns.map((c) => c.title), ...rows.map((row) => columns.map((c) => c.value(row)))];
317 const widths = columns.map((_, at) => Math.max(...cells.map((row) => String(row[at]).length)));
318 return cells.map((row) => " " + row.map((cell, at) => (columns[at].right ? String(cell).padStart(widths[at]) : String(cell).padEnd(widths[at]))).join(" "));
319}
320
321/** The report as plain text. */
322export function format(report, top = 25) {
323 const lines = [`Durable Object errors of ${report.script} on account ${report.account}, ${report.start} to ${report.end}`];
324 for (const query of report.analytics) {
325 lines.push("", query.label);
326 if (query.error) {
327 lines.push(` could not read: ${query.error}`);
328 continue;
329 }
330 const columns = [
331 ...query.dims.map((dim) => ({ title: dim, value: (row) => row[dim] })),
332 ...query.metrics.map((metric) => ({ title: metric, right: true, value: (row) => number(row[metric]) })),
333 ];
334 if (query.metrics.includes("errors")) columns.push({ title: "error_rate", right: true, value: (row) => percent(row.errors, row.requests) });
335 lines.push(...table(query.rows.slice(0, top), columns));
336 lines.push(` total: ${Object.entries(query.totals).map(([metric, value]) => `${metric} ${number(value)}`).join(", ")}`);
337 if (query.by_status?.length) {
338 lines.push(" by status:");
339 lines.push(...table(query.by_status, [
340 { title: "status", value: (row) => row.status },
341 { title: "requests", right: true, value: (row) => number(row.requests) },
342 { title: "errors", right: true, value: (row) => number(row.errors) },
343 ]).map((line) => ` ${line}`));
344 }
345 }
346 for (const question of report.telemetry) {
347 lines.push("", question.label);
348 if (question.error) {
349 lines.push(` could not read: ${question.error}`);
350 continue;
351 }
352 if (question.key === "samples") {
353 if (!question.rows.length) lines.push(" (none)");
354 for (const row of question.rows) {
355 lines.push(` ${row.at ?? "?"} ${row.entrypoint ?? "?"} ${row.event_type ?? "?"} ${row.outcome ?? "?"}${row.object ? ` ${row.object}` : ""}`);
356 if (row.error || row.message) lines.push(` ${String(row.error ?? row.message).slice(0, 300)}`);
357 }
358 continue;
359 }
360 const keys = Object.keys(question.rows[0] ?? {}).filter((key) => key !== "count");
361 lines.push(...table(question.rows.slice(0, top), [
362 ...keys.map((key) => ({ title: key, value: (row) => String(row[key] ?? "").slice(0, 120) })),
363 { title: "count", right: true, value: (row) => number(row.count) },
364 ]));
365 if (question.buckets?.length) {
366 lines.push(" most likely:");
367 for (const bucket of question.buckets) lines.push(` ${String(number(bucket.count)).padStart(6)} ${bucket.bucket}${bucket.expected ? " (expected)" : ""}: ${bucket.why}`);
368 }
369 }
370 if (report.keys) lines.push("", "Telemetry keys:", ...report.keys.map((key) => ` ${key}`));
371 if (report.notes.length) lines.push("", "Notes:", ...report.notes.map((note) => ` ${note}`));
372 return lines.join("\n");
373}
374
375async function post(auth, path, body) {
376 const response = await fetch(`${API}${path}`, {
377 method: "POST",
378 headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" },
379 body: JSON.stringify(body),
380 });
381 const answer = await response.json().catch(() => ({ success: false, errors: [{ message: `HTTP ${response.status}, not JSON` }] }));
382 if (!response.ok && !answer.errors?.length) answer.errors = [{ message: `HTTP ${response.status}` }];
383 return answer;
384}
385
386async function durableNames(auth, account) {
387 const out = {};
388 try {
389 for (let page = 1; page <= 10; page++) {
390 const response = await fetch(`${API}/accounts/${account}/workers/durable_objects/namespaces?per_page=100&page=${page}`, { headers: { ...auth, "user-agent": "g1t-ops" } });
391 const body = await response.json();
392 if (!response.ok || !Array.isArray(body.result)) break;
393 for (const item of body.result) if (item.id) out[item.id] = item.name ?? item.id;
394 if (body.result.length < 100) break;
395 }
396 } catch {
397 // Ids stand in for names.
398 }
399 return out;
400}
401
402async function analytics(auth, options, window, names) {
403 return Promise.all(
404 DO_QUERIES.map(async (query) => {
405 let error = null;
406 for (const variant of query.variants) {
407 try {
408 const body = await post(auth, "/graphql", { query: doQuery(variant), variables: { accountTag: options.account, start: window.start, end: window.end, script: options.script } });
409 const parsed = parseDoGroups(body, variant, names);
410 return { key: query.key, label: query.label, ...parsed, by_status: query.key === "by_namespace" ? byStatus(parsed) : undefined };
411 } catch (thrown) {
412 error ??= String(thrown.message ?? thrown);
413 }
414 }
415 return { key: query.key, label: query.label, error };
416 }),
417 );
418}
419
420async function telemetry(auth, options, window) {
421 const path = `/accounts/${options.account}/workers/observability/telemetry/query`;
422 const from = Date.parse(window.start);
423 const to = Date.parse(window.end);
424 let results = null;
425 for (const serviceKey of SERVICE_KEYS) {
426 results = await Promise.all(
427 telemetryQuestions({ script: options.script, from, to, samples: options.samples, serviceKey }).map(async (question) => {
428 try {
429 const body = await post(auth, path, question.body);
430 if (question.key === "samples") return { key: question.key, label: question.label, rows: parseEvents(body) };
431 const rows = parseCalculations(body);
432 const messageKey = question.key === "exceptions" ? KEYS.error : question.key === "error_logs" ? KEYS.message : null;
433 return { key: question.key, label: question.label, rows, buckets: messageKey ? bucketsOf(rows, messageKey) : undefined };
434 } catch (error) {
435 return { key: question.key, label: question.label, error: String(error.message ?? error) };
436 }
437 }),
438 );
439 // Another key names the Worker when this one found nothing at all.
440 if (results.some((result) => result.rows?.length)) return { serviceKey, results };
441 }
442 return { serviceKey: SERVICE_KEYS.at(-1), results };
443}
444
445async function telemetryKeys(auth, options, window) {
446 const body = await post(auth, `/accounts/${options.account}/workers/observability/telemetry/keys`, {
447 timeframe: { from: Date.parse(window.start), to: Date.parse(window.end) },
448 datasets: ["cloudflare-workers"],
449 filters: [filter(SERVICE_KEYS[0], "eq", options.script)],
450 limit: 500,
451 });
452 if (body?.success === false || body?.errors?.length) throw new Error(messagesOf(body));
453 return (body.result ?? []).map((key) => (typeof key === "string" ? key : `${key.key} (${key.type})`)).sort();
454}
455
456async function main() {
457 const options = parseArgs(process.argv.slice(2));
458 if (options.help) {
459 console.log(HELP);
460 return 0;
461 }
462 const auth = cloudflareAuth();
463 if (!auth) {
464 console.error(`Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read and Workers Observability: Read on account ${options.account}.`);
465 return 2;
466 }
467 const window = windowOf(new Date(), options.days);
468 const names = await durableNames(auth, options.account);
469 const [graph, logs, keys] = await Promise.all([
470 analytics(auth, options, window, names),
471 telemetry(auth, options, window),
472 options.keys ? telemetryKeys(auth, options, window).catch((error) => [`could not list keys: ${error.message}`]) : Promise.resolve(null),
473 ]);
474 const notes = [];
475 if (!Object.keys(names).length) notes.push("Namespace ids are not named: the token cannot read the Durable Objects listing (Workers Scripts: Read).");
476 notes.push(`Telemetry is filtered on ${logs.serviceKey}.`);
477 const report = { account: options.account, script: options.script, start: window.start, end: window.end, analytics: graph, telemetry: logs.results, keys, notes };
478 console.log(options.json ? JSON.stringify(report, null, 2) : format(report));
479 const read = [...graph, ...logs.results].some((part) => !part.error);
480 return read ? 0 : 1;
481}
482
483if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/runner-errors.mjs")) {
484 main().then(
485 (code) => process.exit(code),
486 (error) => {
487 console.error(`runner-errors: ${error.message}`);
488 process.exit(1);
489 },
490 );
491}

This file's history is long; its oldest lines are credited to the oldest commit read.