Skip to content
Sign up

flagon-io/g1t

public
0
377 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Spend guardrails documented, with the owner's dashboard checklist, and scripts/ops/platform-usage.mjs prints the month and the last 24 hours per script, queue, database and namespace1#!/usr/bin/env node
2// What is the platform using on Cloudflare? Asks Cloudflare's GraphQL
3// Analytics API, for the month so far (UTC, from the 1st) and the last 24
4// hours, how much each Worker, D1 database, queue, Durable Object namespace,
5// KV namespace and Artifacts namespace did, and prints the top of each with
6// totals. Workers Logs are added when the account's schema has a dataset for
7// them.
8//
9// Read-only: one GraphQL query per dataset and window, all at once, and a few
10// REST listings to put names on database, queue and namespace ids. A dataset
11// or field Cloudflare refuses is a note in the report, never the end of it.
12//
13// CLOUDFLARE_API_TOKEN=<token with Account Analytics: Read> \
14// node scripts/ops/platform-usage.mjs [--top 10] [--json] [--account <id>]
15//
16// --account (or CLOUDFLARE_ACCOUNT_ID) reads another account than g1t's.
17// --json prints one object with snake_case keys and every row, not just the top.
18// Names come from the D1, Queues, KV and Durable Objects listings when the
19// token may read them (D1: Read, Queues: Read, Workers KV Storage: Read,
20// Workers Scripts: Read); otherwise the ids are printed.
21
22import { ACCOUNT_ID, cloudflareAuth } from "../deploy/cloudflare.mjs";
23
24const API = "https://api.cloudflare.com/client/v4";
25
26const HELP = `node scripts/ops/platform-usage.mjs [--top N] [--json] [--account <id>]
27
28Cloudflare usage per Worker, D1 database, queue, Durable Object namespace,
29KV namespace and Artifacts namespace, month to date (UTC) and the last 24 hours.
30
31 --top N rows shown per dataset in the tables (default 10)
32 --json one JSON object, snake_case keys, every row
33 --account <id> the Cloudflare account (default CLOUDFLARE_ACCOUNT_ID, else g1t's)
34 --help this text
35
36Needs CLOUDFLARE_API_TOKEN (Account Analytics: Read), or CLOUDFLARE_API_KEY
37with CLOUDFLARE_EMAIL.`;
38
39/**
40 * The datasets the report asks for. Each has one or more variants, tried in
41 * order: when Cloudflare refuses a field, the next variant asks for less.
42 * `names` says which REST listing puts a name on the first dimension's ids.
43 */
44export const DATASETS = [
45 {
46 key: "workers",
47 label: "Workers invocations, by script",
48 dataset: "workersInvocationsAdaptive",
49 variants: [
50 { sum: ["requests", "errors", "cpuTimeUs"], dims: ["scriptName"] },
51 { sum: ["requests", "errors"], dims: ["scriptName"] },
52 ],
53 },
54 {
55 key: "d1",
56 label: "D1 rows, by database",
57 dataset: "d1AnalyticsAdaptiveGroups",
58 names: "d1",
59 variants: [
60 { sum: ["rowsRead", "rowsWritten", "readQueries", "writeQueries"], dims: ["databaseId"] },
61 { sum: ["rowsRead", "rowsWritten"], dims: ["databaseId"] },
62 ],
63 },
64 {
65 key: "queues",
66 label: "Queue operations, by queue",
67 dataset: "queueMessageOperationsAdaptiveGroups",
68 names: "queues",
69 variants: [{ sum: ["billableOperations"], dims: ["queueId"] }],
70 },
71 {
72 key: "durable_objects",
73 label: "Durable Object requests, by script",
74 dataset: "durableObjectsInvocationsAdaptiveGroups",
75 variants: [
76 { sum: ["requests", "errors"], dims: ["scriptName"] },
77 { sum: ["requests"], dims: ["scriptName"] },
78 ],
79 },
80 {
81 key: "durable_objects_periodic",
82 label: "Durable Object time and storage, by namespace",
83 dataset: "durableObjectsPeriodicGroups",
84 names: "durable_objects",
85 variants: [
86 { sum: ["activeTime", "cpuTime", "storageReadUnits", "storageWriteUnits"], dims: ["namespaceId"] },
87 { sum: ["activeTime", "storageWriteUnits"], dims: ["namespaceId"] },
88 // Periodic groups may filter by the minute rather than by datetime.
89 { sum: ["activeTime"], dims: ["namespaceId"], time: "datetimeMinute" },
90 ],
91 },
92 {
93 key: "kv",
94 label: "KV operations, by namespace and action",
95 dataset: "kvOperationsAdaptiveGroups",
96 names: "kv",
97 variants: [{ sum: ["requests"], dims: ["namespaceId", "actionType"] }],
98 },
99 {
100 key: "artifacts",
101 label: "Artifacts events, by namespace and type",
102 dataset: "artifactsEventsAdaptiveGroups",
103 variants: [{ count: true, sum: ["durationMs"], dims: ["repositoryNamespace", "eventType"] }],
104 },
105 {
106 key: "workers_logs",
107 label: "Workers Logs events, by script",
108 // Which dataset holds Workers Logs is read from the schema (logsDataset).
109 dataset: null,
110 optional: true,
111 variants: [
112 { count: true, dims: ["scriptName"] },
113 { count: true, dims: [] },
114 ],
115 },
116];
117
118/** The two windows: the UTC month so far, and the last 24 hours. */
119export function windows(now = new Date()) {
120 const end = new Date(now.getTime());
121 const monthStart = new Date(Date.UTC(end.getUTCFullYear(), end.getUTCMonth(), 1));
122 return [
123 { key: "month_to_date", label: "Month to date (UTC)", start: monthStart.toISOString(), end: end.toISOString() },
124 { key: "last_24h", label: "Last 24 hours", start: new Date(end.getTime() - 24 * 3600 * 1000).toISOString(), end: end.toISOString() },
125 ];
126}
127
128/** Which account field holds Workers Logs, from the account type's field names; null when none does. */
129export function logsDataset(fieldNames) {
130 const known = ["workersObservabilityEventsAdaptiveGroups", "workersLogsEventsAdaptiveGroups", "workersLogsAdaptiveGroups"];
131 return known.find((name) => fieldNames.includes(name)) ?? fieldNames.find((name) => /^workers.*(logs|observability).*groups$/i.test(name)) ?? null;
132}
133
134/** One dataset's query, for one variant. The rows come back under `rows`. */
135export function buildQuery(dataset, variant) {
136 const time = variant.time ?? "datetime";
137 const fields = [
138 variant.count ? "count" : "",
139 variant.sum?.length ? `sum { ${variant.sum.join(" ")} }` : "",
140 variant.dims.length ? `dimensions { ${variant.dims.join(" ")} }` : "",
141 ].filter(Boolean);
142 return `query PlatformUsage($accountTag: String!, $start: Time!, $end: Time!) {
143 viewer {
144 accounts(filter: { accountTag: $accountTag }) {
145 rows: ${dataset}(limit: 10000, filter: { ${time}_geq: $start, ${time}_leq: $end }) {
146 ${fields.join("\n ")}
147 }
148 }
149 }
150}`;
151}
152
153/** camelCase to snake_case, for the JSON report's keys. */
154export const snake = (name) => name.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
155
156/** The metric names a variant reports, snake_case: `count` first, then its sums. */
157export const metricsOf = (variant) => [...(variant.count ? ["count"] : []), ...(variant.sum ?? []).map(snake)];
158
159/**
160 * A GraphQL answer to one dataset's query as rows, one per name, summed and
161 * sorted by the first metric (largest first), with totals per metric. Throws
162 * with Cloudflare's message when the answer has errors or no such dataset.
163 */
164export function parseGroups(body, variant, names = {}) {
165 if (body?.errors?.length) throw new Error(body.errors.map((error) => error.message).join("; ").slice(0, 400));
166 const groups = body?.data?.viewer?.accounts?.[0]?.rows;
167 if (!Array.isArray(groups)) throw new Error("no rows in the answer");
168 const metrics = metricsOf(variant);
169 const byName = new Map();
170 for (const group of groups) {
171 const parts = variant.dims.map((dim, at) => {
172 const value = group.dimensions?.[dim];
173 const text = value == null || value === "" ? "(none)" : String(value);
174 return at === 0 ? (names[text] ?? text) : text;
175 });
176 const name = parts.join(" / ") || "(all)";
177 const row = byName.get(name) ?? { name, ...Object.fromEntries(metrics.map((metric) => [metric, 0])) };
178 if (variant.count) row.count += Number(group.count ?? 0);
179 for (const field of variant.sum ?? []) row[snake(field)] += Number(group.sum?.[field] ?? 0);
180 byName.set(name, row);
181 }
182 const rows = sortRows([...byName.values()], metrics[0]);
183 return { metrics, rows, totals: totalsOf(rows, metrics) };
184}
185
186/** Rows largest first by one metric, ties by name. */
187export function sortRows(rows, metric) {
188 return [...rows].sort((a, b) => (b[metric] ?? 0) - (a[metric] ?? 0) || a.name.localeCompare(b.name));
189}
190
191/** Each metric summed over all rows. */
192export function totalsOf(rows, metrics) {
193 return Object.fromEntries(metrics.map((metric) => [metric, rows.reduce((total, row) => total + (row[metric] ?? 0), 0)]));
194}
195
196/**
197 * The report from every dataset's outcome in every window. An outcome is
198 * `{ body, variant }` (a GraphQL answer) or `{ error }` or `{ skipped }`;
199 * whatever cannot be read becomes a note and the other datasets still count.
200 */
201export function assemble({ account, now, windowList, outcomes, names = {} }) {
202 return {
203 account,
204 generated_at: now.toISOString(),
205 windows: windowList.map((window) => {
206 const notes = [];
207 const datasets = [];
208 for (const spec of DATASETS) {
209 const outcome = outcomes[window.key]?.[spec.key];
210 if (!outcome) continue;
211 if (outcome.skipped) {
212 notes.push(`${spec.label}: ${outcome.skipped}`);
213 continue;
214 }
215 try {
216 if (outcome.error) throw outcome.error;
217 const parsed = parseGroups(outcome.body, outcome.variant, names[spec.names] ?? {});
218 datasets.push({ key: spec.key, label: spec.label, dataset: outcome.dataset ?? spec.dataset, ...parsed });
219 } catch (error) {
220 notes.push(`${spec.label} (${outcome.dataset ?? spec.dataset ?? "no dataset"}): ${String(error.message ?? error).split("\n")[0]}`);
221 }
222 }
223 return { key: window.key, label: window.label, start: window.start, end: window.end, datasets, notes };
224 }),
225 };
226}
227
228const number = (value) => (Number.isInteger(value) ? value.toLocaleString("en-US") : value.toLocaleString("en-US", { maximumFractionDigits: 2 }));
229
230/** The report as plain tables: the top rows of each dataset, and a totals line. */
231export function format(report, top = 10) {
232 const lines = [`Cloudflare usage for account ${report.account}, ${report.generated_at}`];
233 for (const window of report.windows) {
234 lines.push("", `== ${window.label}: ${window.start} to ${window.end}`);
235 for (const set of window.datasets) {
236 lines.push("", `${set.label} (${set.dataset})`);
237 const shown = set.rows.slice(0, top);
238 const cells = [["name", ...set.metrics], ...shown.map((row) => [row.name, ...set.metrics.map((m) => number(row[m]))]), ["total", ...set.metrics.map((m) => number(set.totals[m]))]];
239 const widths = cells[0].map((_, at) => Math.max(...cells.map((row) => String(row[at]).length)));
240 const render = (row) => " " + row.map((cell, at) => (at === 0 ? String(cell).padEnd(widths[at]) : String(cell).padStart(widths[at]))).join(" ");
241 lines.push(render(cells[0]));
242 if (!shown.length) lines.push(" (nothing in this window)");
243 for (const row of cells.slice(1, -1)) lines.push(render(row));
244 if (set.rows.length > shown.length) lines.push(` ... and ${set.rows.length - shown.length} more`);
245 lines.push(render(cells.at(-1)));
246 }
247 if (window.notes.length) {
248 lines.push("", "Notes:");
249 for (const note of window.notes) lines.push(` ${note}`);
250 }
251 }
252 return lines.join("\n");
253}
254
255/** The command line: flags and the account. */
256export function parseArgs(argv, env = process.env) {
257 const option = (name) => {
258 const at = argv.indexOf(name);
259 return at >= 0 && argv[at + 1] && !argv[at + 1].startsWith("--") ? argv[at + 1] : null;
260 };
261 return {
262 help: argv.includes("--help") || argv.includes("-h"),
263 json: argv.includes("--json"),
264 top: Math.max(1, Number(option("--top")) || 10),
265 account: option("--account") || env.CLOUDFLARE_ACCOUNT_ID || ACCOUNT_ID,
266 };
267}
268
269async function graphql(auth, account, query, variables = {}) {
270 const response = await fetch(`${API}/graphql`, {
271 method: "POST",
272 headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" },
273 body: JSON.stringify({ query, variables: { accountTag: account, ...variables } }),
274 });
275 const body = await response.json().catch(() => ({ errors: [{ message: `HTTP ${response.status}, not JSON` }] }));
276 if (!response.ok && !body.errors?.length) body.errors = [{ message: `HTTP ${response.status}` }];
277 return body;
278}
279
280/** The account type's field names, to skip datasets the schema lacks; null when it cannot be read. */
281async function schemaFields(auth, account) {
282 try {
283 const body = await graphql(auth, account, `{ __type(name: "account") { fields { name } } }`);
284 const fields = body.data?.__type?.fields;
285 return Array.isArray(fields) ? fields.map((field) => field.name) : null;
286 } catch {
287 return null;
288 }
289}
290
291/** One dataset in one window: each variant in turn until one is answered. */
292async function ask(auth, account, dataset, spec, window) {
293 let last = null;
294 for (const variant of spec.variants) {
295 try {
296 const body = await graphql(auth, account, buildQuery(dataset, variant), { start: window.start, end: window.end });
297 if (!body.errors?.length) return { body, variant, dataset };
298 last = { body, variant, dataset };
299 } catch (error) {
300 last = { error, dataset };
301 }
302 }
303 return last;
304}
305
306/** A REST listing as id to name; empty when the token may not read it. */
307async function listing(auth, path, id, name) {
308 const out = {};
309 try {
310 for (let page = 1; page <= 10; page++) {
311 const response = await fetch(`${API}${path}${path.includes("?") ? "&" : "?"}per_page=100&page=${page}`, { headers: { ...auth, "user-agent": "g1t-ops" } });
312 const body = await response.json();
313 if (!response.ok || !Array.isArray(body.result)) break;
314 for (const item of body.result) if (item[id]) out[item[id]] = item[name] ?? item[id];
315 if (body.result.length < 100) break;
316 }
317 } catch {
318 // Ids stand in for names.
319 }
320 return out;
321}
322
323async function main() {
324 const options = parseArgs(process.argv.slice(2));
325 if (options.help) {
326 console.log(HELP);
327 return 0;
328 }
329 const auth = cloudflareAuth();
330 if (!auth) {
331 console.error(`Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read on account ${options.account}, or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.`);
332 return 2;
333 }
334 const now = new Date();
335 const windowList = windows(now);
336 const account = options.account;
337 const base = `/accounts/${account}`;
338 const [fields, d1, queues, kv, durable] = await Promise.all([
339 schemaFields(auth, account),
340 listing(auth, `${base}/d1/database`, "uuid", "name"),
341 listing(auth, `${base}/queues`, "queue_id", "queue_name"),
342 listing(auth, `${base}/storage/kv/namespaces`, "id", "title"),
343 listing(auth, `${base}/workers/durable_objects/namespaces`, "id", "name"),
344 ]);
345 const names = { d1, queues, kv, durable_objects: durable };
346 const outcomes = {};
347 const jobs = [];
348 for (const window of windowList) {
349 outcomes[window.key] = {};
350 for (const spec of DATASETS) {
351 const dataset = spec.dataset ?? (fields ? logsDataset(fields) : null);
352 if (!dataset) {
353 if (!spec.optional) outcomes[window.key][spec.key] = { skipped: "no dataset" };
354 continue;
355 }
356 if (fields && !fields.includes(dataset)) {
357 if (!spec.optional) outcomes[window.key][spec.key] = { skipped: `${dataset} is not in this account's schema` };
358 continue;
359 }
360 jobs.push(ask(auth, account, dataset, spec, window).then((outcome) => (outcomes[window.key][spec.key] = outcome)));
361 }
362 }
363 await Promise.all(jobs);
364 const report = assemble({ account, now, windowList, outcomes, names });
365 console.log(options.json ? JSON.stringify(report, null, 2) : format(report, options.top));
366 return 0;
367}
368
369if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/platform-usage.mjs")) {
370 main().then(
371 (code) => process.exit(code),
372 (error) => {
373 console.error(`platform-usage: ${error.message}`);
374 process.exit(1);
375 },
376 );
377}