Skip to content
Sign in

flagon-io/g1t

public
0

g1t/scripts/ops/artifacts-usage.mjs

291 lines13,459 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1#!/usr/bin/env node
2// What does Cloudflare count as an Artifacts "operation"? Compares what
3// Cloudflare's analytics say happened (GraphQL `artifactsEventsAdaptiveGroups`:
4// create, fork, push, pull, delete, and errors) with what g1t metered itself
5// (repos D1: `artifacts_meters`, every interaction by kind, and
6// `git_operations`, what workspaces are counted for), day by day, and says
7// which of g1t's meters line up with each of Cloudflare's events.
8// docs/ARTIFACTS.md (R1) explains how to read it.
9//
10// Read-only: one GraphQL query, and SELECTs against the g1t-repos database.
11//
12// CLOUDFLARE_API_TOKEN=<token with Account Analytics: Read> \
13// node scripts/ops/artifacts-usage.mjs [--days 31] [--json]
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships14// node scripts/ops/artifacts-usage.mjs --hours 2026-10-07
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily15//
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships16// --hours DAY shows one UTC day hour by hour (Cloudflare's operations and
17// errors against `git_operations`), and the errors by message and repository:
18// a fix that lands mid-day is judged on the hours after it.
19//
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily20// The D1 queries run through Wrangler with the same environment (so the
21// token needs D1: Read too), or with CLOUDFLARE_D1_TOKEN when that is set,
22// or as you are logged in (`npx wrangler login`) when neither has it.
23
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results24import { ACCOUNT_ID, cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily25import { ROOT } from "../deploy/stack.mjs";
26import { join } from "node:path";
27
28const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
29const DATABASE = "g1t-repos";
30const NAMESPACE = process.env.ARTIFACTS_NAMESPACE || null;
31
32const args = process.argv.slice(2);
33const flag = (name) => args.includes(name);
34const option = (name, fallback) => {
35 const at = args.indexOf(name);
36 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
37};
38const days = Math.min(31, Math.max(1, Number(option("--days", "31")) || 31));
39const asJson = flag("--json");
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships40const hoursOf = option("--hours", null);
41if (hoursOf && !/^\d{4}-\d{2}-\d{2}$/.test(hoursOf)) {
42 console.error("--hours takes a UTC day, YYYY-MM-DD");
43 process.exit(2);
44}
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily45
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results46const auth = cloudflareAuth();
47if (!auth) {
48 console.error(
49 "Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read on account " + ACCOUNT_ID +
50 ", or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.",
51 );
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily52 process.exit(2);
53}
54
55const end = new Date();
56const start = new Date(end.getTime() - days * 24 * 3600 * 1000);
57const day = (date) => date.toISOString().slice(0, 10);
58
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships59async function graphql(query, variables) {
60 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
61 method: "POST",
62 headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" },
63 body: JSON.stringify({ query, variables: { accountTag: ACCOUNT_ID, ...variables } }),
64 });
65 const body = await response.json();
66 if (!response.ok || body.errors?.length) {
67 throw new Error(`GraphQL: ${response.status} ${JSON.stringify(body.errors ?? body).slice(0, 600)}`);
68 }
69 return body.data?.viewer?.accounts?.[0] ?? {};
70}
71
72/** One UTC day by the hour: Cloudflare's operations and errors against `git_operations`. */
73async function hourly(dayText) {
74 const from = `${dayText}T00:00:00Z`;
75 const to = new Date(Date.parse(from) + 24 * 3600 * 1000).toISOString();
76 const nsFilter = NAMESPACE ? `, repositoryNamespace: "${NAMESPACE.replace(/"/g, "")}"` : "";
77 const query = `query ArtifactsHours($accountTag: String!, $start: Time!, $end: Time!) {
78 viewer {
79 accounts(filter: { accountTag: $accountTag }) {
80 hours: artifactsEventsAdaptiveGroups(
81 limit: 10000
82 filter: { datetime_geq: $start, datetime_lt: $end${nsFilter} }
83 orderBy: [datetimeHour_ASC]
84 ) { count dimensions { datetimeHour eventKind eventType } }
85 errors: artifactsEventsAdaptiveGroups(
86 limit: 10000
87 filter: { datetime_geq: $start, datetime_lt: $end, eventKind: "error"${nsFilter} }
88 orderBy: [count_DESC]
89 ) { count dimensions { eventType errorMessage repositoryName } }
90 }
91 }
92 }`;
93 const [account, operations] = await Promise.all([
94 graphql(query, { start: from, end: to }),
95 d1(
96 `SELECT substr(hour, 12, 2) AS h, SUM(operations) AS operations FROM git_operations WHERE hour >= '${dayText}T00' AND hour <= '${dayText}T23' GROUP BY h`,
97 ),
98 ]);
99 const ours = Object.fromEntries(operations.map((row) => [row.h, Number(row.operations)]));
100 const types = ["pull", "push", "create", "fork", "delete"];
101 const byHour = {};
102 for (const group of account.hours ?? []) {
103 const { datetimeHour, eventKind, eventType } = group.dimensions;
104 const key = eventKind === "error" ? "errors" : eventType;
105 if (key !== "errors" && !types.includes(key)) continue;
106 const h = datetimeHour.slice(11, 13);
107 (byHour[h] ??= {})[key] = (byHour[h][key] ?? 0) + group.count;
108 }
109 console.log(`Artifacts by the hour, ${dayText} UTC${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
110 console.log(["hour", ...types.map((t) => pad(`cf.${t}`, 9)), pad("cf.ops", 8), pad("g1t.ops", 8), pad("ratio", 6), pad("cf.errors", 10)].join(" "));
111 let cfTotal = 0;
112 let ourTotal = 0;
113 for (let i = 0; i < 24; i++) {
114 const h = String(i).padStart(2, "0");
115 const cf = byHour[h] ?? {};
116 const cfOps = types.reduce((total, t) => total + (cf[t] ?? 0), 0);
117 const mine = ours[h] ?? 0;
118 if (!cfOps && !mine && !cf.errors) continue;
119 cfTotal += cfOps;
120 ourTotal += mine;
121 console.log(
122 [h + " ", ...types.map((t) => pad(cf[t] ?? 0, 9)), pad(cfOps, 8), pad(mine, 8), pad(mine ? (cfOps / mine).toFixed(2) : "n/a", 6), pad(cf.errors ?? 0, 10)].join(" "),
123 );
124 }
125 console.log(`\nday cf.ops ${cfTotal}, g1t.ops ${ourTotal}${ourTotal ? `, ratio ${(cfTotal / ourTotal).toFixed(2)}` : ""}`);
126 const messages = {};
127 const repositories = {};
128 for (const group of account.errors ?? []) {
129 const { eventType, errorMessage, repositoryName } = group.dimensions;
130 const key = `${eventType}: ${errorMessage || "(no message)"}`;
131 messages[key] = (messages[key] ?? 0) + group.count;
132 repositories[repositoryName] = (repositories[repositoryName] ?? 0) + group.count;
133 }
134 console.log("\nErrors by message:");
135 for (const [message, count] of Object.entries(messages).sort((a, b) => b[1] - a[1])) console.log(` ${pad(count, 6)} ${message}`);
136 console.log("Errors by repository (top 8):");
137 for (const [name, count] of Object.entries(repositories).sort((a, b) => b[1] - a[1]).slice(0, 8)) console.log(` ${pad(count, 6)} ${name}`);
138 console.log(
139 "\ng1t.ops is what workspaces are counted for (billable meters only; nightly backups are g1t's own and not in it). An hour can straddle the two sides of a write by a few seconds.",
140 );
141}
142
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily143/** Cloudflare's own count, by day, event kind and type (and namespace). */
144async function cloudflare() {
145 const query = `query ArtifactsUsage($accountTag: String!, $start: Time!, $end: Time!) {
146 viewer {
147 accounts(filter: { accountTag: $accountTag }) {
148 artifactsEventsAdaptiveGroups(
149 limit: 10000
150 filter: { datetime_geq: $start, datetime_leq: $end }
151 orderBy: [date_ASC]
152 ) {
153 count
154 sum { durationMs }
155 dimensions { date eventKind eventType repositoryNamespace }
156 }
157 }
158 }
159 }`;
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships160 const account = await graphql(query, { start: start.toISOString(), end: end.toISOString() });
161 const groups = account.artifactsEventsAdaptiveGroups ?? [];
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily162 return groups
163 .filter((group) => !NAMESPACE || group.dimensions.repositoryNamespace === NAMESPACE)
164 .map((group) => ({
165 day: group.dimensions.date,
166 kind: group.dimensions.eventKind,
167 type: group.dimensions.eventType,
168 namespace: group.dimensions.repositoryNamespace,
169 count: group.count,
170 ms: group.sum?.durationMs ?? 0,
171 }));
172}
173
174/** A read-only query against the repos database. */
175async function d1(sql) {
176 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
177 if (process.env.CLOUDFLARE_D1_TOKEN) env.CLOUDFLARE_API_TOKEN = process.env.CLOUDFLARE_D1_TOKEN;
178 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
179 cwd: join(ROOT, "services/repos"),
180 env,
181 });
182 if (code !== 0) throw new Error(out.slice(-600));
183 return jsonFrom(out)[0]?.results ?? [];
184}
185
186async function ours() {
187 const since = day(start);
188 const operations = await d1(
189 `SELECT substr(hour, 1, 10) AS day, SUM(operations) AS operations FROM git_operations WHERE hour >= '${since}' GROUP BY day ORDER BY day`,
190 );
191 let meters = [];
192 let mapping = [];
193 try {
194 meters = await d1(
195 `SELECT day, meter, SUM(count) AS count, SUM(bytes_in) AS bytes_in, SUM(bytes_out) AS bytes_out FROM artifacts_meters WHERE day >= '${since}'` +
196 (NAMESPACE ? ` AND store = '${NAMESPACE.replace(/'/g, "")}'` : "") +
197 " GROUP BY day, meter ORDER BY day, meter",
198 );
199 mapping = await d1("SELECT meter, cost_operations, billable_operations FROM operation_mapping ORDER BY meter");
200 } catch (error) {
201 console.error(`(artifacts_meters not readable yet: migration 0011 not applied? ${String(error.message).split("\n")[0]})`);
202 }
203 return { operations, meters, mapping };
204}
205
206/** Cloudflare's events against combinations of g1t's meters: which line up. */
207export function candidates(cfTotals, meterTotals) {
208 const sum = (names) => names.reduce((total, name) => total + (meterTotals[name] ?? 0), 0);
209 const options = {
210 pull: [
211 ["git.fetch"],
212 ["git.fetch", "internal.git.fetch"],
213 ["git.fetch", "internal.git.fetch", "git.ls_refs"],
214 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs"],
215 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs", "cache.info_refs", "cache.ls_refs"],
216 ],
217 push: [["git.receive_pack"], ["git.receive_pack", "internal.git.receive_pack"]],
218 create: [["binding.create"]],
219 fork: [["binding.fork"]],
220 delete: [["binding.delete"]],
221 };
222 const rows = [];
223 for (const [type, combos] of Object.entries(options)) {
224 const theirs = cfTotals[type] ?? 0;
225 for (const combo of combos) {
226 const mine = sum(combo);
227 rows.push({ type, cloudflare: theirs, meters: combo.join(" + "), g1t: mine, ratio: mine ? theirs / mine : null });
228 }
229 }
230 return rows;
231}
232
233const pad = (value, width) => String(value).padStart(width);
234
235async function main() {
236 const [events, mine] = await Promise.all([cloudflare(), ours()]);
237 const cfTotals = {};
238 const cfByDay = {};
239 for (const event of events) {
240 const key = event.kind === "error" ? `error:${event.type}` : event.type;
241 cfTotals[key] = (cfTotals[key] ?? 0) + event.count;
242 (cfByDay[event.day] ??= {})[key] = (cfByDay[event.day]?.[key] ?? 0) + event.count;
243 }
244 const meterTotals = {};
245 const meterByDay = {};
246 for (const row of mine.meters) {
247 meterTotals[row.meter] = (meterTotals[row.meter] ?? 0) + Number(row.count);
248 (meterByDay[row.day] ??= {})[row.meter] = Number(row.count);
249 }
250 const opsByDay = Object.fromEntries(mine.operations.map((row) => [row.day, Number(row.operations)]));
251 const lined = candidates(cfTotals, meterTotals);
252 if (asJson) {
253 console.log(JSON.stringify({ from: day(start), to: day(end), cloudflare: events, meters: mine.meters, git_operations: mine.operations, mapping: mine.mapping, candidates: lined }, null, 2));
254 return;
255 }
256 console.log(`Artifacts usage ${day(start)} to ${day(end)}${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
257 const types = ["pull", "push", "create", "fork", "delete"];
258 console.log(["day ", ...types.map((t) => pad(`cf.${t}`, 10)), pad("cf.errors", 10), pad("g1t.fetch", 10), pad("g1t.push", 10), pad("g1t.ops", 10)].join(" "));
259 const allDays = [...new Set([...Object.keys(cfByDay), ...Object.keys(meterByDay), ...Object.keys(opsByDay)])].sort();
260 for (const d of allDays) {
261 const cf = cfByDay[d] ?? {};
262 const m = meterByDay[d] ?? {};
263 const errors = Object.entries(cf).filter(([key]) => key.startsWith("error:")).reduce((total, [, n]) => total + n, 0);
264 console.log(
265 [
266 d,
267 ...types.map((t) => pad(cf[t] ?? 0, 10)),
268 pad(errors, 10),
269 pad((m["git.fetch"] ?? 0) + (m["internal.git.fetch"] ?? 0), 10),
270 pad((m["git.receive_pack"] ?? 0) + (m["internal.git.receive_pack"] ?? 0), 10),
271 pad(opsByDay[d] ?? 0, 10),
272 ].join(" "),
273 );
274 }
275 console.log("\nCloudflare totals:", JSON.stringify(cfTotals));
276 console.log("g1t meter totals: ", JSON.stringify(meterTotals));
277 console.log("\nWhich g1t meters line up with each Cloudflare event (ratio = Cloudflare / g1t; 1.00 is a match):");
278 for (const row of lined) {
279 console.log(` ${row.type.padEnd(7)} ${pad(row.cloudflare, 8)} vs ${pad(row.g1t, 8)} ${row.ratio == null ? " n/a" : row.ratio.toFixed(2).padStart(5)} ${row.meters}`);
280 }
281 console.log("\nNow counting as operations (operation_mapping):");
282 for (const row of mine.mapping) console.log(` ${row.meter.padEnd(28)} cost ${row.cost_operations} billable ${row.billable_operations}`);
283 console.log(
284 "\nThe days before 2026-10-06's meters were deployed have Cloudflare's numbers only. Errors are Cloudflare's error events (rateLimited, serverError, ...).",
285 );
286}
287
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships288(hoursOf ? hourly(hoursOf) : main()).catch((error) => {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily289 console.error(error.message);
290 process.exit(1);
291});