Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
309 linesCodeBlameRaw
1#!/usr/bin/env node
2// What does Cloudflare count as an Artifacts "operation"? Compares what
3// Cloudflare's analytics say happened (GraphQL `artifactsEventsAdaptiveGroups`:
4// create, fork, push, pull, delete, and errors) with what g1t metered itself
5// (repos D1: `artifacts_meters`, every interaction by kind, and
6// `git_operations`, what workspaces are counted for), day by day, and says
7// which of g1t's meters line up with each of Cloudflare's events.
8//
9// Read-only: one GraphQL query, and SELECTs against the g1t-repos database.
10//
11// CLOUDFLARE_API_TOKEN=<token with Account Analytics: Read> \
12// node scripts/ops/gitstore-usage.mjs [--days 31] [--json]
13// node scripts/ops/gitstore-usage.mjs --hours 2026-10-07
14//
15// --hours DAY shows one UTC day hour by hour (Cloudflare's operations and
16// errors against `git_operations`), and the errors by message and repository:
17// a fix that lands mid-day is judged on the hours after it.
18//
19// The D1 queries run through Wrangler with the same environment (so the
20// token needs D1: Read too), or with CLOUDFLARE_D1_TOKEN when that is set,
21// or as you are logged in (`npx wrangler login`) when neither has it.
22
23import { ACCOUNT_ID, cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
24import { ROOT } from "../deploy/stack.mjs";
25import { join } from "node:path";
26
27const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
28const DATABASE = "g1t-repos";
29const NAMESPACE = process.env.GITSTORE_NAMESPACE || null;
30
31const args = process.argv.slice(2);
32const flag = (name) => args.includes(name);
33const option = (name, fallback) => {
34 const at = args.indexOf(name);
35 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
36};
37const days = Math.min(31, Math.max(1, Number(option("--days", "31")) || 31));
38const asJson = flag("--json");
39const hoursOf = option("--hours", null);
40if (hoursOf && !/^\d{4}-\d{2}-\d{2}$/.test(hoursOf)) {
41 console.error("--hours takes a UTC day, YYYY-MM-DD");
42 process.exit(2);
43}
44
45const auth = cloudflareAuth();
46if (!auth) {
47 console.error("Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read on account " + ACCOUNT_ID + ", or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.");
48 process.exit(2);
49}
50
51const end = new Date();
52const start = new Date(end.getTime() - days * 24 * 3600 * 1000);
53const day = (date) => date.toISOString().slice(0, 10);
54
55async function graphql(query, variables) {
56 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
57 method: "POST",
58 headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" },
59 body: JSON.stringify({ query, variables: { accountTag: ACCOUNT_ID, ...variables } }),
60 });
61 const body = await response.json();
62 if (!response.ok || body.errors?.length) {
63 throw new Error(`GraphQL: ${response.status} ${JSON.stringify(body.errors ?? body).slice(0, 600)}`);
64 }
65 return body.data?.viewer?.accounts?.[0] ?? {};
66}
67
68/** One UTC day by the hour: Cloudflare's operations and errors against `git_operations`. */
69async function hourly(dayText) {
70 const from = `${dayText}T00:00:00Z`;
71 const to = new Date(Date.parse(from) + 24 * 3600 * 1000).toISOString();
72 const nsFilter = NAMESPACE ? `, repositoryNamespace: "${NAMESPACE.replace(/"/g, "")}"` : "";
73 const query = `query ArtifactsHours($accountTag: String!, $start: Time!, $end: Time!) {
74 viewer {
75 accounts(filter: { accountTag: $accountTag }) {
76 hours: artifactsEventsAdaptiveGroups(
77 limit: 10000
78 filter: { datetime_geq: $start, datetime_lt: $end${nsFilter} }
79 orderBy: [datetimeHour_ASC]
80 ) { count dimensions { datetimeHour eventKind eventType } }
81 errors: artifactsEventsAdaptiveGroups(
82 limit: 10000
83 filter: { datetime_geq: $start, datetime_lt: $end, eventKind: "error"${nsFilter} }
84 orderBy: [count_DESC]
85 ) { count dimensions { eventType errorMessage repositoryName } }
86 }
87 }
88 }`;
89 const [account, operations] = await Promise.all([
90 graphql(query, { start: from, end: to }),
91 d1(
92 `SELECT substr(hour, 12, 2) AS h, SUM(operations) AS operations FROM git_operations WHERE hour >= '${dayText}T00' AND hour <= '${dayText}T23' GROUP BY h`,
93 ),
94 ]);
95 const ours = Object.fromEntries(operations.map((row) => [row.h, Number(row.operations)]));
96 const types = ["pull", "push", "create", "fork", "delete"];
97 const byHour = {};
98 for (const group of account.hours ?? []) {
99 const { datetimeHour, eventKind, eventType } = group.dimensions;
100 const key = eventKind === "error" ? "errors" : eventType;
101 if (key !== "errors" && !types.includes(key)) continue;
102 const h = datetimeHour.slice(11, 13);
103 (byHour[h] ??= {})[key] = (byHour[h][key] ?? 0) + group.count;
104 }
105 console.log(`Artifacts by the hour, ${dayText} UTC${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
106 console.log(["hour", ...types.map((t) => pad(`cf.${t}`, 9)), pad("cf.ops", 8), pad("g1t.ops", 8), pad("ratio", 6), pad("cf.errors", 10)].join(" "));
107 let cfTotal = 0;
108 let ourTotal = 0;
109 for (let i = 0; i < 24; i++) {
110 const h = String(i).padStart(2, "0");
111 const cf = byHour[h] ?? {};
112 const cfOps = types.reduce((total, t) => total + (cf[t] ?? 0), 0);
113 const mine = ours[h] ?? 0;
114 if (!cfOps && !mine && !cf.errors) continue;
115 cfTotal += cfOps;
116 ourTotal += mine;
117 console.log(
118 [
119 h + " ",
120 ...types.map((t) => pad(cf[t] ?? 0, 9)),
121 pad(cfOps, 8),
122 pad(mine, 8),
123 pad(mine ? (cfOps / mine).toFixed(2) : "n/a", 6),
124 pad(cf.errors ?? 0, 10),
125 ].join(" "),
126 );
127 }
128 console.log(`\nday cf.ops ${cfTotal}, g1t.ops ${ourTotal}${ourTotal ? `, ratio ${(cfTotal / ourTotal).toFixed(2)}` : ""}`);
129 const messages = {};
130 const repositories = {};
131 for (const group of account.errors ?? []) {
132 const { eventType, errorMessage, repositoryName } = group.dimensions;
133 const key = `${eventType}: ${errorMessage || "(no message)"}`;
134 messages[key] = (messages[key] ?? 0) + group.count;
135 repositories[repositoryName] = (repositories[repositoryName] ?? 0) + group.count;
136 }
137 console.log("\nErrors by message:");
138 for (const [message, count] of Object.entries(messages).sort((a, b) => b[1] - a[1])) console.log(` ${pad(count, 6)} ${message}`);
139 console.log("Errors by repository (top 8):");
140 for (const [name, count] of Object.entries(repositories)
141 .sort((a, b) => b[1] - a[1])
142 .slice(0, 8))
143 console.log(` ${pad(count, 6)} ${name}`);
144 console.log(
145 "\ng1t.ops is what workspaces are counted for (billable meters only; nightly backups are g1t's own and not in it). An hour can straddle the two sides of a write by a few seconds.",
146 );
147}
148
149/** Cloudflare's own count, by day, event kind and type (and namespace). */
150async function cloudflare() {
151 const query = `query ArtifactsUsage($accountTag: String!, $start: Time!, $end: Time!) {
152 viewer {
153 accounts(filter: { accountTag: $accountTag }) {
154 artifactsEventsAdaptiveGroups(
155 limit: 10000
156 filter: { datetime_geq: $start, datetime_leq: $end }
157 orderBy: [date_ASC]
158 ) {
159 count
160 sum { durationMs }
161 dimensions { date eventKind eventType repositoryNamespace }
162 }
163 }
164 }
165 }`;
166 const account = await graphql(query, { start: start.toISOString(), end: end.toISOString() });
167 const groups = account.artifactsEventsAdaptiveGroups ?? [];
168 return groups
169 .filter((group) => !NAMESPACE || group.dimensions.repositoryNamespace === NAMESPACE)
170 .map((group) => ({
171 day: group.dimensions.date,
172 kind: group.dimensions.eventKind,
173 type: group.dimensions.eventType,
174 namespace: group.dimensions.repositoryNamespace,
175 count: group.count,
176 ms: group.sum?.durationMs ?? 0,
177 }));
178}
179
180/** A read-only query against the repos database. */
181async function d1(sql) {
182 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
183 if (process.env.CLOUDFLARE_D1_TOKEN) env.CLOUDFLARE_API_TOKEN = process.env.CLOUDFLARE_D1_TOKEN;
184 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
185 cwd: join(ROOT, "services/repos"),
186 env,
187 });
188 if (code !== 0) throw new Error(out.slice(-600));
189 return jsonFrom(out)[0]?.results ?? [];
190}
191
192async function ours() {
193 const since = day(start);
194 const operations = await d1(
195 `SELECT substr(hour, 1, 10) AS day, SUM(operations) AS operations FROM git_operations WHERE hour >= '${since}' GROUP BY day ORDER BY day`,
196 );
197 let meters = [];
198 let mapping = [];
199 try {
200 meters = await d1(
201 `SELECT day, meter, SUM(count) AS count, SUM(bytes_in) AS bytes_in, SUM(bytes_out) AS bytes_out FROM artifacts_meters WHERE day >= '${since}'` +
202 (NAMESPACE ? ` AND store = '${NAMESPACE.replace(/'/g, "")}'` : "") +
203 " GROUP BY day, meter ORDER BY day, meter",
204 );
205 mapping = await d1("SELECT meter, cost_operations, billable_operations FROM operation_mapping ORDER BY meter");
206 } catch (error) {
207 console.error(`(artifacts_meters not readable yet: migration 0011 not applied? ${String(error.message).split("\n")[0]})`);
208 }
209 return { operations, meters, mapping };
210}
211
212/** Cloudflare's events against combinations of g1t's meters: which line up. */
213export function candidates(cfTotals, meterTotals) {
214 const sum = (names) => names.reduce((total, name) => total + (meterTotals[name] ?? 0), 0);
215 const options = {
216 pull: [
217 ["git.fetch"],
218 ["git.fetch", "internal.git.fetch"],
219 ["git.fetch", "internal.git.fetch", "git.ls_refs"],
220 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs"],
221 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs", "cache.info_refs", "cache.ls_refs"],
222 ],
223 push: [["git.receive_pack"], ["git.receive_pack", "internal.git.receive_pack"]],
224 create: [["binding.create"]],
225 fork: [["binding.fork"]],
226 delete: [["binding.delete"]],
227 };
228 const rows = [];
229 for (const [type, combos] of Object.entries(options)) {
230 const theirs = cfTotals[type] ?? 0;
231 for (const combo of combos) {
232 const mine = sum(combo);
233 rows.push({ type, cloudflare: theirs, meters: combo.join(" + "), g1t: mine, ratio: mine ? theirs / mine : null });
234 }
235 }
236 return rows;
237}
238
239const pad = (value, width) => String(value).padStart(width);
240
241async function main() {
242 const [events, mine] = await Promise.all([cloudflare(), ours()]);
243 const cfTotals = {};
244 const cfByDay = {};
245 for (const event of events) {
246 const key = event.kind === "error" ? `error:${event.type}` : event.type;
247 cfTotals[key] = (cfTotals[key] ?? 0) + event.count;
248 (cfByDay[event.day] ??= {})[key] = (cfByDay[event.day]?.[key] ?? 0) + event.count;
249 }
250 const meterTotals = {};
251 const meterByDay = {};
252 for (const row of mine.meters) {
253 meterTotals[row.meter] = (meterTotals[row.meter] ?? 0) + Number(row.count);
254 (meterByDay[row.day] ??= {})[row.meter] = Number(row.count);
255 }
256 const opsByDay = Object.fromEntries(mine.operations.map((row) => [row.day, Number(row.operations)]));
257 const lined = candidates(cfTotals, meterTotals);
258 if (asJson) {
259 console.log(
260 JSON.stringify(
261 { from: day(start), to: day(end), cloudflare: events, meters: mine.meters, git_operations: mine.operations, mapping: mine.mapping, candidates: lined },
262 null,
263 2,
264 ),
265 );
266 return;
267 }
268 console.log(`Artifacts usage ${day(start)} to ${day(end)}${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
269 const types = ["pull", "push", "create", "fork", "delete"];
270 console.log(
271 ["day ", ...types.map((t) => pad(`cf.${t}`, 10)), pad("cf.errors", 10), pad("g1t.fetch", 10), pad("g1t.push", 10), pad("g1t.ops", 10)].join(" "),
272 );
273 const allDays = [...new Set([...Object.keys(cfByDay), ...Object.keys(meterByDay), ...Object.keys(opsByDay)])].sort();
274 for (const d of allDays) {
275 const cf = cfByDay[d] ?? {};
276 const m = meterByDay[d] ?? {};
277 const errors = Object.entries(cf)
278 .filter(([key]) => key.startsWith("error:"))
279 .reduce((total, [, n]) => total + n, 0);
280 console.log(
281 [
282 d,
283 ...types.map((t) => pad(cf[t] ?? 0, 10)),
284 pad(errors, 10),
285 pad((m["git.fetch"] ?? 0) + (m["internal.git.fetch"] ?? 0), 10),
286 pad((m["git.receive_pack"] ?? 0) + (m["internal.git.receive_pack"] ?? 0), 10),
287 pad(opsByDay[d] ?? 0, 10),
288 ].join(" "),
289 );
290 }
291 console.log("\nCloudflare totals:", JSON.stringify(cfTotals));
292 console.log("g1t meter totals: ", JSON.stringify(meterTotals));
293 console.log("\nWhich g1t meters line up with each Cloudflare event (ratio = Cloudflare / g1t; 1.00 is a match):");
294 for (const row of lined) {
295 console.log(
296 ` ${row.type.padEnd(7)} ${pad(row.cloudflare, 8)} vs ${pad(row.g1t, 8)} ${row.ratio == null ? " n/a" : row.ratio.toFixed(2).padStart(5)} ${row.meters}`,
297 );
298 }
299 console.log("\nNow counting as operations (operation_mapping):");
300 for (const row of mine.mapping) console.log(` ${row.meter.padEnd(28)} cost ${row.cost_operations} billable ${row.billable_operations}`);
301 console.log(
302 "\nThe days before 2026-10-06's meters were deployed have Cloudflare's numbers only. Errors are Cloudflare's error events (rateLimited, serverError, ...).",
303 );
304}
305
306(hoursOf ? hourly(hoursOf) : main()).catch((error) => {
307 console.error(error.message);
308 process.exit(1);
309});