Skip to content
Sign up

flagon-io/g1t

public
0
290 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1#!/usr/bin/env node
2// What does Cloudflare count as an Artifacts "operation"? Compares what
3// Cloudflare's analytics say happened (GraphQL `artifactsEventsAdaptiveGroups`:
4// create, fork, push, pull, delete, and errors) with what g1t metered itself
5// (repos D1: `artifacts_meters`, every interaction by kind, and
6// `git_operations`, what workspaces are counted for), day by day, and says
7// which of g1t's meters line up with each of Cloudflare's events.
8//
9// Read-only: one GraphQL query, and SELECTs against the g1t-repos database.
10//
11// CLOUDFLARE_API_TOKEN=<token with Account Analytics: Read> \
12// node scripts/ops/artifacts-usage.mjs [--days 31] [--json]
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships13// node scripts/ops/artifacts-usage.mjs --hours 2026-10-07
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily14//
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships15// --hours DAY shows one UTC day hour by hour (Cloudflare's operations and
16// errors against `git_operations`), and the errors by message and repository:
17// a fix that lands mid-day is judged on the hours after it.
18//
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily19// The D1 queries run through Wrangler with the same environment (so the
20// token needs D1: Read too), or with CLOUDFLARE_D1_TOKEN when that is set,
21// or as you are logged in (`npx wrangler login`) when neither has it.
22
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results23import { ACCOUNT_ID, cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily24import { ROOT } from "../deploy/stack.mjs";
25import { join } from "node:path";
26
27const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
28const DATABASE = "g1t-repos";
29const NAMESPACE = process.env.ARTIFACTS_NAMESPACE || null;
30
31const args = process.argv.slice(2);
32const flag = (name) => args.includes(name);
33const option = (name, fallback) => {
34 const at = args.indexOf(name);
35 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
36};
37const days = Math.min(31, Math.max(1, Number(option("--days", "31")) || 31));
38const asJson = flag("--json");
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships39const hoursOf = option("--hours", null);
40if (hoursOf && !/^\d{4}-\d{2}-\d{2}$/.test(hoursOf)) {
41 console.error("--hours takes a UTC day, YYYY-MM-DD");
42 process.exit(2);
43}
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily44
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results45const auth = cloudflareAuth();
46if (!auth) {
47 console.error(
48 "Set CLOUDFLARE_API_TOKEN to a token with Account Analytics: Read on account " + ACCOUNT_ID +
49 ", or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.",
50 );
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily51 process.exit(2);
52}
53
54const end = new Date();
55const start = new Date(end.getTime() - days * 24 * 3600 * 1000);
56const day = (date) => date.toISOString().slice(0, 10);
57
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships58async function graphql(query, variables) {
59 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
60 method: "POST",
61 headers: { ...auth, "content-type": "application/json", "user-agent": "g1t-ops" },
62 body: JSON.stringify({ query, variables: { accountTag: ACCOUNT_ID, ...variables } }),
63 });
64 const body = await response.json();
65 if (!response.ok || body.errors?.length) {
66 throw new Error(`GraphQL: ${response.status} ${JSON.stringify(body.errors ?? body).slice(0, 600)}`);
67 }
68 return body.data?.viewer?.accounts?.[0] ?? {};
69}
70
71/** One UTC day by the hour: Cloudflare's operations and errors against `git_operations`. */
72async function hourly(dayText) {
73 const from = `${dayText}T00:00:00Z`;
74 const to = new Date(Date.parse(from) + 24 * 3600 * 1000).toISOString();
75 const nsFilter = NAMESPACE ? `, repositoryNamespace: "${NAMESPACE.replace(/"/g, "")}"` : "";
76 const query = `query ArtifactsHours($accountTag: String!, $start: Time!, $end: Time!) {
77 viewer {
78 accounts(filter: { accountTag: $accountTag }) {
79 hours: artifactsEventsAdaptiveGroups(
80 limit: 10000
81 filter: { datetime_geq: $start, datetime_lt: $end${nsFilter} }
82 orderBy: [datetimeHour_ASC]
83 ) { count dimensions { datetimeHour eventKind eventType } }
84 errors: artifactsEventsAdaptiveGroups(
85 limit: 10000
86 filter: { datetime_geq: $start, datetime_lt: $end, eventKind: "error"${nsFilter} }
87 orderBy: [count_DESC]
88 ) { count dimensions { eventType errorMessage repositoryName } }
89 }
90 }
91 }`;
92 const [account, operations] = await Promise.all([
93 graphql(query, { start: from, end: to }),
94 d1(
95 `SELECT substr(hour, 12, 2) AS h, SUM(operations) AS operations FROM git_operations WHERE hour >= '${dayText}T00' AND hour <= '${dayText}T23' GROUP BY h`,
96 ),
97 ]);
98 const ours = Object.fromEntries(operations.map((row) => [row.h, Number(row.operations)]));
99 const types = ["pull", "push", "create", "fork", "delete"];
100 const byHour = {};
101 for (const group of account.hours ?? []) {
102 const { datetimeHour, eventKind, eventType } = group.dimensions;
103 const key = eventKind === "error" ? "errors" : eventType;
104 if (key !== "errors" && !types.includes(key)) continue;
105 const h = datetimeHour.slice(11, 13);
106 (byHour[h] ??= {})[key] = (byHour[h][key] ?? 0) + group.count;
107 }
108 console.log(`Artifacts by the hour, ${dayText} UTC${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
109 console.log(["hour", ...types.map((t) => pad(`cf.${t}`, 9)), pad("cf.ops", 8), pad("g1t.ops", 8), pad("ratio", 6), pad("cf.errors", 10)].join(" "));
110 let cfTotal = 0;
111 let ourTotal = 0;
112 for (let i = 0; i < 24; i++) {
113 const h = String(i).padStart(2, "0");
114 const cf = byHour[h] ?? {};
115 const cfOps = types.reduce((total, t) => total + (cf[t] ?? 0), 0);
116 const mine = ours[h] ?? 0;
117 if (!cfOps && !mine && !cf.errors) continue;
118 cfTotal += cfOps;
119 ourTotal += mine;
120 console.log(
121 [h + " ", ...types.map((t) => pad(cf[t] ?? 0, 9)), pad(cfOps, 8), pad(mine, 8), pad(mine ? (cfOps / mine).toFixed(2) : "n/a", 6), pad(cf.errors ?? 0, 10)].join(" "),
122 );
123 }
124 console.log(`\nday cf.ops ${cfTotal}, g1t.ops ${ourTotal}${ourTotal ? `, ratio ${(cfTotal / ourTotal).toFixed(2)}` : ""}`);
125 const messages = {};
126 const repositories = {};
127 for (const group of account.errors ?? []) {
128 const { eventType, errorMessage, repositoryName } = group.dimensions;
129 const key = `${eventType}: ${errorMessage || "(no message)"}`;
130 messages[key] = (messages[key] ?? 0) + group.count;
131 repositories[repositoryName] = (repositories[repositoryName] ?? 0) + group.count;
132 }
133 console.log("\nErrors by message:");
134 for (const [message, count] of Object.entries(messages).sort((a, b) => b[1] - a[1])) console.log(` ${pad(count, 6)} ${message}`);
135 console.log("Errors by repository (top 8):");
136 for (const [name, count] of Object.entries(repositories).sort((a, b) => b[1] - a[1]).slice(0, 8)) console.log(` ${pad(count, 6)} ${name}`);
137 console.log(
138 "\ng1t.ops is what workspaces are counted for (billable meters only; nightly backups are g1t's own and not in it). An hour can straddle the two sides of a write by a few seconds.",
139 );
140}
141
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily142/** Cloudflare's own count, by day, event kind and type (and namespace). */
143async function cloudflare() {
144 const query = `query ArtifactsUsage($accountTag: String!, $start: Time!, $end: Time!) {
145 viewer {
146 accounts(filter: { accountTag: $accountTag }) {
147 artifactsEventsAdaptiveGroups(
148 limit: 10000
149 filter: { datetime_geq: $start, datetime_leq: $end }
150 orderBy: [date_ASC]
151 ) {
152 count
153 sum { durationMs }
154 dimensions { date eventKind eventType repositoryNamespace }
155 }
156 }
157 }
158 }`;
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships159 const account = await graphql(query, { start: start.toISOString(), end: end.toISOString() });
160 const groups = account.artifactsEventsAdaptiveGroups ?? [];
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily161 return groups
162 .filter((group) => !NAMESPACE || group.dimensions.repositoryNamespace === NAMESPACE)
163 .map((group) => ({
164 day: group.dimensions.date,
165 kind: group.dimensions.eventKind,
166 type: group.dimensions.eventType,
167 namespace: group.dimensions.repositoryNamespace,
168 count: group.count,
169 ms: group.sum?.durationMs ?? 0,
170 }));
171}
172
173/** A read-only query against the repos database. */
174async function d1(sql) {
175 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
176 if (process.env.CLOUDFLARE_D1_TOKEN) env.CLOUDFLARE_API_TOKEN = process.env.CLOUDFLARE_D1_TOKEN;
177 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
178 cwd: join(ROOT, "services/repos"),
179 env,
180 });
181 if (code !== 0) throw new Error(out.slice(-600));
182 return jsonFrom(out)[0]?.results ?? [];
183}
184
185async function ours() {
186 const since = day(start);
187 const operations = await d1(
188 `SELECT substr(hour, 1, 10) AS day, SUM(operations) AS operations FROM git_operations WHERE hour >= '${since}' GROUP BY day ORDER BY day`,
189 );
190 let meters = [];
191 let mapping = [];
192 try {
193 meters = await d1(
194 `SELECT day, meter, SUM(count) AS count, SUM(bytes_in) AS bytes_in, SUM(bytes_out) AS bytes_out FROM artifacts_meters WHERE day >= '${since}'` +
195 (NAMESPACE ? ` AND store = '${NAMESPACE.replace(/'/g, "")}'` : "") +
196 " GROUP BY day, meter ORDER BY day, meter",
197 );
198 mapping = await d1("SELECT meter, cost_operations, billable_operations FROM operation_mapping ORDER BY meter");
199 } catch (error) {
200 console.error(`(artifacts_meters not readable yet: migration 0011 not applied? ${String(error.message).split("\n")[0]})`);
201 }
202 return { operations, meters, mapping };
203}
204
205/** Cloudflare's events against combinations of g1t's meters: which line up. */
206export function candidates(cfTotals, meterTotals) {
207 const sum = (names) => names.reduce((total, name) => total + (meterTotals[name] ?? 0), 0);
208 const options = {
209 pull: [
210 ["git.fetch"],
211 ["git.fetch", "internal.git.fetch"],
212 ["git.fetch", "internal.git.fetch", "git.ls_refs"],
213 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs"],
214 ["git.fetch", "internal.git.fetch", "git.ls_refs", "git.info_refs", "internal.git.info_refs", "cache.info_refs", "cache.ls_refs"],
215 ],
216 push: [["git.receive_pack"], ["git.receive_pack", "internal.git.receive_pack"]],
217 create: [["binding.create"]],
218 fork: [["binding.fork"]],
219 delete: [["binding.delete"]],
220 };
221 const rows = [];
222 for (const [type, combos] of Object.entries(options)) {
223 const theirs = cfTotals[type] ?? 0;
224 for (const combo of combos) {
225 const mine = sum(combo);
226 rows.push({ type, cloudflare: theirs, meters: combo.join(" + "), g1t: mine, ratio: mine ? theirs / mine : null });
227 }
228 }
229 return rows;
230}
231
232const pad = (value, width) => String(value).padStart(width);
233
234async function main() {
235 const [events, mine] = await Promise.all([cloudflare(), ours()]);
236 const cfTotals = {};
237 const cfByDay = {};
238 for (const event of events) {
239 const key = event.kind === "error" ? `error:${event.type}` : event.type;
240 cfTotals[key] = (cfTotals[key] ?? 0) + event.count;
241 (cfByDay[event.day] ??= {})[key] = (cfByDay[event.day]?.[key] ?? 0) + event.count;
242 }
243 const meterTotals = {};
244 const meterByDay = {};
245 for (const row of mine.meters) {
246 meterTotals[row.meter] = (meterTotals[row.meter] ?? 0) + Number(row.count);
247 (meterByDay[row.day] ??= {})[row.meter] = Number(row.count);
248 }
249 const opsByDay = Object.fromEntries(mine.operations.map((row) => [row.day, Number(row.operations)]));
250 const lined = candidates(cfTotals, meterTotals);
251 if (asJson) {
252 console.log(JSON.stringify({ from: day(start), to: day(end), cloudflare: events, meters: mine.meters, git_operations: mine.operations, mapping: mine.mapping, candidates: lined }, null, 2));
253 return;
254 }
255 console.log(`Artifacts usage ${day(start)} to ${day(end)}${NAMESPACE ? ` (namespace ${NAMESPACE})` : ""}\n`);
256 const types = ["pull", "push", "create", "fork", "delete"];
257 console.log(["day ", ...types.map((t) => pad(`cf.${t}`, 10)), pad("cf.errors", 10), pad("g1t.fetch", 10), pad("g1t.push", 10), pad("g1t.ops", 10)].join(" "));
258 const allDays = [...new Set([...Object.keys(cfByDay), ...Object.keys(meterByDay), ...Object.keys(opsByDay)])].sort();
259 for (const d of allDays) {
260 const cf = cfByDay[d] ?? {};
261 const m = meterByDay[d] ?? {};
262 const errors = Object.entries(cf).filter(([key]) => key.startsWith("error:")).reduce((total, [, n]) => total + n, 0);
263 console.log(
264 [
265 d,
266 ...types.map((t) => pad(cf[t] ?? 0, 10)),
267 pad(errors, 10),
268 pad((m["git.fetch"] ?? 0) + (m["internal.git.fetch"] ?? 0), 10),
269 pad((m["git.receive_pack"] ?? 0) + (m["internal.git.receive_pack"] ?? 0), 10),
270 pad(opsByDay[d] ?? 0, 10),
271 ].join(" "),
272 );
273 }
274 console.log("\nCloudflare totals:", JSON.stringify(cfTotals));
275 console.log("g1t meter totals: ", JSON.stringify(meterTotals));
276 console.log("\nWhich g1t meters line up with each Cloudflare event (ratio = Cloudflare / g1t; 1.00 is a match):");
277 for (const row of lined) {
278 console.log(` ${row.type.padEnd(7)} ${pad(row.cloudflare, 8)} vs ${pad(row.g1t, 8)} ${row.ratio == null ? " n/a" : row.ratio.toFixed(2).padStart(5)} ${row.meters}`);
279 }
280 console.log("\nNow counting as operations (operation_mapping):");
281 for (const row of mine.mapping) console.log(` ${row.meter.padEnd(28)} cost ${row.cost_operations} billable ${row.billable_operations}`);
282 console.log(
283 "\nThe days before 2026-10-06's meters were deployed have Cloudflare's numbers only. Errors are Cloudflare's error events (rateLimited, serverError, ...).",
284 );
285}
286
Artifacts: an hourly view of operations and errors, and SSH must meter its git before it ships287(hoursOf ? hourly(hoursOf) : main()).catch((error) => {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily288 console.error(error.message);
289 process.exit(1);
290});