Sign in

flagon-io/g1t

public

g1t/scripts/ops/artifacts-namespaces.mjs

261 lines14,101 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Repositories shard across git store namespaces, move between them, and can keep to the EU; a namespace can be served read-only from the self-hosted git store, rebuilt from the nightly backups (#20, #25)1#!/usr/bin/env node
2// How each git store namespace stands (docs/ARTIFACTS.md, R7): what it
3// holds, how busy its busiest minute was against Cloudflare's limit of
4// 2,000 control-plane requests per 10 seconds, how it has been failing,
5// whether it takes new repositories, and its limits. Also queues and lists
6// moves of repositories between namespaces (services/repos/src/moves.rs).
7//
8// node scripts/ops/artifacts-namespaces.mjs # the report, as a table
9// node scripts/ops/artifacts-namespaces.mjs --json # the same, as JSON
10// node scripts/ops/artifacts-namespaces.mjs --cloudflare # with Cloudflare's own event counts per namespace
11// node scripts/ops/artifacts-namespaces.mjs moves # moves asked for, newest first
12// node scripts/ops/artifacts-namespaces.mjs move acme/rocket g1t-us-1 # queue one; the hourly sweep runs it
13//
14// The report and `moves` are read-only: SELECTs against the g1t-repos
15// database through Wrangler (as you are logged in, or CLOUDFLARE_D1_TOKEN),
16// and with --cloudflare one GraphQL query (CLOUDFLARE_API_TOKEN with Account
17// Analytics: Read). `move` inserts one row into repo_moves, nothing else.
18// What is configured is read from services/repos/wrangler.jsonc, so the
19// report says what the next deploy will do.
20
21import { readFileSync } from "node:fs";
22import { join } from "node:path";
23
24import { ACCOUNT_ID, cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs";
25import { ROOT, parseJsonc } from "../deploy/stack.mjs";
26
27const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js");
28const DATABASE = "g1t-repos";
29/** Cloudflare's control-plane limit for one namespace, per minute (shards.rs). */
30export const LIMIT_PER_MINUTE = 12_000;
31/** Past this share of it, the repos service stops placing new repositories there. */
32export const HOT_SHARE = 0.7;
33
34/** What services/repos/wrangler.jsonc configures: each namespace, its binding and jurisdiction, and the placement variables. */
35export function configured(wrangler) {
36 const vars = wrangler.vars ?? {};
37 let named = {};
38 try {
39 named = JSON.parse(vars.ARTIFACTS_NAMESPACES ?? "{}");
40 } catch {}
41 const bindings = new Map((wrangler.artifacts ?? []).map((one) => [one.binding, one]));
42 if (!named.ARTIFACTS) named.ARTIFACTS = "g1t";
43 let limits = {};
44 try {
45 limits = JSON.parse(vars.ARTIFACTS_NAMESPACE_LIMITS ?? "{}");
46 } catch {}
47 const newRepos = String(vars.ARTIFACTS_NEW_REPOS ?? "")
48 .split(",")
49 .map((name) => name.trim())
50 .filter(Boolean);
51 const eu = vars.ARTIFACTS_EU_NAMESPACE?.trim() || null;
52 return Object.entries(named).map(([binding, namespace]) => ({
53 namespace,
54 binding,
55 bound: bindings.has(binding) && bindings.get(binding).namespace === namespace,
56 // Set when the namespace is made, not in the binding: known with --cloudflare.
57 jurisdiction: null,
58 default: binding === "ARTIFACTS",
59 eu: eu === namespace,
60 takes_new_repos: newRepos.includes(namespace),
61 max_repos: limits[namespace]?.max_repos ?? null,
62 }));
63}
64
65/** A store key's namespace, as the registry keeps it: none means the default. */
66export function namespaceOf(store, defaultNamespace = "g1t") {
67 const at = (store ?? "").indexOf("/");
68 return at > 0 ? store.slice(0, at) : defaultNamespace;
69}
70
71/**
72 * Every namespace's standing, from what is configured, what the registry
73 * holds (`held`: ns, repos, forks, stored_bytes), how it answered
74 * (`health`: store, peak, calls, errors, rate_limited, rejected, the last
75 * hour and the last day), Cloudflare's own counts (`events`) and the
76 * namespaces Cloudflare has (`made`: namespace, jurisdiction), when asked.
77 */
78export function standings(config, held, health, events = [], made = null) {
79 const defaultNamespace = config.find((one) => one.default)?.namespace ?? "g1t";
80 const names = [...new Set([...config.map((one) => one.namespace), ...held.map((row) => row.ns || defaultNamespace)])];
81 return names.map((namespace) => {
82 const known = made?.find((one) => one.namespace === namespace);
83 const set = { ...(config.find((one) => one.namespace === namespace) ?? { namespace, binding: null, bound: false }) };
84 if (known) set.jurisdiction = known.jurisdiction ?? "any";
85 const holds = held.filter((row) => (row.ns || defaultNamespace) === namespace);
86 const sum = (rows, field) => rows.reduce((total, row) => total + Number(row[field] ?? 0), 0);
87 const hour = health.find((row) => row.store === namespace && row.window === "hour") ?? {};
88 const day = health.find((row) => row.store === namespace && row.window === "day") ?? {};
89 const fallback = health.find((row) => row.store === `${namespace}@fallback` && row.window === "hour");
90 const peak = Number(day.peak ?? 0);
91 const repos = sum(holds, "repos");
92 const warnings = [];
93 if (!set.bound && repos > 0) warnings.push("holds repositories but is not bound");
94 if (set.takes_new_repos && !set.bound) warnings.push("named in ARTIFACTS_NEW_REPOS but not bound: passed over");
95 if (peak >= LIMIT_PER_MINUTE * HOT_SHARE) warnings.push(`busiest minute at ${Math.round((peak / LIMIT_PER_MINUTE) * 100)}% of the limit`);
96 if (set.max_repos && repos >= set.max_repos) warnings.push("at its max_repos: takes no new repositories while another can");
97 if (Number(hour.rate_limited ?? 0) > 0) warnings.push(`${hour.rate_limited} calls rate limited in the last hour`);
98 if (made && set.binding && !known) warnings.push("named in ARTIFACTS_NAMESPACES, but Cloudflare has no namespace of this name: make it before deploying");
99 if (set.eu && known && known.jurisdiction !== "eu") warnings.push(`named as the EU namespace, but Cloudflare says its jurisdiction is ${known.jurisdiction ?? "unrestricted"}`);
100 if (fallback) warnings.push(`served from the fallback store lately (${fallback.calls} calls in the last hour)`);
101 return {
102 ...set,
103 repos,
104 forks: sum(holds, "forks"),
105 stored_bytes: sum(holds, "stored_bytes"),
106 peak_per_minute_day: peak,
107 peak_per_minute_hour: Number(hour.peak ?? 0),
108 peak_share: peak / LIMIT_PER_MINUTE,
109 calls_day: Number(day.calls ?? 0),
110 errors_day: Number(day.errors ?? 0),
111 rate_limited_day: Number(day.rate_limited ?? 0),
112 rejected_day: Number(day.rejected ?? 0),
113 cloudflare_events: events.filter((event) => event.namespace === namespace).reduce((total, event) => total + event.count, 0),
114 warnings,
115 };
116 });
117}
118
119const gb = (bytes) => `${(bytes / 1e9).toFixed(2)} GB`;
120const pct = (share) => `${(share * 100).toFixed(1)}%`;
121
122export function table(rows) {
123 const header = ["namespace", "binding", "where", "new", "repos", "forks", "stored", "peak/min (24h)", "of limit", "calls 24h", "errors", "429s"];
124 const lines = rows.map((row) => [
125 row.namespace + (row.default ? " *" : ""),
126 row.bound ? row.binding : `${row.binding ?? "-"} (not bound)`,
127 row.jurisdiction ?? "?",
128 row.takes_new_repos ? "yes" : row.eu ? "eu" : "no",
129 String(row.repos),
130 String(row.forks),
131 gb(row.stored_bytes),
132 String(row.peak_per_minute_day),
133 pct(row.peak_share),
134 String(row.calls_day),
135 String(row.errors_day),
136 String(row.rate_limited_day),
137 ]);
138 const widths = header.map((title, at) => Math.max(title.length, ...lines.map((line) => line[at].length)));
139 const format = (line) => line.map((cell, at) => cell.padEnd(widths[at])).join(" ");
140 const out = [format(header), format(widths.map((width) => "-".repeat(width))), ...lines.map(format)];
141 for (const row of rows) for (const warning of row.warnings) out.push(`! ${row.namespace}: ${warning}`);
142 out.push("* the default namespace: keys without a namespace are in it. Limit: 12,000 control-plane requests a minute per namespace.");
143 return out.join("\n");
144}
145
146// ---------------------------------------------------------------------
147
148async function d1(sql) {
149 const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
150 if (process.env.CLOUDFLARE_D1_TOKEN) env.CLOUDFLARE_API_TOKEN = process.env.CLOUDFLARE_D1_TOKEN;
151 const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
152 cwd: join(ROOT, "services/repos"),
153 env,
154 });
155 if (code !== 0) throw new Error(out.slice(-600));
156 return jsonFrom(out)[0]?.results ?? [];
157}
158
159const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`;
160const minuteAgo = (minutes) => new Date(Date.now() - minutes * 60_000).toISOString().slice(0, 16);
161
162async function readHeld() {
163 return d1(`SELECT CASE WHEN instr(coalesce(store, ''), '/') > 0 THEN substr(store, 1, instr(store, '/') - 1) ELSE '' END AS ns,
164 count(*) AS repos, sum(CASE WHEN fork_of IS NULL THEN 0 ELSE 1 END) AS forks, sum(coalesce(stored_bytes, 0)) AS stored_bytes
165 FROM repos WHERE deleted_at IS NULL AND retired_at IS NULL GROUP BY ns`);
166}
167
168async function readHealth() {
169 const window = (name, minutes) =>
170 `SELECT '${name}' AS window, store, max(calls) AS peak, sum(calls) AS calls, sum(errors) AS errors,
171 sum(rate_limited) AS rate_limited, sum(rejected) AS rejected
172 FROM store_health WHERE minute >= '${minuteAgo(minutes)}' GROUP BY store`;
173 return d1(`${window("hour", 60)} UNION ALL ${window("day", 24 * 60)}`);
174}
175
176async function readEvents() {
177 const auth = cloudflareAuth();
178 if (!auth) throw new Error("--cloudflare needs CLOUDFLARE_API_TOKEN with Account Analytics: Read");
179 const end = new Date();
180 const start = new Date(end.getTime() - 24 * 3600 * 1000);
181 const query = `query Q($accountTag: String!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $accountTag }) {
182 artifactsEventsAdaptiveGroups(limit: 10000, filter: { datetime_geq: $start, datetime_leq: $end }) { count dimensions { repositoryNamespace } } } } }`;
183 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
184 method: "POST",
185 headers: { ...auth, "content-type": "application/json" },
186 body: JSON.stringify({ query, variables: { accountTag: ACCOUNT_ID, start: start.toISOString(), end: end.toISOString() } }),
187 });
188 const body = await response.json();
189 if (body.errors?.length) throw new Error(`GraphQL: ${JSON.stringify(body.errors).slice(0, 400)}`);
190 return (body.data?.viewer?.accounts?.[0]?.artifactsEventsAdaptiveGroups ?? []).map((group) => ({
191 namespace: group.dimensions.repositoryNamespace,
192 count: group.count,
193 }));
194}
195
196/** The namespaces Cloudflare has, with their jurisdictions (CLOUDFLARE_API_TOKEN with Artifacts: Read). */
197async function readNamespaces() {
198 const auth = cloudflareAuth();
199 if (!auth) throw new Error("--cloudflare needs CLOUDFLARE_API_TOKEN");
200 const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/artifacts/namespaces`, { headers: auth });
201 const body = await response.json().catch(() => ({}));
202 if (!response.ok || body.success === false) throw new Error(`listing namespaces: ${response.status} ${JSON.stringify(body.errors ?? body).slice(0, 300)}`);
203 const list = Array.isArray(body.result) ? body.result : (body.result?.namespaces ?? []);
204 return list.map((one) => ({ namespace: one.namespace ?? one.name, jurisdiction: one.jurisdiction ?? null }));
205}
206
207function wranglerConfig() {
208 return parseJsonc(readFileSync(join(ROOT, "services/repos/wrangler.jsonc"), "utf8"));
209}
210
211async function main() {
212 const args = process.argv.slice(2);
213 const command = args[0] && !args[0].startsWith("--") ? args[0] : "report";
214
215 if (command === "moves") {
216 const rows = await d1(`SELECT m.*, r.namespace AS workspace, r.name FROM repo_moves m LEFT JOIN repos r ON r.id = m.repo_id
217 ORDER BY m.queued_ms DESC LIMIT 50`);
218 if (args.includes("--json")) console.log(JSON.stringify(rows, null, 2));
219 else for (const row of rows) console.log(`${row.id} ${row.status.padEnd(8)} ${row.workspace}/${row.name} -> ${row.to_namespace}${row.note ? ` (${row.note})` : ""}`);
220 return 0;
221 }
222
223 if (command === "move") {
224 const [path, namespace] = args.slice(1);
225 const [workspace, name] = String(path ?? "").toLowerCase().split("/");
226 if (!workspace || !name || !namespace) throw new Error("usage: move <workspace/name> <namespace>");
227 const config = configured(wranglerConfig());
228 if (!config.some((one) => one.namespace === namespace && one.bound)) throw new Error(`${namespace} is not a bound namespace in services/repos/wrangler.jsonc`);
229 const [repo] = await d1(`SELECT id, store FROM repos WHERE namespace = ${quoted(workspace)} AND name = ${quoted(name)} AND deleted_at IS NULL AND fork_of IS NULL`);
230 if (!repo) throw new Error(`no repository ${workspace}/${name}`);
231 if (namespaceOf(repo.store, config.find((one) => one.default)?.namespace) === namespace) throw new Error(`${workspace}/${name} is in ${namespace} already`);
232 const id = `mov_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
233 await d1(`INSERT INTO repo_moves (id, repo_id, to_namespace, status, requested_by, queued_ms)
234 VALUES (${quoted(id)}, ${quoted(repo.id)}, ${quoted(namespace)}, 'queued', 'scripts/ops/artifacts-namespaces.mjs', ${Date.now()})`);
235 console.log(`queued ${id}: ${workspace}/${name} (${repo.store}) -> ${namespace}. The hourly sweep (:23) moves it; watch with \`moves\`.`);
236 return 0;
237 }
238
239 const config = configured(wranglerConfig());
240 const cloudflare = args.includes("--cloudflare");
241 const [held, health, events, made] = await Promise.all([
242 readHeld(),
243 readHealth(),
244 cloudflare ? readEvents() : [],
245 cloudflare ? readNamespaces() : null,
246 ]);
247 const rows = standings(config, held, health, events, made);
248 if (args.includes("--json")) console.log(JSON.stringify(rows, null, 2));
249 else console.log(table(rows));
250 return rows.some((row) => row.warnings.length) ? 1 : 0;
251}
252
253if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/artifacts-namespaces.mjs")) {
254 main().then(
255 (code) => process.exit(code),
256 (error) => {
257 console.error(`namespaces: ${error.message}`);
258 process.exit(2);
259 },
260 );
261}