| 1 | #!/usr/bin/env node |
| 2 | // How each git store namespace stands: what it |
| 3 | // holds, how busy its busiest minute was against Cloudflare's limit of |
| 4 | // 2,000 control-plane requests per 10 seconds, how it has been failing, |
| 5 | // whether it takes new repositories, and its limits. Also queues and lists |
| 6 | // moves of repositories between namespaces (services/repos/src/moves.rs). |
| 7 | // |
| 8 | // node scripts/ops/gitstore-namespaces.mjs # the report, as a table |
| 9 | // node scripts/ops/gitstore-namespaces.mjs --json # the same, as JSON |
| 10 | // node scripts/ops/gitstore-namespaces.mjs --cloudflare # with Cloudflare's own event counts per namespace |
| 11 | // node scripts/ops/gitstore-namespaces.mjs moves # moves asked for, newest first |
| 12 | // node scripts/ops/gitstore-namespaces.mjs move acme/rocket g1t-us-1 # queue one; the hourly sweep runs it |
| 13 | // |
| 14 | // The report and `moves` are read-only: SELECTs against the g1t-repos |
| 15 | // database through Wrangler (as you are logged in, or CLOUDFLARE_D1_TOKEN), |
| 16 | // and with --cloudflare one GraphQL query (CLOUDFLARE_API_TOKEN with Account |
| 17 | // Analytics: Read). `move` inserts one row into repo_moves, nothing else. |
| 18 | // What is configured is read from services/repos/wrangler.jsonc, so the |
| 19 | // report says what the next deploy will do. |
| 20 | |
| 21 | import { readFileSync } from "node:fs"; |
| 22 | import { join } from "node:path"; |
| 23 | |
| 24 | import { ACCOUNT_ID, cloudflareAuth, exec, jsonFrom, wranglerEnv } from "../deploy/cloudflare.mjs"; |
| 25 | import { ROOT, parseJsonc } from "../deploy/stack.mjs"; |
| 26 | |
| 27 | const WRANGLER = join(ROOT, "node_modules/wrangler/bin/wrangler.js"); |
| 28 | const DATABASE = "g1t-repos"; |
| 29 | /** Cloudflare's control-plane limit for one namespace, per minute (shards.rs). */ |
| 30 | export const LIMIT_PER_MINUTE = 12_000; |
| 31 | /** Past this share of it, the repos service stops placing new repositories there. */ |
| 32 | export const HOT_SHARE = 0.7; |
| 33 | |
| 34 | /** What services/repos/wrangler.jsonc configures: each namespace, its binding and jurisdiction, and the placement variables. */ |
| 35 | export function configured(wrangler) { |
| 36 | const vars = wrangler.vars ?? {}; |
| 37 | let named = {}; |
| 38 | try { |
| 39 | named = JSON.parse(vars.GITSTORE_NAMESPACES ?? "{}"); |
| 40 | } catch {} |
| 41 | const bindings = new Map((wrangler.artifacts ?? []).map((one) => [one.binding, one])); |
| 42 | if (!named.GITSTORE) named.GITSTORE = "g1t"; |
| 43 | let limits = {}; |
| 44 | try { |
| 45 | limits = JSON.parse(vars.GITSTORE_NAMESPACE_LIMITS ?? "{}"); |
| 46 | } catch {} |
| 47 | const newRepos = String(vars.GITSTORE_NEW_REPOS ?? "") |
| 48 | .split(",") |
| 49 | .map((name) => name.trim()) |
| 50 | .filter(Boolean); |
| 51 | const eu = vars.GITSTORE_EU_NAMESPACE?.trim() || null; |
| 52 | return Object.entries(named).map(([binding, namespace]) => ({ |
| 53 | namespace, |
| 54 | binding, |
| 55 | bound: bindings.has(binding) && bindings.get(binding).namespace === namespace, |
| 56 | // Set when the namespace is made, not in the binding: known with --cloudflare. |
| 57 | jurisdiction: null, |
| 58 | default: binding === "GITSTORE", |
| 59 | eu: eu === namespace, |
| 60 | takes_new_repos: newRepos.includes(namespace), |
| 61 | max_repos: limits[namespace]?.max_repos ?? null, |
| 62 | })); |
| 63 | } |
| 64 | |
| 65 | /** A store key's namespace, as the registry keeps it: none means the default. */ |
| 66 | export function namespaceOf(store, defaultNamespace = "g1t") { |
| 67 | const at = (store ?? "").indexOf("/"); |
| 68 | return at > 0 ? store.slice(0, at) : defaultNamespace; |
| 69 | } |
| 70 | |
| 71 | /** |
| 72 | * Every namespace's standing, from what is configured, what the registry |
| 73 | * holds (`held`: ns, repos, forks, stored_bytes), how it answered |
| 74 | * (`health`: store, peak, calls, errors, rate_limited, rejected, the last |
| 75 | * hour and the last day), Cloudflare's own counts (`events`) and the |
| 76 | * namespaces Cloudflare has (`made`: namespace, jurisdiction), when asked. |
| 77 | */ |
| 78 | export function standings(config, held, health, events = [], made = null) { |
| 79 | const defaultNamespace = config.find((one) => one.default)?.namespace ?? "g1t"; |
| 80 | const names = [...new Set([...config.map((one) => one.namespace), ...held.map((row) => row.ns || defaultNamespace)])]; |
| 81 | return names.map((namespace) => { |
| 82 | const known = made?.find((one) => one.namespace === namespace); |
| 83 | const set = { ...(config.find((one) => one.namespace === namespace) ?? { namespace, binding: null, bound: false }) }; |
| 84 | if (known) set.jurisdiction = known.jurisdiction ?? "any"; |
| 85 | const holds = held.filter((row) => (row.ns || defaultNamespace) === namespace); |
| 86 | const sum = (rows, field) => rows.reduce((total, row) => total + Number(row[field] ?? 0), 0); |
| 87 | const hour = health.find((row) => row.store === namespace && row.window === "hour") ?? {}; |
| 88 | const day = health.find((row) => row.store === namespace && row.window === "day") ?? {}; |
| 89 | const fallback = health.find((row) => row.store === `${namespace}@fallback` && row.window === "hour"); |
| 90 | const peak = Number(day.peak ?? 0); |
| 91 | const repos = sum(holds, "repos"); |
| 92 | const warnings = []; |
| 93 | if (!set.bound && repos > 0) warnings.push("holds repositories but is not bound"); |
| 94 | if (set.takes_new_repos && !set.bound) warnings.push("named in GITSTORE_NEW_REPOS but not bound: passed over"); |
| 95 | if (peak >= LIMIT_PER_MINUTE * HOT_SHARE) warnings.push(`busiest minute at ${Math.round((peak / LIMIT_PER_MINUTE) * 100)}% of the limit`); |
| 96 | if (set.max_repos && repos >= set.max_repos) warnings.push("at its max_repos: takes no new repositories while another can"); |
| 97 | if (Number(hour.rate_limited ?? 0) > 0) warnings.push(`${hour.rate_limited} calls rate limited in the last hour`); |
| 98 | if (made && set.binding && !known) warnings.push("named in GITSTORE_NAMESPACES, but Cloudflare has no namespace of this name: make it before deploying"); |
| 99 | if (set.eu && known && known.jurisdiction !== "eu") |
| 100 | warnings.push(`named as the EU namespace, but Cloudflare says its jurisdiction is ${known.jurisdiction ?? "unrestricted"}`); |
| 101 | if (fallback) warnings.push(`served from the fallback store lately (${fallback.calls} calls in the last hour)`); |
| 102 | return { |
| 103 | ...set, |
| 104 | repos, |
| 105 | forks: sum(holds, "forks"), |
| 106 | stored_bytes: sum(holds, "stored_bytes"), |
| 107 | peak_per_minute_day: peak, |
| 108 | peak_per_minute_hour: Number(hour.peak ?? 0), |
| 109 | peak_share: peak / LIMIT_PER_MINUTE, |
| 110 | calls_day: Number(day.calls ?? 0), |
| 111 | errors_day: Number(day.errors ?? 0), |
| 112 | rate_limited_day: Number(day.rate_limited ?? 0), |
| 113 | rejected_day: Number(day.rejected ?? 0), |
| 114 | cloudflare_events: events.filter((event) => event.namespace === namespace).reduce((total, event) => total + event.count, 0), |
| 115 | warnings, |
| 116 | }; |
| 117 | }); |
| 118 | } |
| 119 | |
| 120 | const gb = (bytes) => `${(bytes / 1e9).toFixed(2)} GB`; |
| 121 | const pct = (share) => `${(share * 100).toFixed(1)}%`; |
| 122 | |
| 123 | export function table(rows) { |
| 124 | const header = ["namespace", "binding", "where", "new", "repos", "forks", "stored", "peak/min (24h)", "of limit", "calls 24h", "errors", "429s"]; |
| 125 | const lines = rows.map((row) => [ |
| 126 | row.namespace + (row.default ? " *" : ""), |
| 127 | row.bound ? row.binding : `${row.binding ?? "-"} (not bound)`, |
| 128 | row.jurisdiction ?? "?", |
| 129 | row.takes_new_repos ? "yes" : row.eu ? "eu" : "no", |
| 130 | String(row.repos), |
| 131 | String(row.forks), |
| 132 | gb(row.stored_bytes), |
| 133 | String(row.peak_per_minute_day), |
| 134 | pct(row.peak_share), |
| 135 | String(row.calls_day), |
| 136 | String(row.errors_day), |
| 137 | String(row.rate_limited_day), |
| 138 | ]); |
| 139 | const widths = header.map((title, at) => Math.max(title.length, ...lines.map((line) => line[at].length))); |
| 140 | const format = (line) => line.map((cell, at) => cell.padEnd(widths[at])).join(" "); |
| 141 | const out = [format(header), format(widths.map((width) => "-".repeat(width))), ...lines.map(format)]; |
| 142 | for (const row of rows) for (const warning of row.warnings) out.push(`! ${row.namespace}: ${warning}`); |
| 143 | out.push("* the default namespace: keys without a namespace are in it. Limit: 12,000 control-plane requests a minute per namespace."); |
| 144 | return out.join("\n"); |
| 145 | } |
| 146 | |
| 147 | // --------------------------------------------------------------------- |
| 148 | |
| 149 | async function d1(sql) { |
| 150 | const env = { ...wranglerEnv({ ...process.env, CI: "true" }) }; |
| 151 | if (process.env.CLOUDFLARE_D1_TOKEN) env.CLOUDFLARE_API_TOKEN = process.env.CLOUDFLARE_D1_TOKEN; |
| 152 | const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], { |
| 153 | cwd: join(ROOT, "services/repos"), |
| 154 | env, |
| 155 | }); |
| 156 | if (code !== 0) throw new Error(out.slice(-600)); |
| 157 | return jsonFrom(out)[0]?.results ?? []; |
| 158 | } |
| 159 | |
| 160 | const quoted = (text) => `'${String(text).replaceAll("'", "''")}'`; |
| 161 | const minuteAgo = (minutes) => new Date(Date.now() - minutes * 60_000).toISOString().slice(0, 16); |
| 162 | |
| 163 | async function readHeld() { |
| 164 | return d1(`SELECT CASE WHEN instr(coalesce(store, ''), '/') > 0 THEN substr(store, 1, instr(store, '/') - 1) ELSE '' END AS ns, |
| 165 | count(*) AS repos, sum(CASE WHEN fork_of IS NULL THEN 0 ELSE 1 END) AS forks, sum(coalesce(stored_bytes, 0)) AS stored_bytes |
| 166 | FROM repos WHERE deleted_at IS NULL AND retired_at IS NULL GROUP BY ns`); |
| 167 | } |
| 168 | |
| 169 | async function readHealth() { |
| 170 | const window = (name, minutes) => |
| 171 | `SELECT '${name}' AS window, store, max(calls) AS peak, sum(calls) AS calls, sum(errors) AS errors, |
| 172 | sum(rate_limited) AS rate_limited, sum(rejected) AS rejected |
| 173 | FROM store_health WHERE minute >= '${minuteAgo(minutes)}' GROUP BY store`; |
| 174 | return d1(`${window("hour", 60)} UNION ALL ${window("day", 24 * 60)}`); |
| 175 | } |
| 176 | |
| 177 | async function readEvents() { |
| 178 | const auth = cloudflareAuth(); |
| 179 | if (!auth) throw new Error("--cloudflare needs CLOUDFLARE_API_TOKEN with Account Analytics: Read"); |
| 180 | const end = new Date(); |
| 181 | const start = new Date(end.getTime() - 24 * 3600 * 1000); |
| 182 | const query = `query Q($accountTag: String!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $accountTag }) { |
| 183 | artifactsEventsAdaptiveGroups(limit: 10000, filter: { datetime_geq: $start, datetime_leq: $end }) { count dimensions { repositoryNamespace } } } } }`; |
| 184 | const response = await fetch("https://api.cloudflare.com/client/v4/graphql", { |
| 185 | method: "POST", |
| 186 | headers: { ...auth, "content-type": "application/json" }, |
| 187 | body: JSON.stringify({ query, variables: { accountTag: ACCOUNT_ID, start: start.toISOString(), end: end.toISOString() } }), |
| 188 | }); |
| 189 | const body = await response.json(); |
| 190 | if (body.errors?.length) throw new Error(`GraphQL: ${JSON.stringify(body.errors).slice(0, 400)}`); |
| 191 | return (body.data?.viewer?.accounts?.[0]?.artifactsEventsAdaptiveGroups ?? []).map((group) => ({ |
| 192 | namespace: group.dimensions.repositoryNamespace, |
| 193 | count: group.count, |
| 194 | })); |
| 195 | } |
| 196 | |
| 197 | /** The namespaces Cloudflare has, with their jurisdictions (CLOUDFLARE_API_TOKEN with Artifacts: Read). */ |
| 198 | async function readNamespaces() { |
| 199 | const auth = cloudflareAuth(); |
| 200 | if (!auth) throw new Error("--cloudflare needs CLOUDFLARE_API_TOKEN"); |
| 201 | const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/artifacts/namespaces`, { headers: auth }); |
| 202 | const body = await response.json().catch(() => ({})); |
| 203 | if (!response.ok || body.success === false) throw new Error(`listing namespaces: ${response.status} ${JSON.stringify(body.errors ?? body).slice(0, 300)}`); |
| 204 | const list = Array.isArray(body.result) ? body.result : (body.result?.namespaces ?? []); |
| 205 | return list.map((one) => ({ namespace: one.namespace ?? one.name, jurisdiction: one.jurisdiction ?? null })); |
| 206 | } |
| 207 | |
| 208 | function wranglerConfig() { |
| 209 | return parseJsonc(readFileSync(join(ROOT, "services/repos/wrangler.jsonc"), "utf8")); |
| 210 | } |
| 211 | |
| 212 | async function main() { |
| 213 | const args = process.argv.slice(2); |
| 214 | const command = args[0] && !args[0].startsWith("--") ? args[0] : "report"; |
| 215 | |
| 216 | if (command === "moves") { |
| 217 | const rows = await d1(`SELECT m.*, r.namespace AS workspace, r.name FROM repo_moves m LEFT JOIN repos r ON r.id = m.repo_id |
| 218 | ORDER BY m.queued_ms DESC LIMIT 50`); |
| 219 | if (args.includes("--json")) console.log(JSON.stringify(rows, null, 2)); |
| 220 | else |
| 221 | for (const row of rows) |
| 222 | console.log(`${row.id} ${row.status.padEnd(8)} ${row.workspace}/${row.name} -> ${row.to_namespace}${row.note ? ` (${row.note})` : ""}`); |
| 223 | return 0; |
| 224 | } |
| 225 | |
| 226 | if (command === "move") { |
| 227 | const [path, namespace] = args.slice(1); |
| 228 | const [workspace, name] = String(path ?? "") |
| 229 | .toLowerCase() |
| 230 | .split("/"); |
| 231 | if (!workspace || !name || !namespace) throw new Error("usage: move <workspace/name> <namespace>"); |
| 232 | const config = configured(wranglerConfig()); |
| 233 | if (!config.some((one) => one.namespace === namespace && one.bound)) |
| 234 | throw new Error(`${namespace} is not a bound namespace in services/repos/wrangler.jsonc`); |
| 235 | const [repo] = await d1( |
| 236 | `SELECT id, store FROM repos WHERE namespace = ${quoted(workspace)} AND name = ${quoted(name)} AND deleted_at IS NULL AND fork_of IS NULL`, |
| 237 | ); |
| 238 | if (!repo) throw new Error(`no repository ${workspace}/${name}`); |
| 239 | if (namespaceOf(repo.store, config.find((one) => one.default)?.namespace) === namespace) throw new Error(`${workspace}/${name} is in ${namespace} already`); |
| 240 | const id = `mov_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; |
| 241 | await d1(`INSERT INTO repo_moves (id, repo_id, to_namespace, status, requested_by, queued_ms) |
| 242 | VALUES (${quoted(id)}, ${quoted(repo.id)}, ${quoted(namespace)}, 'queued', 'scripts/ops/gitstore-namespaces.mjs', ${Date.now()})`); |
| 243 | console.log(`queued ${id}: ${workspace}/${name} (${repo.store}) -> ${namespace}. The hourly sweep (:23) moves it; watch with \`moves\`.`); |
| 244 | return 0; |
| 245 | } |
| 246 | |
| 247 | const config = configured(wranglerConfig()); |
| 248 | const cloudflare = args.includes("--cloudflare"); |
| 249 | const [held, health, events, made] = await Promise.all([readHeld(), readHealth(), cloudflare ? readEvents() : [], cloudflare ? readNamespaces() : null]); |
| 250 | const rows = standings(config, held, health, events, made); |
| 251 | if (args.includes("--json")) console.log(JSON.stringify(rows, null, 2)); |
| 252 | else console.log(table(rows)); |
| 253 | return rows.some((row) => row.warnings.length) ? 1 : 0; |
| 254 | } |
| 255 | |
| 256 | if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/ops/gitstore-namespaces.mjs")) { |
| 257 | main().then( |
| 258 | (code) => process.exit(code), |
| 259 | (error) => { |
| 260 | console.error(`namespaces: ${error.message}`); |
| 261 | process.exit(2); |
| 262 | }, |
| 263 | ); |
| 264 | } |