Sign in

flagon-io/g1t

public

g1t/scripts/ops/fork-storage-test.mjs

224 lines10,905 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1#!/usr/bin/env node
2// Do Artifacts forks copy their source's objects, or share them?
3//
4// Cloudflare does not document it (docs/ARTIFACTS.md, M2), and it decides
5// how much every pull request costs in storage. This makes a throwaway
6// repository holding ~100 MB that cannot be compressed, forks it five
7// times, and reads whatever storage figures Cloudflare reports before and
8// after, then deletes everything it made. Nothing of g1t's is touched: it
9// works in its own namespace (default `g1t-storage-test`), straight
10// against Cloudflare's API, never through g1t.sh.
11//
12// export CLOUDFLARE_API_TOKEN=<token: Artifacts edit, Account Analytics read>
13// node scripts/ops/fork-storage-test.mjs run # make, fork 5x, measure for 20 min, delete
14// node scripts/ops/fork-storage-test.mjs run --keep # ... and keep it, to measure again tomorrow
15// node scripts/ops/fork-storage-test.mjs measure # read the figures again (e.g. the next day)
16// node scripts/ops/fork-storage-test.mjs cleanup # delete the test repositories
17// node scripts/ops/fork-storage-test.mjs schema # list the analytics datasets Cloudflare offers for Artifacts
18//
19// Options: --namespace <name> (default g1t-storage-test), --mb <size> (100),
20// --forks <n> (5), --minutes <n> to keep measuring (20).
21//
22// How to read the result: docs/ARTIFACTS.md, "R2: the fork storage test".
23// Needs git on PATH. Costs a few cents of Artifacts storage and operations.
24
25import { execFileSync } from "node:child_process";
26import { randomBytes } from "node:crypto";
27import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
28import { tmpdir } from "node:os";
29import { join } from "node:path";
30
31const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "1e6f2cffa3f445920836e8ebe446bb58";
32const API = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}`;
33const args = process.argv.slice(2);
34const command = args[0] ?? "run";
35const option = (name, fallback) => {
36 const at = args.indexOf(name);
37 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
38};
39const NAMESPACE = option("--namespace", "g1t-storage-test");
40const MB = Number(option("--mb", "100"));
41const FORKS = Number(option("--forks", "5"));
42const MINUTES = Number(option("--minutes", "20"));
43const KEEP = args.includes("--keep");
44const SOURCE = "fork-test-source";
45const forkName = (n) => `fork-test-copy-${n}`;
46
47const token = process.env.CLOUDFLARE_API_TOKEN;
48if (!token) {
49 console.error("Set CLOUDFLARE_API_TOKEN (Artifacts edit, Account Analytics read).");
50 process.exit(2);
51}
52
53async function api(method, path, body) {
54 const response = await fetch(`${API}${path}`, {
55 method,
56 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
57 body: body ? JSON.stringify(body) : undefined,
58 });
59 const json = await response.json().catch(() => ({}));
60 return { status: response.status, ok: response.ok && json.success !== false, json };
61}
62
63async function graphql(query, variables = {}) {
64 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
65 method: "POST",
66 headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
67 body: JSON.stringify({ query, variables }),
68 });
69 const json = await response.json();
70 if (json.errors?.length) throw new Error(JSON.stringify(json.errors).slice(0, 500));
71 return json.data;
72}
73
74/** The account's analytics datasets about Artifacts, and their numeric fields. */
75async function datasets() {
76 const data = await graphql(`{ __type(name: "account") { fields { name type { name ofType { name ofType { name ofType { name } } } } } } }`);
77 const fields = data.__type?.fields ?? [];
78 const found = [];
79 for (const field of fields.filter((f) => /artifact/i.test(f.name))) {
80 let type = field.type;
81 while (type && !type.name) type = type.ofType;
82 while (type?.ofType) type = type.ofType;
83 const typeName = type?.name;
84 const shape = typeName ? await graphql(`{ __type(name: "${typeName}") { fields { name type { name kind ofType { name kind } } } } }`) : null;
85 found.push({ dataset: field.name, type: typeName, fields: (shape?.__type?.fields ?? []).map((f) => f.name) });
86 }
87 return found;
88}
89
90/** Every storage-looking figure Cloudflare reports for the test repositories. */
91async function storageFigures() {
92 const out = {};
93 const sets = await datasets();
94 const start = new Date(Date.now() - 2 * 24 * 3600 * 1000).toISOString();
95 const end = new Date().toISOString();
96 for (const set of sets) {
97 if (set.dataset === "artifactsEventsAdaptiveGroups") continue;
98 // A dataset other than events: ask for its sums, maxes and dimensions,
99 // filtered to this namespace where it can be.
100 for (const aggregate of ["max", "sum", "avg"]) {
101 if (!set.fields.includes(aggregate)) continue;
102 try {
103 const inner = await graphql(`{ __type(name: "${set.type}") { fields { name type { name ofType { name } } } } }`);
104 const aggregateType = inner.__type.fields.find((f) => f.name === aggregate)?.type;
105 const aggregateName = aggregateType?.name ?? aggregateType?.ofType?.name;
106 const numeric = aggregateName ? await graphql(`{ __type(name: "${aggregateName}") { fields { name } } }`) : null;
107 const names = (numeric?.__type?.fields ?? []).map((f) => f.name);
108 if (!names.length) continue;
109 const data = await graphql(
110 `query Q($account: String!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $account }) {
111 ${set.dataset}(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end }) {
112 ${aggregate} { ${names.join(" ")} } dimensions { repositoryNamespace repositoryName date }
113 } } } }`,
114 { account: ACCOUNT_ID, start, end },
115 );
116 const rows = data.viewer.accounts[0][set.dataset].filter((row) => !row.dimensions?.repositoryNamespace || row.dimensions.repositoryNamespace === NAMESPACE);
117 out[`${set.dataset}.${aggregate}`] = rows;
118 } catch (error) {
119 out[`${set.dataset}.${aggregate}`] = `not readable: ${String(error.message).slice(0, 200)}`;
120 }
121 }
122 }
123 // The events themselves: errors such as storageLimitReached, and pushes.
124 const events = await graphql(
125 `query Q($account: String!, $start: Time!, $end: Time!, $ns: String!) { viewer { accounts(filter: { accountTag: $account }) {
126 artifactsEventsAdaptiveGroups(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end, repositoryNamespace: $ns }) {
127 count sum { durationMs } dimensions { repositoryName eventKind eventType }
128 } } } }`,
129 { account: ACCOUNT_ID, start, end, ns: NAMESPACE },
130 );
131 out.events = events.viewer.accounts[0].artifactsEventsAdaptiveGroups;
132 return out;
133}
134
135async function repoInfo(name) {
136 const got = await api("GET", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
137 return got.ok ? got.json.result : null;
138}
139
140async function setup() {
141 const made = await api("POST", "/artifacts/namespaces", { namespace: NAMESPACE });
142 if (!made.ok && made.status !== 409) console.log(`namespace: ${made.status} ${JSON.stringify(made.json.errors ?? "")}`);
143 const created = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos`, { name: SOURCE, description: "g1t fork storage test; safe to delete" });
144 if (!created.ok) throw new Error(`create: ${created.status} ${JSON.stringify(created.json.errors ?? created.json)}`);
145 const { remote, token: repoToken } = created.json.result;
146 console.log(`made ${NAMESPACE}/${SOURCE}`);
147 // ~MB of random bytes, in files under the 32 MB limit, so nothing compresses.
148 const dir = mkdtempSync(join(tmpdir(), "g1t-fork-test-"));
149 try {
150 const git = (...a) => execFileSync("git", a, { cwd: dir, stdio: ["ignore", "pipe", "pipe"] }).toString();
151 git("init", "-q", "-b", "main");
152 git("config", "user.email", "fork-test@g1t.invalid");
153 git("config", "user.name", "g1t fork test");
154 const files = Math.ceil(MB / 25);
155 for (let n = 0; n < files; n++) writeFileSync(join(dir, `random-${n}.bin`), randomBytes(Math.min(25, MB - n * 25) * 1024 * 1024));
156 git("add", ".");
157 git("commit", "-q", "-m", "incompressible data");
158 const started = Date.now();
159 execFileSync("git", ["-c", `http.extraHeader=Authorization: Bearer ${repoToken}`, "push", "-q", remote, "main"], { cwd: dir, stdio: "inherit" });
160 console.log(`pushed ${MB} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`);
161 } finally {
162 rmSync(dir, { recursive: true, force: true });
163 }
164 const forks = [];
165 for (let n = 1; n <= FORKS; n++) {
166 const started = Date.now();
167 const forked = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos/${SOURCE}/fork`, { name: forkName(n), default_branch_only: true });
168 const ms = Date.now() - started;
169 forks.push({ name: forkName(n), status: forked.status, ms, result: forked.json.result ?? forked.json.errors });
170 console.log(`fork ${n}: ${forked.status} in ${ms} ms ${JSON.stringify(forked.json.result ?? forked.json.errors ?? {}).slice(0, 300)}`);
171 }
172 return forks;
173}
174
175async function cleanup() {
176 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) {
177 const deleted = await api("DELETE", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
178 console.log(`delete ${name}: ${deleted.status}`);
179 }
180 console.log(`The namespace ${NAMESPACE} is left, empty (the API has no call to delete one).`);
181}
182
183async function measure(label) {
184 const figures = await storageFigures();
185 const info = {};
186 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) info[name] = await repoInfo(name);
187 console.log(`\n== ${label} (${new Date().toISOString()}) ==`);
188 console.log(JSON.stringify({ figures, repos: info }, null, 2));
189 return figures;
190}
191
192async function main() {
193 if (command === "schema") {
194 console.log(JSON.stringify(await datasets(), null, 2));
195 return;
196 }
197 if (command === "cleanup") return cleanup();
198 if (command === "measure") {
199 await measure("now");
200 return;
201 }
202 if (command !== "run") throw new Error(`unknown command ${command}`);
203 console.log("Artifacts analytics datasets:", JSON.stringify((await datasets()).map((d) => d.dataset)));
204 await measure("before");
205 let forks;
206 try {
207 forks = await setup();
208 await measure("right after");
209 const until = Date.now() + MINUTES * 60 * 1000;
210 while (Date.now() < until) {
211 await new Promise((resolve) => setTimeout(resolve, 5 * 60 * 1000));
212 await measure(`after, ${Math.round((MINUTES * 60 * 1000 - (until - Date.now())) / 60000)} min`);
213 }
214 } finally {
215 if (KEEP) console.log(`\nKept. Run \`measure\` tomorrow, then \`cleanup\`.`);
216 else await cleanup();
217 }
218 console.log("\nForks:", JSON.stringify(forks, null, 2));
219}
220
221main().catch((error) => {
222 console.error(error.message);
223 process.exit(1);
224});