Sign in

flagon-io/g1t

public

g1t/scripts/ops/fork-storage-test.mjs

227 lines11,010 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1#!/usr/bin/env node
2// Do Artifacts forks copy their source's objects, or share them?
3//
4// Cloudflare does not document it (docs/ARTIFACTS.md, M2), and it decides
5// how much every pull request costs in storage. This makes a throwaway
6// repository holding ~100 MB that cannot be compressed, forks it five
7// times, and reads whatever storage figures Cloudflare reports before and
8// after, then deletes everything it made. Nothing of g1t's is touched: it
9// works in its own namespace (default `g1t-storage-test`), straight
10// against Cloudflare's API, never through g1t.sh.
11//
12// export CLOUDFLARE_API_TOKEN=<token: Artifacts edit, Account Analytics read>
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results13// (or a global API key: CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL)
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily14// node scripts/ops/fork-storage-test.mjs run # make, fork 5x, measure for 20 min, delete
15// node scripts/ops/fork-storage-test.mjs run --keep # ... and keep it, to measure again tomorrow
16// node scripts/ops/fork-storage-test.mjs measure # read the figures again (e.g. the next day)
17// node scripts/ops/fork-storage-test.mjs cleanup # delete the test repositories
18// node scripts/ops/fork-storage-test.mjs schema # list the analytics datasets Cloudflare offers for Artifacts
19//
20// Options: --namespace <name> (default g1t-storage-test), --mb <size> (100),
21// --forks <n> (5), --minutes <n> to keep measuring (20).
22//
23// How to read the result: docs/ARTIFACTS.md, "R2: the fork storage test".
24// Needs git on PATH. Costs a few cents of Artifacts storage and operations.
25
26import { execFileSync } from "node:child_process";
27import { randomBytes } from "node:crypto";
28import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
29import { tmpdir } from "node:os";
30import { join } from "node:path";
31
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results32import { cloudflareAuth } from "../deploy/cloudflare.mjs";
33
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily34const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "1e6f2cffa3f445920836e8ebe446bb58";
35const API = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}`;
36const args = process.argv.slice(2);
37const command = args[0] ?? "run";
38const option = (name, fallback) => {
39 const at = args.indexOf(name);
40 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
41};
42const NAMESPACE = option("--namespace", "g1t-storage-test");
43const MB = Number(option("--mb", "100"));
44const FORKS = Number(option("--forks", "5"));
45const MINUTES = Number(option("--minutes", "20"));
46const KEEP = args.includes("--keep");
47const SOURCE = "fork-test-source";
48const forkName = (n) => `fork-test-copy-${n}`;
49
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results50const auth = cloudflareAuth();
51if (!auth) {
52 console.error("Set CLOUDFLARE_API_TOKEN (Artifacts edit, Account Analytics read), or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.");
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily53 process.exit(2);
54}
55
56async function api(method, path, body) {
57 const response = await fetch(`${API}${path}`, {
58 method,
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results59 headers: { ...auth, "content-type": "application/json" },
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily60 body: body ? JSON.stringify(body) : undefined,
61 });
62 const json = await response.json().catch(() => ({}));
63 return { status: response.status, ok: response.ok && json.success !== false, json };
64}
65
66async function graphql(query, variables = {}) {
67 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
68 method: "POST",
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results69 headers: { ...auth, "content-type": "application/json" },
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily70 body: JSON.stringify({ query, variables }),
71 });
72 const json = await response.json();
73 if (json.errors?.length) throw new Error(JSON.stringify(json.errors).slice(0, 500));
74 return json.data;
75}
76
77/** The account's analytics datasets about Artifacts, and their numeric fields. */
78async function datasets() {
79 const data = await graphql(`{ __type(name: "account") { fields { name type { name ofType { name ofType { name ofType { name } } } } } } }`);
80 const fields = data.__type?.fields ?? [];
81 const found = [];
82 for (const field of fields.filter((f) => /artifact/i.test(f.name))) {
83 let type = field.type;
84 while (type && !type.name) type = type.ofType;
85 while (type?.ofType) type = type.ofType;
86 const typeName = type?.name;
87 const shape = typeName ? await graphql(`{ __type(name: "${typeName}") { fields { name type { name kind ofType { name kind } } } } }`) : null;
88 found.push({ dataset: field.name, type: typeName, fields: (shape?.__type?.fields ?? []).map((f) => f.name) });
89 }
90 return found;
91}
92
93/** Every storage-looking figure Cloudflare reports for the test repositories. */
94async function storageFigures() {
95 const out = {};
96 const sets = await datasets();
97 const start = new Date(Date.now() - 2 * 24 * 3600 * 1000).toISOString();
98 const end = new Date().toISOString();
99 for (const set of sets) {
100 if (set.dataset === "artifactsEventsAdaptiveGroups") continue;
101 // A dataset other than events: ask for its sums, maxes and dimensions,
102 // filtered to this namespace where it can be.
103 for (const aggregate of ["max", "sum", "avg"]) {
104 if (!set.fields.includes(aggregate)) continue;
105 try {
106 const inner = await graphql(`{ __type(name: "${set.type}") { fields { name type { name ofType { name } } } } }`);
107 const aggregateType = inner.__type.fields.find((f) => f.name === aggregate)?.type;
108 const aggregateName = aggregateType?.name ?? aggregateType?.ofType?.name;
109 const numeric = aggregateName ? await graphql(`{ __type(name: "${aggregateName}") { fields { name } } }`) : null;
110 const names = (numeric?.__type?.fields ?? []).map((f) => f.name);
111 if (!names.length) continue;
112 const data = await graphql(
113 `query Q($account: String!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $account }) {
114 ${set.dataset}(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end }) {
115 ${aggregate} { ${names.join(" ")} } dimensions { repositoryNamespace repositoryName date }
116 } } } }`,
117 { account: ACCOUNT_ID, start, end },
118 );
119 const rows = data.viewer.accounts[0][set.dataset].filter((row) => !row.dimensions?.repositoryNamespace || row.dimensions.repositoryNamespace === NAMESPACE);
120 out[`${set.dataset}.${aggregate}`] = rows;
121 } catch (error) {
122 out[`${set.dataset}.${aggregate}`] = `not readable: ${String(error.message).slice(0, 200)}`;
123 }
124 }
125 }
126 // The events themselves: errors such as storageLimitReached, and pushes.
127 const events = await graphql(
128 `query Q($account: String!, $start: Time!, $end: Time!, $ns: String!) { viewer { accounts(filter: { accountTag: $account }) {
129 artifactsEventsAdaptiveGroups(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end, repositoryNamespace: $ns }) {
130 count sum { durationMs } dimensions { repositoryName eventKind eventType }
131 } } } }`,
132 { account: ACCOUNT_ID, start, end, ns: NAMESPACE },
133 );
134 out.events = events.viewer.accounts[0].artifactsEventsAdaptiveGroups;
135 return out;
136}
137
138async function repoInfo(name) {
139 const got = await api("GET", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
140 return got.ok ? got.json.result : null;
141}
142
143async function setup() {
144 const made = await api("POST", "/artifacts/namespaces", { namespace: NAMESPACE });
145 if (!made.ok && made.status !== 409) console.log(`namespace: ${made.status} ${JSON.stringify(made.json.errors ?? "")}`);
146 const created = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos`, { name: SOURCE, description: "g1t fork storage test; safe to delete" });
147 if (!created.ok) throw new Error(`create: ${created.status} ${JSON.stringify(created.json.errors ?? created.json)}`);
148 const { remote, token: repoToken } = created.json.result;
149 console.log(`made ${NAMESPACE}/${SOURCE}`);
150 // ~MB of random bytes, in files under the 32 MB limit, so nothing compresses.
151 const dir = mkdtempSync(join(tmpdir(), "g1t-fork-test-"));
152 try {
153 const git = (...a) => execFileSync("git", a, { cwd: dir, stdio: ["ignore", "pipe", "pipe"] }).toString();
154 git("init", "-q", "-b", "main");
155 git("config", "user.email", "fork-test@g1t.invalid");
156 git("config", "user.name", "g1t fork test");
157 const files = Math.ceil(MB / 25);
158 for (let n = 0; n < files; n++) writeFileSync(join(dir, `random-${n}.bin`), randomBytes(Math.min(25, MB - n * 25) * 1024 * 1024));
159 git("add", ".");
160 git("commit", "-q", "-m", "incompressible data");
161 const started = Date.now();
162 execFileSync("git", ["-c", `http.extraHeader=Authorization: Bearer ${repoToken}`, "push", "-q", remote, "main"], { cwd: dir, stdio: "inherit" });
163 console.log(`pushed ${MB} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`);
164 } finally {
165 rmSync(dir, { recursive: true, force: true });
166 }
167 const forks = [];
168 for (let n = 1; n <= FORKS; n++) {
169 const started = Date.now();
170 const forked = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos/${SOURCE}/fork`, { name: forkName(n), default_branch_only: true });
171 const ms = Date.now() - started;
172 forks.push({ name: forkName(n), status: forked.status, ms, result: forked.json.result ?? forked.json.errors });
173 console.log(`fork ${n}: ${forked.status} in ${ms} ms ${JSON.stringify(forked.json.result ?? forked.json.errors ?? {}).slice(0, 300)}`);
174 }
175 return forks;
176}
177
178async function cleanup() {
179 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) {
180 const deleted = await api("DELETE", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
181 console.log(`delete ${name}: ${deleted.status}`);
182 }
183 console.log(`The namespace ${NAMESPACE} is left, empty (the API has no call to delete one).`);
184}
185
186async function measure(label) {
187 const figures = await storageFigures();
188 const info = {};
189 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) info[name] = await repoInfo(name);
190 console.log(`\n== ${label} (${new Date().toISOString()}) ==`);
191 console.log(JSON.stringify({ figures, repos: info }, null, 2));
192 return figures;
193}
194
195async function main() {
196 if (command === "schema") {
197 console.log(JSON.stringify(await datasets(), null, 2));
198 return;
199 }
200 if (command === "cleanup") return cleanup();
201 if (command === "measure") {
202 await measure("now");
203 return;
204 }
205 if (command !== "run") throw new Error(`unknown command ${command}`);
206 console.log("Artifacts analytics datasets:", JSON.stringify((await datasets()).map((d) => d.dataset)));
207 await measure("before");
208 let forks;
209 try {
210 forks = await setup();
211 await measure("right after");
212 const until = Date.now() + MINUTES * 60 * 1000;
213 while (Date.now() < until) {
214 await new Promise((resolve) => setTimeout(resolve, 5 * 60 * 1000));
215 await measure(`after, ${Math.round((MINUTES * 60 * 1000 - (until - Date.now())) / 60000)} min`);
216 }
217 } finally {
218 if (KEEP) console.log(`\nKept. Run \`measure\` tomorrow, then \`cleanup\`.`);
219 else await cleanup();
220 }
221 console.log("\nForks:", JSON.stringify(forks, null, 2));
222}
223
224main().catch((error) => {
225 console.error(error.message);
226 process.exit(1);
227});