Skip to content
Sign up

flagon-io/g1t

public
0
226 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1#!/usr/bin/env node
2// Do Artifacts forks copy their source's objects, or share them?
3//
The docs folder is gone, and what it held lives where people read it: how a self-hosted g1t runs and how to deploy g1t to Cloudflare are pages on docs.g1t.sh under Run g1t yourself, and speed, rate limits and operating g1t.sh are sections of CONTRIBUTING.md; code that cited a file in docs/ now points to the page or section that covers it, or says what it means itself, and applied migrations and the runner images are left as they were.4// Cloudflare does not document it, and it decides
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily5// how much every pull request costs in storage. This makes a throwaway
6// repository holding ~100 MB that cannot be compressed, forks it five
7// times, and reads whatever storage figures Cloudflare reports before and
8// after, then deletes everything it made. Nothing of g1t's is touched: it
9// works in its own namespace (default `g1t-storage-test`), straight
10// against Cloudflare's API, never through g1t.sh.
11//
12// export CLOUDFLARE_API_TOKEN=<token: Artifacts edit, Account Analytics read>
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results13// (or a global API key: CLOUDFLARE_API_KEY with CLOUDFLARE_EMAIL)
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily14// node scripts/ops/fork-storage-test.mjs run # make, fork 5x, measure for 20 min, delete
15// node scripts/ops/fork-storage-test.mjs run --keep # ... and keep it, to measure again tomorrow
16// node scripts/ops/fork-storage-test.mjs measure # read the figures again (e.g. the next day)
17// node scripts/ops/fork-storage-test.mjs cleanup # delete the test repositories
18// node scripts/ops/fork-storage-test.mjs schema # list the analytics datasets Cloudflare offers for Artifacts
19//
20// Options: --namespace <name> (default g1t-storage-test), --mb <size> (100),
21// --forks <n> (5), --minutes <n> to keep measuring (20).
22//
23// Needs git on PATH. Costs a few cents of Artifacts storage and operations.
24
25import { execFileSync } from "node:child_process";
26import { randomBytes } from "node:crypto";
27import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
28import { tmpdir } from "node:os";
29import { join } from "node:path";
30
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results31import { cloudflareAuth } from "../deploy/cloudflare.mjs";
32
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily33const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID || "1e6f2cffa3f445920836e8ebe446bb58";
34const API = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}`;
35const args = process.argv.slice(2);
36const command = args[0] ?? "run";
37const option = (name, fallback) => {
38 const at = args.indexOf(name);
39 return at >= 0 && args[at + 1] ? args[at + 1] : fallback;
40};
41const NAMESPACE = option("--namespace", "g1t-storage-test");
42const MB = Number(option("--mb", "100"));
43const FORKS = Number(option("--forks", "5"));
44const MINUTES = Number(option("--minutes", "20"));
45const KEEP = args.includes("--keep");
46const SOURCE = "fork-test-source";
47const forkName = (n) => `fork-test-copy-${n}`;
48
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results49const auth = cloudflareAuth();
50if (!auth) {
51 console.error("Set CLOUDFLARE_API_TOKEN (Artifacts edit, Account Analytics read), or CLOUDFLARE_API_KEY and CLOUDFLARE_EMAIL.");
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily52 process.exit(2);
53}
54
55async function api(method, path, body) {
56 const response = await fetch(`${API}${path}`, {
57 method,
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results58 headers: { ...auth, "content-type": "application/json" },
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily59 body: body ? JSON.stringify(body) : undefined,
60 });
61 const json = await response.json().catch(() => ({}));
62 return { status: response.status, ok: response.ok && json.success !== false, json };
63}
64
65async function graphql(query, variables = {}) {
66 const response = await fetch("https://api.cloudflare.com/client/v4/graphql", {
67 method: "POST",
g1t-runner 0.1.0 is released: signed binaries for five platforms at g1t.sh/downloads/runner; the Artifacts checks' results68 headers: { ...auth, "content-type": "application/json" },
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily69 body: JSON.stringify({ query, variables }),
70 });
71 const json = await response.json();
72 if (json.errors?.length) throw new Error(JSON.stringify(json.errors).slice(0, 500));
73 return json.data;
74}
75
76/** The account's analytics datasets about Artifacts, and their numeric fields. */
77async function datasets() {
78 const data = await graphql(`{ __type(name: "account") { fields { name type { name ofType { name ofType { name ofType { name } } } } } } }`);
79 const fields = data.__type?.fields ?? [];
80 const found = [];
81 for (const field of fields.filter((f) => /artifact/i.test(f.name))) {
82 let type = field.type;
83 while (type && !type.name) type = type.ofType;
84 while (type?.ofType) type = type.ofType;
85 const typeName = type?.name;
86 const shape = typeName ? await graphql(`{ __type(name: "${typeName}") { fields { name type { name kind ofType { name kind } } } } }`) : null;
87 found.push({ dataset: field.name, type: typeName, fields: (shape?.__type?.fields ?? []).map((f) => f.name) });
88 }
89 return found;
90}
91
92/** Every storage-looking figure Cloudflare reports for the test repositories. */
93async function storageFigures() {
94 const out = {};
95 const sets = await datasets();
96 const start = new Date(Date.now() - 2 * 24 * 3600 * 1000).toISOString();
97 const end = new Date().toISOString();
98 for (const set of sets) {
99 if (set.dataset === "artifactsEventsAdaptiveGroups") continue;
100 // A dataset other than events: ask for its sums, maxes and dimensions,
101 // filtered to this namespace where it can be.
102 for (const aggregate of ["max", "sum", "avg"]) {
103 if (!set.fields.includes(aggregate)) continue;
104 try {
105 const inner = await graphql(`{ __type(name: "${set.type}") { fields { name type { name ofType { name } } } } }`);
106 const aggregateType = inner.__type.fields.find((f) => f.name === aggregate)?.type;
107 const aggregateName = aggregateType?.name ?? aggregateType?.ofType?.name;
108 const numeric = aggregateName ? await graphql(`{ __type(name: "${aggregateName}") { fields { name } } }`) : null;
109 const names = (numeric?.__type?.fields ?? []).map((f) => f.name);
110 if (!names.length) continue;
111 const data = await graphql(
112 `query Q($account: String!, $start: Time!, $end: Time!) { viewer { accounts(filter: { accountTag: $account }) {
113 ${set.dataset}(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end }) {
114 ${aggregate} { ${names.join(" ")} } dimensions { repositoryNamespace repositoryName date }
115 } } } }`,
116 { account: ACCOUNT_ID, start, end },
117 );
118 const rows = data.viewer.accounts[0][set.dataset].filter((row) => !row.dimensions?.repositoryNamespace || row.dimensions.repositoryNamespace === NAMESPACE);
119 out[`${set.dataset}.${aggregate}`] = rows;
120 } catch (error) {
121 out[`${set.dataset}.${aggregate}`] = `not readable: ${String(error.message).slice(0, 200)}`;
122 }
123 }
124 }
125 // The events themselves: errors such as storageLimitReached, and pushes.
126 const events = await graphql(
127 `query Q($account: String!, $start: Time!, $end: Time!, $ns: String!) { viewer { accounts(filter: { accountTag: $account }) {
128 artifactsEventsAdaptiveGroups(limit: 1000, filter: { datetime_geq: $start, datetime_leq: $end, repositoryNamespace: $ns }) {
129 count sum { durationMs } dimensions { repositoryName eventKind eventType }
130 } } } }`,
131 { account: ACCOUNT_ID, start, end, ns: NAMESPACE },
132 );
133 out.events = events.viewer.accounts[0].artifactsEventsAdaptiveGroups;
134 return out;
135}
136
137async function repoInfo(name) {
138 const got = await api("GET", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
139 return got.ok ? got.json.result : null;
140}
141
142async function setup() {
143 const made = await api("POST", "/artifacts/namespaces", { namespace: NAMESPACE });
144 if (!made.ok && made.status !== 409) console.log(`namespace: ${made.status} ${JSON.stringify(made.json.errors ?? "")}`);
145 const created = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos`, { name: SOURCE, description: "g1t fork storage test; safe to delete" });
146 if (!created.ok) throw new Error(`create: ${created.status} ${JSON.stringify(created.json.errors ?? created.json)}`);
147 const { remote, token: repoToken } = created.json.result;
148 console.log(`made ${NAMESPACE}/${SOURCE}`);
149 // ~MB of random bytes, in files under the 32 MB limit, so nothing compresses.
150 const dir = mkdtempSync(join(tmpdir(), "g1t-fork-test-"));
151 try {
152 const git = (...a) => execFileSync("git", a, { cwd: dir, stdio: ["ignore", "pipe", "pipe"] }).toString();
153 git("init", "-q", "-b", "main");
154 git("config", "user.email", "fork-test@g1t.invalid");
155 git("config", "user.name", "g1t fork test");
156 const files = Math.ceil(MB / 25);
157 for (let n = 0; n < files; n++) writeFileSync(join(dir, `random-${n}.bin`), randomBytes(Math.min(25, MB - n * 25) * 1024 * 1024));
158 git("add", ".");
159 git("commit", "-q", "-m", "incompressible data");
160 const started = Date.now();
161 execFileSync("git", ["-c", `http.extraHeader=Authorization: Bearer ${repoToken}`, "push", "-q", remote, "main"], { cwd: dir, stdio: "inherit" });
162 console.log(`pushed ${MB} MB in ${((Date.now() - started) / 1000).toFixed(1)} s`);
163 } finally {
164 rmSync(dir, { recursive: true, force: true });
165 }
166 const forks = [];
167 for (let n = 1; n <= FORKS; n++) {
168 const started = Date.now();
169 const forked = await api("POST", `/artifacts/namespaces/${NAMESPACE}/repos/${SOURCE}/fork`, { name: forkName(n), default_branch_only: true });
170 const ms = Date.now() - started;
171 forks.push({ name: forkName(n), status: forked.status, ms, result: forked.json.result ?? forked.json.errors });
172 console.log(`fork ${n}: ${forked.status} in ${ms} ms ${JSON.stringify(forked.json.result ?? forked.json.errors ?? {}).slice(0, 300)}`);
173 }
174 return forks;
175}
176
177async function cleanup() {
178 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) {
179 const deleted = await api("DELETE", `/artifacts/namespaces/${NAMESPACE}/repos/${name}`);
180 console.log(`delete ${name}: ${deleted.status}`);
181 }
182 console.log(`The namespace ${NAMESPACE} is left, empty (the API has no call to delete one).`);
183}
184
185async function measure(label) {
186 const figures = await storageFigures();
187 const info = {};
188 for (const name of [SOURCE, ...Array.from({ length: FORKS }, (_, n) => forkName(n + 1))]) info[name] = await repoInfo(name);
189 console.log(`\n== ${label} (${new Date().toISOString()}) ==`);
190 console.log(JSON.stringify({ figures, repos: info }, null, 2));
191 return figures;
192}
193
194async function main() {
195 if (command === "schema") {
196 console.log(JSON.stringify(await datasets(), null, 2));
197 return;
198 }
199 if (command === "cleanup") return cleanup();
200 if (command === "measure") {
201 await measure("now");
202 return;
203 }
204 if (command !== "run") throw new Error(`unknown command ${command}`);
205 console.log("Artifacts analytics datasets:", JSON.stringify((await datasets()).map((d) => d.dataset)));
206 await measure("before");
207 let forks;
208 try {
209 forks = await setup();
210 await measure("right after");
211 const until = Date.now() + MINUTES * 60 * 1000;
212 while (Date.now() < until) {
213 await new Promise((resolve) => setTimeout(resolve, 5 * 60 * 1000));
214 await measure(`after, ${Math.round((MINUTES * 60 * 1000 - (until - Date.now())) / 60000)} min`);
215 }
216 } finally {
217 if (KEEP) console.log(`\nKept. Run \`measure\` tomorrow, then \`cleanup\`.`);
218 else await cleanup();
219 }
220 console.log("\nForks:", JSON.stringify(forks, null, 2));
221}
222
223main().catch((error) => {
224 console.error(error.message);
225 process.exit(1);
226});