flagon-io/g1t

public

Drill: hash bundles as a stream, and read production as you are logged in

syntaqxcommitted 9h agoParentbd8816dBrowse files

1 file+12−70/1 viewed

+12−7View BlameViewed

30	30		// The live repository is read as G1T_USER with G1T_TOKEN (an access token
31	31		// with code:read) when they are set, which private repositories need. The
32	32		// database and the bucket are read through Wrangler, as you are logged in
33		−	// (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN.
	33	+	// (`npx wrangler login`), or with CLOUDFLARE_DEPLOY_TOKEN when that is set.
34	34
35	35		import { createHash } from "node:crypto";
36		−	import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
	36	+	import { createReadStream, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
37	37		import { tmpdir } from "node:os";
38	38		import { join } from "node:path";
39	39
⋯
53	53		return out.trim();
54	54		}
55	55
	56	+	/** A file's SHA-256, read as a stream: a bundle can be a gigabyte. */
	57	+	async function sha256Of(file) {
	58	+	const hash = createHash("sha256");
	59	+	for await (const chunk of createReadStream(file)) hash.update(chunk);
	60	+	return hash.digest("hex");
	61	+	}
	62	+
56	63		/** `<hash> <name>` lines (for-each-ref) or `<hash>\t<name>` (ls-remote), as a map. Peeled tags are left out. */
57	64		export function parseRefs(listing) {
58	65		const refs = {};
⋯
111	118		const size = statSync(file).size;
112	119		if (size !== entry.size) throw new Error(`${entry.key}: ${size} bytes, the manifest says ${entry.size}`);
113	120		if (entry.sha256) {
114		−	const sha256 = createHash("sha256").update(readFileSync(file)).digest("hex");
	121	+	const sha256 = await sha256Of(file);
115	122		if (sha256 !== entry.sha256) throw new Error(`${entry.key}: SHA-256 ${sha256}, the manifest says ${entry.sha256}`);
116	123		}
117	124		await git(["bundle", "verify", "--quiet", file], { cwd: dir });
⋯
135	142		// ---------------------------------------------------------------------
136	143
137	144		async function d1(sql) {
138		−	const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
139		−	if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = "";
	145	+	const env = wranglerEnv();
140	146		const { code, out } = await exec(process.execPath, [WRANGLER, "d1", "execute", DATABASE, "--remote", "--json", "--command", sql], {
141	147		cwd: join(ROOT, "services/repos"),
142	148		env,
⋯
175	181		function bucketReader(localCopy) {
176	182		if (localCopy) return async (key) => join(localCopy, key);
177	183		return async (key, file) => {
178		−	const env = { ...wranglerEnv({ ...process.env, CI: "true" }) };
179		−	if (!process.env.CLOUDFLARE_DEPLOY_TOKEN) env.CLOUDFLARE_API_TOKEN = "";
	184	+	const env = wranglerEnv();
180	185		const { code, out } = await exec(process.execPath, [WRANGLER, "r2", "object", "get", `${BUCKET}/${key}`, "--remote", "--file", file], { env });
181	186		if (code !== 0) throw new Error(`${key} could not be read: ${out.slice(-400)}`);
182	187		return file;