flagon-io/g1t

public

Download ZIP reads blobs eight batches at a time and deflates files together

syntaqxcommitted 9h agoParent03c23b4Browse files

2 files+19−100/2 viewed

32	32		const parts: Uint8Array[] = [];
33	33		const central: Uint8Array[] = [];
34	34		let offset = 0;
35		−	for (const file of files) {
	35	+	// Every file deflated at once: the streams interleave rather than queue.
	36	+	const deflatedAll = await Promise.all(files.map((file) => (file.data.length > 0 ? deflate(file.data) : file.data)));
	37	+	for (const [index, file] of files.entries()) {
36	38		const name = encoder.encode(file.path);
37	39		const crc = crc32(file.data);
38		−	const deflated = file.data.length > 0 ? await deflate(file.data) : file.data;
	40	+	const deflated = deflatedAll[index]!;
39	41		const [method, body] = deflated.length < file.data.length ? [8, deflated] : [0, file.data];
40	42		const header = (central: boolean) => {
41	43		const out = new Uint8Array((central ? 46 : 30) + name.length);

+15−8View BlameViewed

13	13		const MAX_FILES = 10_000;
14	14		// The bytes, their base64 and the zip are all held at once, in a Worker of 128 MB.
15	15		const MAX_BYTES = 24 * 1024 * 1024;
16		−	/** Blobs read per call to the repository. */
	16	+	/** Blobs read per call to the repository, and calls at once. */
17	17		const PER_READ = 100;
	18	+	const READS_AT_ONCE = 8;
18	19
19	20		function refused(status: number, message: string): Response {
20	21		return new Response(`${message}\n`, { status, headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" } });
⋯
37	38		const unique = [...new Set(files.map((file) => file.hash))];
38	39		const bytes = new Map<string, Uint8Array>();
39	40		let total = 0;
40		−	for (let at = 0; at < unique.length; at += PER_READ) {
41		−	for (const blob of await repos.rawBlobs(repo.value.id, unique.slice(at, at + PER_READ), MAX_BYTES)) {
42		−	total += blob.size;
43		−	if (total > MAX_BYTES \|\| (blob.data == null && blob.size > 0)) {
44		−	return refused(413, `It is over ${MAX_BYTES / 1024 / 1024} MB, too large for a download. Clone it instead: ${clone}`);
	41	+	let tooLarge = false;
	42	+	const batches: string[][] = [];
	43	+	for (let at = 0; at < unique.length; at += PER_READ) batches.push(unique.slice(at, at + PER_READ));
	44	+	// A few reads at a time, each taking the next batch, until all are read or it is too large.
	45	+	const reader = async () => {
	46	+	for (let batch = batches.shift(); batch && !tooLarge; batch = batches.shift()) {
	47	+	for (const blob of await repos.rawBlobs(repo.value.id, batch, MAX_BYTES)) {
	48	+	total += blob.size;
	49	+	if (total > MAX_BYTES \|\| (blob.data == null && blob.size > 0)) tooLarge = true;
	50	+	else bytes.set(blob.hash, Uint8Array.from(atob(blob.data ?? ""), (c) => c.charCodeAt(0)));
45	51		}
46		−	bytes.set(blob.hash, Uint8Array.from(atob(blob.data ?? ""), (c) => c.charCodeAt(0)));
47	52		}
48		−	}
	53	+	};
	54	+	await Promise.all(Array.from({ length: READS_AT_ONCE }, reader));
	55	+	if (tooLarge) return refused(413, `It is over ${MAX_BYTES / 1024 / 1024} MB, too large for a download. Clone it instead: ${clone}`);
49	56		// A branch name can hold slashes; the folder and file name cannot.
50	57		const label = `${repo.value.name}-${ref.replaceAll("/", "-")}`;
51	58		const archive = await zip(files.map((file) => ({ path: `${label}/${file.path}`, data: bytes.get(file.hash) ?? new Uint8Array() })));