flagon-io/g1t

public

npm installs on g1t's cloud runners stop failing with "network aborted". The upstream sometimes broke off in the middle of a tarball's body, after its headers had already been handed to the sandbox, so the retry added for failed requests couldn't help: npm got a truncated file and gave up. Now an answer to a request that can be sent again (a GET or HEAD with no body), of known length up to 8 MB and sent uncompressed, as package tarballs are, is read whole by the runner Worker before the sandbox gets it; one whose body breaks off is tried again, twice, and still failing becomes a 502 saying why, never a truncated answer. Larger answers and ones of unknown length stream as before. And the repository's installs open at most 8 connections at once rather than npm's 15, since each passes through the Worker and bursts of parallel downloads were when it happened.

syntaqxcommitted yesterdayParentfd960fcBrowse files

1	1		import assert from "node:assert/strict";
2	2		import { test } from "node:test";
3	3
4		−	import { RETRY_WAITS_MS, forward } from "./forward.ts";
	4	+	import { BUFFERED_MAX_BYTES, RETRY_WAITS_MS, bufferable, forward } from "./forward.ts";
5	5
6	6		/** A fetch that throws `failures` times, then answers 200; it counts its calls. */
7	7		function flaky(failures: number) {
⋯
52	52		const response = await forward(new Request("https://example.com/"), async () => new Response("no", { status: 503 }), sleep);
53	53		assert.equal(response.status, 503);
54	54		});
	55	+
	56	+	/** A tarball's answer whose body breaks off after its first chunk, as an upstream that drops mid-body does. */
	57	+	function brokenOff(length: number): Response {
	58	+	const body = new ReadableStream<Uint8Array>({
	59	+	start(controller) {
	60	+	controller.enqueue(new Uint8Array(16));
	61	+	controller.error(new Error("aborted"));
	62	+	},
	63	+	});
	64	+	return new Response(body, { headers: { "content-length": String(length), "content-type": "application/octet-stream" } });
	65	+	}
	66	+
	67	+	test("a body that breaks off midway is tried again, and the whole answer comes back", async () => {
	68	+	let calls = 0;
	69	+	const send = async () => {
	70	+	calls += 1;
	71	+	return calls === 1 ? brokenOff(1024) : new Response(new Uint8Array(1024), { headers: { "content-length": "1024" } });
	72	+	};
	73	+	const response = await forward(new Request("https://registry.npmjs.org/react/-/react-19.0.0.tgz"), send, sleep);
	74	+	assert.equal(calls, 2);
	75	+	assert.equal(response.status, 200);
	76	+	assert.equal((await response.arrayBuffer()).byteLength, 1024);
	77	+	});
	78	+
	79	+	test("a body that keeps breaking off ends as a 502, not a truncated answer", async () => {
	80	+	let calls = 0;
	81	+	const response = await forward(
	82	+	new Request("https://registry.npmjs.org/x.tgz"),
	83	+	async () => {
	84	+	calls += 1;
	85	+	return brokenOff(1024);
	86	+	},
	87	+	sleep,
	88	+	);
	89	+	assert.equal(calls, 3);
	90	+	assert.equal(response.status, 502);
	91	+	});
	92	+
	93	+	test("large, compressed or unsized answers stream as they come", () => {
	94	+	const sized = (length: number, headers: Record<string, string> = {}) => new Response("x", { headers: { "content-length": String(length), ...headers } });
	95	+	assert.equal(bufferable(sized(1024)), true);
	96	+	assert.equal(bufferable(sized(BUFFERED_MAX_BYTES + 1)), false);
	97	+	assert.equal(bufferable(sized(1024, { "content-encoding": "gzip" })), false);
	98	+	assert.equal(bufferable(new Response(new ReadableStream())), false);
	99	+	assert.equal(bufferable(new Response(null, { status: 304 })), false);
	100	+	});

6	6		* than something to retry. So a request that can be sent again (a GET or
7	7		* HEAD with no body) is tried twice more after a short wait, and what
8	8		* still fails comes back as a `502` saying which host and why, which
9		−	* those tools do retry. It never throws.
	9	+	* those tools do retry. It never throws. An answer of known, modest size
	10	+	* to such a request is read whole before it is handed on, so one whose
	11	+	* body breaks off midway is tried again too (`BUFFERED_MAX_BYTES`).
10	12		*/
11	13
12	14		import { normalizeHost } from "./egress.ts";
⋯
14	16		/** The waits before the second and third tries, in milliseconds. */
15	17		export const RETRY_WAITS_MS: readonly number[] = [250, 1000];
16	18
	19	+	/**
	20	+	* The largest answer to a replayable request that is read whole before the
	21	+	* sandbox is answered. An upstream can also break off in the middle of a
	22	+	* body, after its headers: npm then reports `network aborted` with a
	23	+	* truncated tarball, and by then the answer was already the sandbox's, too
	24	+	* late to try again. An answer read whole first is tried again like any
	25	+	* other failure. Package tarballs and most API answers are well under this;
	26	+	* larger ones, and ones of unknown length, stream as they come.
	27	+	*/
	28	+	export const BUFFERED_MAX_BYTES = 8 * 1024 * 1024;
	29	+
	30	+	/** Whether an answer is to be read whole: a known length within the limit, sent as it is. */
	31	+	export function bufferable(response: Response): boolean {
	32	+	if (response.body === null) return false;
	33	+	const length = Number(response.headers.get("content-length") ?? NaN);
	34	+	const encoding = (response.headers.get("content-encoding") ?? "identity").toLowerCase();
	35	+	return Number.isFinite(length) && length <= BUFFERED_MAX_BYTES && encoding === "identity";
	36	+	}
	37	+
	38	+	/** An answer whose body is read whole; throws when the body breaks off. */
	39	+	async function whole(response: Response): Promise<Response> {
	40	+	const body = await response.arrayBuffer();
	41	+	const expected = Number(response.headers.get("content-length"));
	42	+	if (body.byteLength !== expected) throw new Error(`the answer broke off after ${body.byteLength} of ${expected} bytes`);
	43	+	return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
	44	+	}
	45	+
17	46		/** Whether a request can be sent again as it was: no body, nothing changed. */
18	47		export function replayable(request: Request): boolean {
19	48		return (request.method === "GET" \|\| request.method === "HEAD") && request.body === null;
⋯
46	75		for (let attempt = 1; attempt <= tries; attempt++) {
47	76		if (attempt > 1) await sleep(RETRY_WAITS_MS[attempt - 2]);
48	77		try {
49		−	return await send(request);
	78	+	const response = await send(request);
	79	+	// Read whole first when it can be sent again, so a body that breaks off is retried too.
	80	+	return tries > 1 && bufferable(response) ? await whole(response) : response;
50	81		} catch (error) {
51	82		failure = error;
52	83		console.log("egress upstream failed", normalizeHost(host), request.method, `try ${attempt} of ${tries}`, String(error));

4	4		fetch-retries=5
5	5		fetch-retry-mintimeout=5000
6	6		fetch-retry-maxtimeout=60000
	7	+	# Fewer connections at once than npm's 15: on g1t's cloud runners each one
	8	+	# passes through the runner Worker, and bursts of parallel tarball
	9	+	# downloads are when a body broke off midway.
	10	+	maxsockets=8