Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 1 | import assert from "node:assert/strict"; |
| 2 | import { test } from "node:test"; | |
| 3 | ||
| npm installs on g1t's cloud runners stop failing with "network aborted". The upstream sometimes broke off in the middle of a tarball's body, after its headers had already been handed to the sandbox, so the retry added for failed requests couldn't help: npm got a truncated file and gave up. Now an answer to a request that can be sent again (a GET or HEAD with no body), of known length up to 8 MB and sent uncompressed, as package tarballs are, is read whole by the runner Worker before the sandbox gets it; one whose body breaks off is tried again, twice, and still failing becomes a 502 saying why, never a truncated answer. Larger answers and ones of unknown length stream as before. And the repository's installs open at most 8 connections at once rather than npm's 15, since each passes through the Worker and bursts of parallel downloads were when it happened. | 4 | import { BUFFERED_MAX_BYTES, RETRY_WAITS_MS, bufferable, forward } from "./forward.ts"; |
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 5 | |
| 6 | /** A fetch that throws `failures` times, then answers 200; it counts its calls. */ | |
| 7 | function flaky(failures: number) { | |
| 8 | const calls: Request[] = []; | |
| 9 | const send = async (request: Request) => { | |
| 10 | calls.push(request); | |
| 11 | if (calls.length <= failures) throw new Error("Network connection lost."); | |
| 12 | return new Response("ok"); | |
| 13 | }; | |
| 14 | return { calls, send }; | |
| 15 | } | |
| 16 | ||
| 17 | const waits: number[] = []; | |
| 18 | const sleep = async (ms: number) => { | |
| 19 | waits.push(ms); | |
| 20 | }; | |
| 21 | ||
| 22 | test("a GET that fails is tried again, and its answer comes back", async () => { | |
| 23 | waits.length = 0; | |
| 24 | const { calls, send } = flaky(2); | |
| 25 | const response = await forward(new Request("https://registry.npmjs.org/react"), send, sleep); | |
| 26 | assert.equal(response.status, 200); | |
| 27 | assert.equal(await response.text(), "ok"); | |
| 28 | assert.equal(calls.length, 3); | |
| 29 | assert.deepEqual(waits, [...RETRY_WAITS_MS]); | |
| 30 | }); | |
| 31 | ||
| 32 | test("one that keeps failing gets a 502 naming the host and why, never a throw", async () => { | |
| 33 | const { calls, send } = flaky(10); | |
| 34 | const response = await forward(new Request("https://Registry.npmjs.org:443/react"), send, sleep); | |
| 35 | assert.equal(calls.length, 3); | |
| 36 | assert.equal(response.status, 502); | |
| 37 | assert.equal(response.headers.get("x-g1t-egress"), "upstream-failed"); | |
| 38 | assert.match(response.headers.get("content-type") ?? "", /^text\/plain/); | |
| 39 | const body = await response.text(); | |
| 40 | assert.match(body, /registry\.npmjs\.org could not be reached/); | |
| 41 | assert.match(body, /Network connection lost\./); | |
| 42 | }); | |
| 43 | ||
| 44 | test("a POST is never sent twice", async () => { | |
| 45 | const { calls, send } = flaky(1); | |
| 46 | const response = await forward(new Request("https://api.example.com/upload", { method: "POST", body: "data" }), send, sleep); | |
| 47 | assert.equal(calls.length, 1); | |
| 48 | assert.equal(response.status, 502); | |
| 49 | }); | |
| 50 | ||
| 51 | test("an answer, even an error, is passed on as it is", async () => { | |
| 52 | const response = await forward(new Request("https://example.com/"), async () => new Response("no", { status: 503 }), sleep); | |
| 53 | assert.equal(response.status, 503); | |
| 54 | }); | |
| npm installs on g1t's cloud runners stop failing with "network aborted". The upstream sometimes broke off in the middle of a tarball's body, after its headers had already been handed to the sandbox, so the retry added for failed requests couldn't help: npm got a truncated file and gave up. Now an answer to a request that can be sent again (a GET or HEAD with no body), of known length up to 8 MB and sent uncompressed, as package tarballs are, is read whole by the runner Worker before the sandbox gets it; one whose body breaks off is tried again, twice, and still failing becomes a 502 saying why, never a truncated answer. Larger answers and ones of unknown length stream as before. And the repository's installs open at most 8 connections at once rather than npm's 15, since each passes through the Worker and bursts of parallel downloads were when it happened. | 55 | |
| 56 | /** A tarball's answer whose body breaks off after its first chunk, as an upstream that drops mid-body does. */ | |
| 57 | function brokenOff(length: number): Response { | |
| 58 | const body = new ReadableStream<Uint8Array>({ | |
| 59 | start(controller) { | |
| 60 | controller.enqueue(new Uint8Array(16)); | |
| 61 | controller.error(new Error("aborted")); | |
| 62 | }, | |
| 63 | }); | |
| 64 | return new Response(body, { headers: { "content-length": String(length), "content-type": "application/octet-stream" } }); | |
| 65 | } | |
| 66 | ||
| 67 | test("a body that breaks off midway is tried again, and the whole answer comes back", async () => { | |
| 68 | let calls = 0; | |
| 69 | const send = async () => { | |
| 70 | calls += 1; | |
| 71 | return calls === 1 ? brokenOff(1024) : new Response(new Uint8Array(1024), { headers: { "content-length": "1024" } }); | |
| 72 | }; | |
| 73 | const response = await forward(new Request("https://registry.npmjs.org/react/-/react-19.0.0.tgz"), send, sleep); | |
| 74 | assert.equal(calls, 2); | |
| 75 | assert.equal(response.status, 200); | |
| 76 | assert.equal((await response.arrayBuffer()).byteLength, 1024); | |
| 77 | }); | |
| 78 | ||
| 79 | test("a body that keeps breaking off ends as a 502, not a truncated answer", async () => { | |
| 80 | let calls = 0; | |
| 81 | const response = await forward( | |
| 82 | new Request("https://registry.npmjs.org/x.tgz"), | |
| 83 | async () => { | |
| 84 | calls += 1; | |
| 85 | return brokenOff(1024); | |
| 86 | }, | |
| 87 | sleep, | |
| 88 | ); | |
| 89 | assert.equal(calls, 3); | |
| 90 | assert.equal(response.status, 502); | |
| 91 | }); | |
| 92 | ||
| 93 | test("large, compressed or unsized answers stream as they come", () => { | |
| 94 | const sized = (length: number, headers: Record<string, string> = {}) => new Response("x", { headers: { "content-length": String(length), ...headers } }); | |
| 95 | assert.equal(bufferable(sized(1024)), true); | |
| 96 | assert.equal(bufferable(sized(BUFFERED_MAX_BYTES + 1)), false); | |
| 97 | assert.equal(bufferable(sized(1024, { "content-encoding": "gzip" })), false); | |
| 98 | assert.equal(bufferable(new Response(new ReadableStream())), false); | |
| 99 | assert.equal(bufferable(new Response(null, { status: 304 })), false); | |
| 100 | }); |
This file's history is long; its oldest lines are credited to the oldest commit read.