Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 1 | /** |
| 2 | * Brokered secrets, as the runner Worker fills them in. A workflow job is | |
| 3 | * given a stand-in (`g1tsec_…`) in place of a brokered secret's value | |
| 4 | * (services/actions broker.rs), and every HTTP and HTTPS request its | |
| 5 | * sandbox makes passes through `egress` (guard.ts). There, a request to one | |
| 6 | * of the secret's hosts has the stand-in replaced by the value; a request | |
| 7 | * anywhere else goes as it is, stand-in and all, which is useless to | |
| 8 | * whoever receives it, and the job's log says so once. | |
| 9 | * | |
| 10 | * Where a stand-in is filled in: every request header, as it is, and | |
| 11 | * inside `Authorization: Basic` (decoded, swapped, encoded again), and the | |
| 12 | * URL's query string. Never the path or the body. An answer that echoes the | |
| 13 | * value back has it swapped for the stand-in again in its headers, and in | |
| 14 | * its body when that is small, plain text. | |
| 15 | * | |
| 16 | * Pure, so it can be tested on its own, but for `brokerFor`, which asks | |
| 17 | * the actions service and keeps the answer a minute per job. | |
| 18 | */ | |
| 19 | ||
| 20 | import type { BrokeredSecret, ServiceBinding } from "@g1t/contracts"; | |
| 21 | ||
| 22 | import { allows, normalizeHost } from "./egress.ts"; | |
| 23 | ||
| 24 | export type { BrokeredSecret }; | |
| 25 | ||
| 26 | /** | |
| 27 | * A stand-in, wherever it is: `STAND_IN_PREFIX` (the actions contracts) | |
| 28 | * and 48 hex characters. | |
| 29 | */ | |
| 30 | const STAND_IN = /g1tsec_[0-9a-f]{48}/g; | |
| 31 | ||
| 32 | /** The job a sandbox runs, for the outbound handler: who to ask for stand-ins. */ | |
| 33 | export type BrokeredJob = { id: string; token: string }; | |
| 34 | ||
| 35 | /** Base64 to text, or null when it is not base64 of UTF-8. */ | |
| 36 | function fromBase64(text: string): string | null { | |
| 37 | try { | |
| 38 | const bytes = Uint8Array.from(atob(text.trim()), (c) => c.charCodeAt(0)); | |
| 39 | return new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes); | |
| 40 | } catch { | |
| 41 | return null; | |
| 42 | } | |
| 43 | } | |
| 44 | ||
| 45 | function toBase64(text: string): string { | |
| 46 | let binary = ""; | |
| 47 | for (const byte of new TextEncoder().encode(text)) binary += String.fromCharCode(byte); | |
| 48 | return btoa(binary); | |
| 49 | } | |
| 50 | ||
| 51 | /** A header's value as a stand-in may sit in it: Basic credentials decoded. */ | |
| 52 | function readable(value: string): string { | |
| 53 | const basic = /^basic\s+(.+)$/i.exec(value.trim()); | |
| 54 | return (basic && fromBase64(basic[1])) || value; | |
| 55 | } | |
| 56 | ||
| 57 | /** The stand-ins in a request: in its URL, its headers, or its Basic credentials. */ | |
| 58 | export function standInsIn(url: string, headers: Headers): string[] { | |
| 59 | const found = new Set<string>(); | |
| 60 | const look = (text: string) => { | |
| 61 | for (const match of text.matchAll(STAND_IN)) found.add(match[0]); | |
| 62 | }; | |
| 63 | look(url); | |
| 64 | headers.forEach((value) => look(readable(value))); | |
| 65 | return [...found]; | |
| 66 | } | |
| 67 | ||
| 68 | /** What filling in a request came to. */ | |
| 69 | export type Filled = { | |
| 70 | url: string; | |
| 71 | headers: Headers; | |
| 72 | /** The secrets filled in, whose values are kept out of the answer. */ | |
| 73 | filled: BrokeredSecret[]; | |
| 74 | /** The secrets whose stand-ins went to a host they are not for, left as they were. */ | |
| 75 | misdirected: BrokeredSecret[]; | |
| 76 | }; | |
| 77 | ||
| 78 | function swap(text: string, secrets: readonly BrokeredSecret[], encode: (value: string) => string = (v) => v): string { | |
| 79 | let out = text; | |
| 80 | for (const secret of secrets) out = out.split(secret.standIn).join(encode(secret.value)); | |
| 81 | return out; | |
| 82 | } | |
| 83 | ||
| 84 | /** | |
| 85 | * Fills in the stand-ins of `secrets` that a request to `url` carries, for | |
| 86 | * those whose hosts include its host. A stand-in the job was not given is | |
| 87 | * left alone; one for another host is left as it is and reported. | |
| 88 | */ | |
| 89 | export function fill(url: string, headers: Headers, secrets: readonly BrokeredSecret[]): Filled { | |
| 90 | const present = new Set(standInsIn(url, headers)); | |
| 91 | const carried = secrets.filter((secret) => present.has(secret.standIn)); | |
| A brokered secret is filled in only over HTTPS. A job's request to one of the secret's hosts over plain HTTP would have carried the real value across the internet readable by anyone on the way; it now goes with the stand-in, and the job's log notes it once, as for a host the secret is not for. The secrets guide says so. | 92 | const parsed = new URL(url); |
| 93 | const host = normalizeHost(parsed.host); | |
| 94 | // Only over HTTPS: a value filled into plain HTTP would cross the | |
| 95 | // internet readable by anyone on the way, its host or not. Such a | |
| 96 | // request goes with the stand-in, and is reported as misdirected. | |
| 97 | const secure = parsed.protocol === "https:"; | |
| 98 | const filled = carried.filter((secret) => secure && allows(secret.hosts, host)); | |
| 99 | const misdirected = carried.filter((secret) => !filled.includes(secret)); | |
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 100 | if (filled.length === 0) return { url, headers, filled, misdirected }; |
| 101 | ||
| 102 | const out = new Headers(); | |
| 103 | headers.forEach((value, name) => { | |
| 104 | const basic = /^(basic\s+)(.+)$/i.exec(value.trim()); | |
| 105 | const decoded = basic ? fromBase64(basic[2]) : null; | |
| 106 | if (basic && decoded != null && filled.some((secret) => decoded.includes(secret.standIn))) { | |
| 107 | out.append(name, `${basic[1]}${toBase64(swap(decoded, filled))}`); | |
| 108 | } else { | |
| 109 | out.append(name, swap(value, filled)); | |
| 110 | } | |
| 111 | }); | |
| 112 | // Only the query string: a value in a path is rare, and is left as the | |
| 113 | // job wrote it. | |
| 114 | if (parsed.search) parsed.search = swap(parsed.search, filled, encodeURIComponent); | |
| 115 | return { url: parsed.toString(), headers: out, filled, misdirected }; | |
| 116 | } | |
| 117 | ||
| 118 | /** The largest answer whose body is searched for an echoed value. */ | |
| 119 | export const MAX_SCRUBBED_BYTES = 256 * 1024; | |
| 120 | ||
| 121 | /** Whether an answer's body is small, plain text the Worker may rewrite. */ | |
| 122 | export function scrubbable(headers: Headers): boolean { | |
| 123 | const type = (headers.get("content-type") ?? "").toLowerCase(); | |
| 124 | const textual = /^(text\/|application\/(json|xml|x-www-form-urlencoded|[a-z0-9.+-]+\+(json|xml)))/.test(type); | |
| 125 | const encoding = (headers.get("content-encoding") ?? "identity").toLowerCase(); | |
| 126 | const length = Number(headers.get("content-length") ?? NaN); | |
| 127 | return textual && encoding === "identity" && Number.isFinite(length) && length <= MAX_SCRUBBED_BYTES; | |
| 128 | } | |
| 129 | ||
| 130 | /** Text with each filled-in value swapped back for its stand-in. */ | |
| 131 | export function scrub(text: string, filled: readonly BrokeredSecret[]): string { | |
| 132 | let out = text; | |
| 133 | for (const secret of filled) { | |
| 134 | if (secret.value) out = out.split(secret.value).join(secret.standIn); | |
| 135 | } | |
| 136 | return out; | |
| 137 | } | |
| 138 | ||
| 139 | /** An answer's headers, with any filled-in value swapped back for its stand-in. */ | |
| 140 | export function scrubHeaders(headers: Headers, filled: readonly BrokeredSecret[]): Headers { | |
| 141 | const out = new Headers(); | |
| 142 | headers.forEach((value, name) => out.append(name, scrub(value, filled))); | |
| 143 | return out; | |
| 144 | } | |
| 145 | ||
| 146 | /** | |
| 147 | * The line a job's log gets when a brokered secret's stand-in went to a | |
| A brokered secret is filled in only over HTTPS. A job's request to one of the secret's hosts over plain HTTP would have carried the real value across the internet readable by anyone on the way; it now goes with the stand-in, and the job's log notes it once, as for a host the secret is not for. The secrets guide says so. | 148 | * host it is not for, or over plain HTTP. |
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 149 | */ |
| 150 | export function misdirectedNote(name: string, host: string): string { | |
| A brokered secret is filled in only over HTTPS. A job's request to one of the secret's hosts over plain HTTP would have carried the real value across the internet readable by anyone on the way; it now goes with the stand-in, and the job's log notes it once, as for a host the secret is not for. The secrets guide says so. | 151 | return `##[warning]${name} was sent to ${normalizeHost(host)}, which it is not for or not over HTTPS, so g1t did not fill it in.`; |
| Merge brokered secrets: a secret a job uses only to call an API can be tied to the hosts it is for, and the job gets a stand-in in its place, which g1t fills in on the way out to those hosts and nowhere else, so the value never enters the sandbox; and a request to an allowed host that fails upstream is retried when it can be, or answered with a 502 that says why, rather than reset. | 152 | } |
| 153 | ||
| 154 | /** The most misdirected secrets noted in one job's log. */ | |
| 155 | const MAX_MISDIRECTED_NOTED = 25; | |
| 156 | ||
| 157 | /** Notes a misdirected secret once per secret and host. `seen` is what was noted. */ | |
| 158 | export function newlyMisdirected(seen: readonly string[], name: string, host: string): { note: string; seen: string[] } | null { | |
| 159 | const key = `${name} ${normalizeHost(host)}`; | |
| 160 | if (seen.includes(key) || seen.length >= MAX_MISDIRECTED_NOTED) return null; | |
| 161 | return { note: misdirectedNote(name, host), seen: [...seen, key] }; | |
| 162 | } | |
| 163 | ||
| 164 | type Cached = { at: number; secrets: BrokeredSecret[] }; | |
| 165 | /** Each job's stand-ins, in this isolate, for a minute: requests come in bursts. */ | |
| 166 | const CACHE = new Map<string, Cached>(); | |
| 167 | const CACHE_MS = 60_000; | |
| 168 | /** A stand-in not in the cache is asked about again, but not more often than this. */ | |
| 169 | const RECHECK_MS = 5_000; | |
| 170 | const CACHE_JOBS = 500; | |
| 171 | ||
| 172 | type Outcome<T> = { ok: true; value: T } | { ok: false; error: { message: string } }; | |
| 173 | ||
| 174 | /** | |
| 175 | * A running job's stand-ins and what they stand for, from the actions | |
| 176 | * service with the job's token. Kept a minute; asked again sooner when the | |
| 177 | * request carries a stand-in the kept answer lacks. Empty when the service | |
| 178 | * cannot say: the stand-ins then go as they are. | |
| 179 | */ | |
| 180 | export async function brokerFor(actions: ServiceBinding, job: BrokeredJob, wanted: readonly string[], now = Date.now()): Promise<BrokeredSecret[]> { | |
| 181 | const kept = CACHE.get(job.id); | |
| 182 | const fresh = kept && now - kept.at < CACHE_MS; | |
| 183 | const complete = kept && wanted.every((standIn) => kept.secrets.some((secret) => secret.standIn === standIn)); | |
| 184 | if (kept && fresh && (complete || now - kept.at < RECHECK_MS)) return kept.secrets; | |
| 185 | try { | |
| 186 | const response = await actions.fetch("https://actions/rpc/brokered_secrets", { | |
| 187 | method: "POST", | |
| 188 | headers: { "content-type": "application/json" }, | |
| 189 | body: JSON.stringify({ job: job.id, token: job.token }), | |
| 190 | }); | |
| 191 | if (!response.ok) throw new Error(`brokered_secrets failed with status ${response.status}`); | |
| 192 | const answer = (await response.json()) as Outcome<BrokeredSecret[]>; | |
| 193 | const secrets = answer.ok ? answer.value : []; | |
| 194 | if (CACHE.size >= CACHE_JOBS) CACHE.delete(CACHE.keys().next().value as string); | |
| 195 | CACHE.set(job.id, { at: now, secrets }); | |
| 196 | return secrets; | |
| 197 | } catch (error) { | |
| 198 | // Never the value: only that it could not be had. | |
| 199 | console.log("brokered secrets not read", job.id, String(error)); | |
| 200 | return kept?.secrets ?? []; | |
| 201 | } | |
| 202 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.