| 1 | /** |
| 2 | * A branch, tag or commit of a repository as one archive: a zip |
| 3 | * (`archive/<ref>.zip`, the Files page's Download ZIP) or a gzipped tar |
| 4 | * (`archive/<ref>.tar.gz`), its files under `<repo>-<ref>/` as |
| 5 | * `git archive` would name them, executables and symlinks kept as they are |
| 6 | * in git. For anyone who can see the repository. |
| 7 | * |
| 8 | * The REST API's `GET /repos/{owner}/{repo}/tarball/{ref}` and |
| 9 | * `/zipball/{ref}` redirect here with `?token=`, a link the repos service |
| 10 | * made for someone who can read the repository (services/repos |
| 11 | * archive_links.rs), good for one commit for ten minutes with no session: |
| 12 | * that is how `actions/checkout` downloads a private repository in a job |
| 13 | * container without git. Those archives are named as GitHub's are, |
| 14 | * `<owner>-<repo>-<short sha>/`. |
| 15 | * |
| 16 | * Made on request and not kept, so it is capped: a repository past the caps |
| 17 | * is cloned instead. |
| 18 | */ |
| 19 | import type { Route } from "./+types/archive"; |
| 20 | import { cloneUrl } from "../../lib/addresses"; |
| 21 | import { addresses } from "../../lib/addresses.server"; |
| 22 | import { contentDisposition } from "../../lib/content-safety"; |
| 23 | import { repos } from "../../lib/services.server"; |
| 24 | import { getViewer } from "../../lib/session.server"; |
| 25 | import { tarGz } from "../../lib/tar"; |
| 26 | import { ZIP_MODES, zip } from "../../lib/zip"; |
| 27 | |
| 28 | /** The most files, and bytes in all, an archive is made of. */ |
| 29 | const MAX_FILES = 10_000; |
| 30 | // The bytes, their base64 and the archive are all held at once, in a Worker of 128 MB. |
| 31 | const MAX_BYTES = 24 * 1024 * 1024; |
| 32 | /** Blobs read per call to the repository, and calls at once. */ |
| 33 | const PER_READ = 100; |
| 34 | const READS_AT_ONCE = 8; |
| 35 | |
| 36 | const FORMATS = [ |
| 37 | { suffix: ".tar.gz", type: "application/x-gzip" }, |
| 38 | { suffix: ".zip", type: "application/zip" }, |
| 39 | ] as const; |
| 40 | |
| 41 | function refused(status: number, message: string): Response { |
| 42 | return new Response(`${message}\n`, { status, headers: { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" } }); |
| 43 | } |
| 44 | |
| 45 | /** What the archive is of, once whoever asked may have it. */ |
| 46 | type Source = { repoId: string; fullName: string; label: string }; |
| 47 | |
| 48 | export async function loader({ params, context, request }: Route.LoaderArgs) { |
| 49 | const asked = params["*"] ?? ""; |
| 50 | const format = FORMATS.find((candidate) => asked.endsWith(candidate.suffix) && asked.length > candidate.suffix.length); |
| 51 | if (!format) return refused(404, "Ask for <branch, tag or commit>.zip or <branch, tag or commit>.tar.gz."); |
| 52 | const ref = asked.slice(0, -format.suffix.length); |
| 53 | const token = new URL(request.url).searchParams.get("token"); |
| 54 | let source: Source; |
| 55 | if (token) { |
| 56 | // A link from the API: one commit, named as GitHub names its archives. |
| 57 | const grant = await repos.openArchiveToken({ namespace: params.owner, name: params.repo }, token).catch(() => null); |
| 58 | if (!grant || grant.commit !== ref) return refused(403, "This download link has ended or is not for this commit. Ask the API for a new one."); |
| 59 | source = { repoId: grant.repoId, fullName: `${grant.namespace}/${grant.name}`, label: `${grant.namespace}-${grant.name}-${grant.commit.slice(0, 7)}` }; |
| 60 | } else { |
| 61 | const viewer = getViewer(context); |
| 62 | const repo = await repos.get({ namespace: params.owner, name: params.repo }, viewer).catch(() => null); |
| 63 | if (!repo?.ok) return refused(404, "There is no such repository, or you cannot see it."); |
| 64 | // A branch name can hold slashes; the folder and file name cannot. |
| 65 | source = { repoId: repo.value.id, fullName: `${repo.value.namespace}/${repo.value.name}`, label: `${repo.value.name}-${ref.replaceAll("/", "-")}` }; |
| 66 | } |
| 67 | const listed = await repos.listFiles(source.repoId, ref, MAX_FILES + 1, { links: true }).catch(() => null); |
| 68 | if (!listed?.commit) return refused(404, `There is no branch, tag or commit named ${ref}.`); |
| 69 | const clone = `git clone ${cloneUrl(addresses(), source.fullName)}`; |
| 70 | if (listed.truncated || listed.files.length > MAX_FILES) { |
| 71 | return refused(413, `It has more than ${MAX_FILES.toLocaleString("en-US")} files, too many for a download. Clone it instead: ${clone}`); |
| 72 | } |
| 73 | const files = listed.files.filter((file): file is (typeof listed.files)[number] & { hash: string } => file.hash != null); |
| 74 | const unique = [...new Set(files.map((file) => file.hash))]; |
| 75 | const bytes = new Map<string, Uint8Array>(); |
| 76 | let total = 0; |
| 77 | let tooLarge = false; |
| 78 | const batches: string[][] = []; |
| 79 | for (let at = 0; at < unique.length; at += PER_READ) batches.push(unique.slice(at, at + PER_READ)); |
| 80 | // A few reads at a time, each taking the next batch, until all are read or it is too large. |
| 81 | const reader = async () => { |
| 82 | for (let batch = batches.shift(); batch && !tooLarge; batch = batches.shift()) { |
| 83 | for (const blob of await repos.rawBlobs(source.repoId, batch, MAX_BYTES)) { |
| 84 | total += blob.size; |
| 85 | if (total > MAX_BYTES || (blob.data == null && blob.size > 0)) tooLarge = true; |
| 86 | else bytes.set(blob.hash, Uint8Array.from(atob(blob.data ?? ""), (c) => c.charCodeAt(0))); |
| 87 | } |
| 88 | } |
| 89 | }; |
| 90 | await Promise.all(Array.from({ length: READS_AT_ONCE }, reader)); |
| 91 | if (tooLarge) return refused(413, `It is over ${MAX_BYTES / 1024 / 1024} MB, too large for a download. Clone it instead: ${clone}`); |
| 92 | const entries = files.map((file) => ({ path: `${source.label}/${file.path}`, data: bytes.get(file.hash) ?? new Uint8Array(), kind: file.kind })); |
| 93 | const archive = |
| 94 | format.suffix === ".zip" |
| 95 | ? await zip(entries.map((entry) => ({ ...entry, mode: ZIP_MODES[entry.kind ?? "blob"] }))) |
| 96 | : await tarGz(entries, { mtime: listed.date ? Date.parse(listed.date) / 1000 || 0 : 0, comment: listed.commit }); |
| 97 | return new Response(archive, { |
| 98 | headers: { |
| 99 | "content-type": format.type, |
| 100 | // A ref may hold quotes; the header never does. |
| 101 | "content-disposition": contentDisposition(`${source.label}${format.suffix}`), |
| 102 | // A commit's files never change; a branch's do. |
| 103 | "cache-control": /^[0-9a-f]{40}$/.test(ref) ? "private, max-age=31536000, immutable" : "private, no-cache", |
| 104 | }, |
| 105 | }); |
| 106 | } |