Merge branch 'worktree-agent-a19b06d959c105bf7' into claude/cool-newton-7nm8r9
35 files+579−1919 files not counted0/35 viewed
| 1 | + | //! A repository's files at a branch, tag or commit as one archive, at | |
| 2 | + | //! GitHub's addresses: `GET /repos/{owner}/{repo}/tarball/{ref}` and | |
| 3 | + | //! `/zipball/{ref}` (and without a ref, the default branch). | |
| 4 | + | //! | |
| 5 | + | //! As GitHub's redirect to codeload, these answer `302 Found` with a link to | |
| 6 | + | //! the site's archive route (apps/web routes/repo/archive.ts) for the commit | |
| 7 | + | //! the ref names, carrying a token the repos service made for whoever asked | |
| 8 | + | //! (services/repos archive_links.rs). The link is good for 10 minutes and | |
| 9 | + | //! needs no token of its own, so a client following the redirect, which | |
| 10 | + | //! drops its `Authorization` header on the way, gets a private | |
| 11 | + | //! repository's files too. The archive holds one folder, | |
| 12 | + | //! `<owner>-<repo>-<short sha>/`, as GitHub's does: `actions/checkout` | |
| 13 | + | //! downloads one this way in a job container without git, and expects it. | |
| 14 | + | ||
| 15 | + | use g1t_contracts::repos::{ArchiveLink, ArchiveLinkArgs}; | |
| 16 | + | use g1t_contracts::{FailureCode, Outcome, Viewer}; | |
| 17 | + | use serde_json::{Value, json}; | |
| 18 | + | use worker::Result; | |
| 19 | + | ||
| 20 | + | use crate::operations::{Services, repo_path}; | |
| 21 | + | ||
| 22 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq)] | |
| 23 | + | pub enum ArchiveOp { | |
| 24 | + | DownloadTarball, | |
| 25 | + | DownloadZipball, | |
| 26 | + | } | |
| 27 | + | ||
| 28 | + | impl ArchiveOp { | |
| 29 | + | #[cfg(test)] | |
| 30 | + | pub const ALL: [ArchiveOp; 2] = [ArchiveOp::DownloadTarball, ArchiveOp::DownloadZipball]; | |
| 31 | + | ||
| 32 | + | pub fn name(self) -> &'static str { | |
| 33 | + | match self { | |
| 34 | + | ArchiveOp::DownloadTarball => "download_tarball_archive", | |
| 35 | + | ArchiveOp::DownloadZipball => "download_zipball_archive", | |
| 36 | + | } | |
| 37 | + | } | |
| 38 | + | ||
| 39 | + | /// For the API reference. | |
| 40 | + | pub fn title(self) -> &'static str { | |
| 41 | + | match self { | |
| 42 | + | ArchiveOp::DownloadTarball => "Download a repository archive (tar)", | |
| 43 | + | ArchiveOp::DownloadZipball => "Download a repository archive (zip)", | |
| 44 | + | } | |
| 45 | + | } | |
| 46 | + | ||
| 47 | + | pub fn description(self) -> &'static str { | |
| 48 | + | match self { | |
| 49 | + | ArchiveOp::DownloadTarball => "A link to download a branch, tag or commit as a gzipped tar archive (the default branch without ref), good for 10 minutes and needing no token. Over REST, GET …/tarball/{ref} answers 302 with the link in Location, as GitHub does: `curl -L` follows it. Over MCP the link is returned as url, with commit and expires_at. The archive holds one folder, <owner>-<repo>-<first 7 of the commit>/, with executable files and symlinks as git has them and the commit in its pax header. Up to 10,000 files and 24 MB; clone a larger repository. Needs the Read role; a public repository's are open to anyone.", | |
| 50 | + | ArchiveOp::DownloadZipball => "A link to download a branch, tag or commit as a zip archive (the default branch without ref), good for 10 minutes and needing no token. Over REST, GET …/zipball/{ref} answers 302 with the link in Location, as GitHub does: `curl -L` follows it. Over MCP the link is returned as url, with commit and expires_at. The archive holds one folder, <owner>-<repo>-<first 7 of the commit>/. Up to 10,000 files and 24 MB; clone a larger repository. Needs the Read role; a public repository's are open to anyone.", | |
| 51 | + | } | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | fn suffix(self) -> &'static str { | |
| 55 | + | match self { | |
| 56 | + | ArchiveOp::DownloadTarball => "tar.gz", | |
| 57 | + | ArchiveOp::DownloadZipball => "zip", | |
| 58 | + | } | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | pub fn input(self) -> Value { | |
| 62 | + | json!({ | |
| 63 | + | "type": "object", | |
| 64 | + | "properties": { | |
| 65 | + | "repo": { "type": "string", "description": "Repository as \"owner/name\", e.g. \"flagon-io/hello\"." }, | |
| 66 | + | "ref": { "type": "string", "description": "A branch, tag or commit; the default branch when left out. A branch with slashes is sent URL-encoded as one path segment." }, | |
| 67 | + | }, | |
| 68 | + | "required": ["repo"], | |
| 69 | + | }) | |
| 70 | + | } | |
| 71 | + | } | |
| 72 | + | ||
| 73 | + | /// Where the site serves the archive of `link`. | |
| 74 | + | pub fn archive_url(site: &str, link: &ArchiveLink, op: ArchiveOp) -> String { | |
| 75 | + | format!( | |
| 76 | + | "{site}/{}/{}/archive/{}.{}?token={}", | |
| 77 | + | link.repo.namespace, | |
| 78 | + | link.repo.name, | |
| 79 | + | link.commit, | |
| 80 | + | op.suffix(), | |
| 81 | + | link.token | |
| 82 | + | ) | |
| 83 | + | } | |
| 84 | + | ||
| 85 | + | pub async fn run(op: ArchiveOp, services: &Services, viewer: &Viewer, input: &Value) -> Result<Outcome<Value>> { | |
| 86 | + | let Some(path) = repo_path(input) else { | |
| 87 | + | return Ok(Outcome::fail(FailureCode::Invalid, "Give the repository as \"owner/name\".")); | |
| 88 | + | }; | |
| 89 | + | let git_ref = input["ref"].as_str().map(str::trim).filter(|git_ref| !git_ref.is_empty()).map(str::to_owned); | |
| 90 | + | let found: Outcome<ArchiveLink> = g1t_kit::call(&services.repos, "archive_link", &ArchiveLinkArgs { path, viewer: viewer.clone(), git_ref }).await?; | |
| 91 | + | Ok(match found { | |
| 92 | + | Outcome::Ok(link) => Outcome::Ok(json!({ | |
| 93 | + | "url": archive_url(&services.addresses.site, &link, op), | |
| 94 | + | "commit": link.commit, | |
| 95 | + | "expires_at": link.expires_at, | |
| 96 | + | })), | |
| 97 | + | Outcome::Fail(refused) => Outcome::Fail(refused), | |
| 98 | + | }) | |
| 99 | + | } | |
| 100 | + | ||
| 101 | + | #[cfg(test)] | |
| 102 | + | mod tests { | |
| 103 | + | use super::*; | |
| 104 | + | ||
| 105 | + | fn link() -> ArchiveLink { | |
| 106 | + | let repo: g1t_contracts::repos::Repo = serde_json::from_value(json!({ | |
| 107 | + | "id": "repo_1", "namespace": "acme", "name": "web", "description": null, "isPrivate": true, | |
| 108 | + | "ownerId": "ws_1", "defaultBranch": "main", "forkOf": null, "createdAt": "2026-10-01T00:00:00.000Z", | |
| 109 | + | })) | |
| 110 | + | .unwrap(); | |
| 111 | + | ArchiveLink { repo, commit: "4807077b296e6edbf410d55e72749d3e1170c291".into(), token: "abc_-123".into(), expires_at: "2026-10-12T00:10:00.000Z".into() } | |
| 112 | + | } | |
| 113 | + | ||
| 114 | + | #[test] | |
| 115 | + | fn the_link_is_the_sites_archive_of_the_commit() { | |
| 116 | + | let link = link(); | |
| 117 | + | assert_eq!( | |
| 118 | + | archive_url("https://g1t.sh", &link, ArchiveOp::DownloadTarball), | |
| 119 | + | "https://g1t.sh/acme/web/archive/4807077b296e6edbf410d55e72749d3e1170c291.tar.gz?token=abc_-123" | |
| 120 | + | ); | |
| 121 | + | assert_eq!( | |
| 122 | + | archive_url("http://localhost:8787", &link, ArchiveOp::DownloadZipball), | |
| 123 | + | "http://localhost:8787/acme/web/archive/4807077b296e6edbf410d55e72749d3e1170c291.zip?token=abc_-123" | |
| 124 | + | ); | |
| 125 | + | } | |
| 126 | + | ||
| 127 | + | #[test] | |
| 128 | + | fn each_operation_is_described_with_a_schema() { | |
| 129 | + | for op in ArchiveOp::ALL { | |
| 130 | + | assert!(crate::operations::Op::ALL.contains(&crate::operations::Op::Archives(op)), "{}", op.name()); | |
| 131 | + | assert!(!op.title().is_empty() && op.description().len() > 40, "{}", op.name()); | |
| 132 | + | assert_eq!(op.input()["required"], json!(["repo"])); | |
| 133 | + | assert_eq!(g1t_contracts::scopes::scope_for(op.name()), Some(g1t_contracts::scopes::Scope::CodeRead), "{}", op.name()); | |
| 134 | + | } | |
| 135 | + | } | |
| 136 | + | } |
| 6 | 6 | ||
| 7 | 7 | mod about; | |
| 8 | 8 | mod run_artifacts; | |
| 9 | + | mod archives; | |
| 9 | 10 | mod addresses; | |
| 10 | 11 | mod alerts; | |
| 11 | 12 | mod audit; | |
| ⋯ | |||
| 864 | 865 | return fail(FailureCode::NotFound, "No such endpoint."); | |
| 865 | 866 | }; | |
| 866 | 867 | match audit::run(route.op, &services, &viewer, &input).await? { | |
| 867 | − | // A download is a redirect to its signed link, as GitHub's is. | |
| 868 | − | Outcome::Ok(value) if route.op == operations::Op::Artifacts(run_artifacts::ArtifactsOp::DownloadArtifact) => { | |
| 868 | + | // A download is a redirect to its signed link, as GitHub's is: an | |
| 869 | + | // artifact's, and a repository's tarball and zipball. | |
| 870 | + | Outcome::Ok(value) if route.op.redirects() => { | |
| 869 | 871 | match value["url"].as_str().and_then(|url| worker::Url::parse(url).ok()) { | |
| 870 | 872 | Some(url) => Response::redirect_with_status(url, 302), | |
| 871 | 873 | None => reply(&value), | |
| 9 | 9 | ||
| 10 | 10 | use crate::about::AboutOp; | |
| 11 | 11 | use crate::run_artifacts::ArtifactsOp; | |
| 12 | + | use crate::archives::ArchiveOp; | |
| 12 | 13 | use crate::customize::CustomizeOp; | |
| 13 | 14 | use crate::deploy_keys::DeployKeysOp; | |
| 14 | 15 | use crate::mirrors::MirrorsOp; | |
| ⋯ | |||
| 114 | 115 | ), | |
| 115 | 116 | ( | |
| 116 | 117 | "Repositories", | |
| 117 | − | "A repository, how it handles pull requests, and its timeline: renaming, archiving, moving and deleting it.", | |
| 118 | + | "A repository, how it handles pull requests, and its timeline: renaming, archiving, moving and deleting it; and its files at a branch, tag or commit as a tarball or zipball.", | |
| 118 | 119 | &[ | |
| 119 | 120 | Op::ListRepos, | |
| 120 | 121 | Op::CreateRepo, | |
| 121 | 122 | Op::GetRepo, | |
| 123 | + | Op::Archives(ArchiveOp::DownloadTarball), | |
| 124 | + | Op::Archives(ArchiveOp::DownloadZipball), | |
| 122 | 125 | Op::UpdateRepo, | |
| 123 | 126 | Op::RenameRepo, | |
| 124 | 127 | Op::RenameBranch, | |
| ⋯ | |||
| 832 | 835 | Op::Protection(op) => op.title(), | |
| 833 | 836 | Op::Tokens(op) => op.title(), | |
| 834 | 837 | Op::Artifacts(op) => op.title(), | |
| 838 | + | Op::Archives(op) => op.title(), | |
| 835 | 839 | Op::DeployKeys(op) => op.title(), | |
| 836 | 840 | Op::Mirrors(op) => op.title(), | |
| 837 | 841 | Op::Packages(op) => op.title(), | |
| 29 | 29 | use crate::checks::ChecksOp; | |
| 30 | 30 | use crate::about::AboutOp; | |
| 31 | 31 | use crate::run_artifacts::ArtifactsOp; | |
| 32 | + | use crate::archives::ArchiveOp; | |
| 32 | 33 | use crate::customize::CustomizeOp; | |
| 33 | 34 | use crate::deploy_keys::DeployKeysOp; | |
| 34 | 35 | use crate::mirrors::MirrorsOp; | |
| ⋯ | |||
| 328 | 329 | Tokens(TokenOp), | |
| 329 | 330 | /// Workflow run artifacts, and how long they are kept: run_artifacts.rs. | |
| 330 | 331 | Artifacts(ArtifactsOp), | |
| 332 | + | /// A branch, tag or commit as a tarball or zipball: archives.rs. | |
| 333 | + | Archives(ArchiveOp), | |
| 331 | 334 | /// A repository's deploy keys: deploy_keys.rs. | |
| 332 | 335 | DeployKeys(DeployKeysOp), | |
| 333 | 336 | /// A repository's mirroring: its remotes, takeovers and hand-backs: | |
| ⋯ | |||
| 711 | 714 | } | |
| 712 | 715 | ||
| 713 | 716 | impl Op { | |
| 714 | − | pub const ALL: [Op; 380] = [ | |
| 717 | + | pub const ALL: [Op; 382] = [ | |
| 715 | 718 | Op::Whoami, | |
| 716 | 719 | Op::GetWorkspace, | |
| 717 | 720 | Op::CreateWorkspace, | |
| ⋯ | |||
| 1003 | 1006 | Op::Artifacts(ArtifactsOp::DeleteArtifact), | |
| 1004 | 1007 | Op::Artifacts(ArtifactsOp::GetArtifactRetention), | |
| 1005 | 1008 | Op::Artifacts(ArtifactsOp::SetArtifactRetention), | |
| 1009 | + | Op::Archives(ArchiveOp::DownloadTarball), | |
| 1010 | + | Op::Archives(ArchiveOp::DownloadZipball), | |
| 1006 | 1011 | Op::DeployKeys(DeployKeysOp::ListDeployKeys), | |
| 1007 | 1012 | Op::DeployKeys(DeployKeysOp::GetDeployKey), | |
| 1008 | 1013 | Op::DeployKeys(DeployKeysOp::CreateDeployKey), | |
| ⋯ | |||
| 1306 | 1311 | Op::Protection(op) => op.name(), | |
| 1307 | 1312 | Op::Tokens(op) => op.name(), | |
| 1308 | 1313 | Op::Artifacts(op) => op.name(), | |
| 1314 | + | Op::Archives(op) => op.name(), | |
| 1309 | 1315 | Op::DeployKeys(op) => op.name(), | |
| 1310 | 1316 | Op::Mirrors(op) => op.name(), | |
| 1311 | 1317 | Op::Packages(op) => op.name(), | |
| ⋯ | |||
| 1879 | 1885 | Op::Protection(op) => op.description(), | |
| 1880 | 1886 | Op::Tokens(op) => op.description(), | |
| 1881 | 1887 | Op::Artifacts(op) => op.description(), | |
| 1888 | + | Op::Archives(op) => op.description(), | |
| 1882 | 1889 | Op::DeployKeys(op) => op.description(), | |
| 1883 | 1890 | Op::Mirrors(op) => op.description(), | |
| 1884 | 1891 | Op::Packages(op) => op.description(), | |
| ⋯ | |||
| 3373 | 3380 | Op::Protection(op) => op.input(), | |
| 3374 | 3381 | Op::Tokens(op) => op.input(), | |
| 3375 | 3382 | Op::Artifacts(op) => op.input(), | |
| 3383 | + | Op::Archives(op) => op.input(), | |
| 3376 | 3384 | Op::DeployKeys(op) => op.input(), | |
| 3377 | 3385 | Op::Mirrors(op) => op.input(), | |
| 3378 | 3386 | Op::Packages(op) => op.input(), | |
| ⋯ | |||
| 3381 | 3389 | } | |
| 3382 | 3390 | } | |
| 3383 | 3391 | ||
| 3392 | + | /// Whether REST answers with a redirect to the `url` it returns, as | |
| 3393 | + | /// GitHub's downloads do. | |
| 3394 | + | pub(crate) fn redirects(self) -> bool { | |
| 3395 | + | matches!(self, Op::Artifacts(ArtifactsOp::DownloadArtifact) | Op::Archives(_)) | |
| 3396 | + | } | |
| 3397 | + | ||
| 3384 | 3398 | /// Whether the operation refuses an anonymous caller outright. | |
| 3385 | 3399 | pub(crate) fn needs_user(self) -> bool { | |
| 3386 | 3400 | // A public repository's checks are anyone's to read. | |
| ⋯ | |||
| 3394 | 3408 | if let Op::Artifacts(op) = self { | |
| 3395 | 3409 | return op.writes(); | |
| 3396 | 3410 | } | |
| 3411 | + | // And its archives. | |
| 3412 | + | if let Op::Archives(_) = self { | |
| 3413 | + | return false; | |
| 3414 | + | } | |
| 3397 | 3415 | // So are public packages. | |
| 3398 | 3416 | if let Op::Packages(op) = self { | |
| 3399 | 3417 | return !op.anonymous(); | |
| ⋯ | |||
| 5733 | 5751 | Op::Protection(op) => crate::protection::run(op, services, viewer, input).await, | |
| 5734 | 5752 | Op::Tokens(op) => crate::token_policy::run(op, services, viewer, input).await, | |
| 5735 | 5753 | Op::Artifacts(op) => crate::run_artifacts::run(op, services, viewer, input).await, | |
| 5754 | + | Op::Archives(op) => crate::archives::run(op, services, viewer, input).await, | |
| 5736 | 5755 | Op::DeployKeys(op) => crate::deploy_keys::run(op, services, viewer, input).await, | |
| 5737 | 5756 | Op::Mirrors(op) => crate::mirrors::run(op, services, viewer, input).await, | |
| 5738 | 5757 | Op::Packages(op) => crate::packages::run(op, services, viewer, input).await, | |
| 13211 | 13211 | }, | |
| 13212 | 13212 | "notes": "Over REST the answer is `302 Found` with the link in `Location`, as GitHub's is, so `curl -L -o web-dist.zip …/zip` saves the file; the body above is what the MCP `workflow` tool's `download_artifact` returns. The link needs no token and stops working after 10 minutes." | |
| 13213 | 13213 | }, | |
| 13214 | + | "download_tarball_archive": { | |
| 13215 | + | "params": { | |
| 13216 | + | "owner": "flagon-io", | |
| 13217 | + | "name": "hello", | |
| 13218 | + | "ref": "main" | |
| 13219 | + | }, | |
| 13220 | + | "response": { | |
| 13221 | + | "url": "https://g1t.sh/flagon-io/hello/archive/7c1e9a4b2d6f80135ac9e2b7d4f6a8c0e1b3d5f7.tar.gz?token=…", | |
| 13222 | + | "commit": "7c1e9a4b2d6f80135ac9e2b7d4f6a8c0e1b3d5f7", | |
| 13223 | + | "expires_at": "2026-10-12T15:13:00.000Z" | |
| 13224 | + | }, | |
| 13225 | + | "notes": "Over REST the answer is `302 Found` with the link in `Location`, as GitHub's is, so `curl -L -H \"Authorization: Bearer g1t_…\" -o hello.tar.gz https://api.g1t.sh/repos/flagon-io/hello/tarball/main` saves the file and `tar -xzf hello.tar.gz` makes one folder, `flagon-io-hello-7c1e9a4/`; the body above is what the MCP `repository` tool's `tarball` returns. The link is for the commit the ref named when you asked, needs no token and stops working after 10 minutes. `actions/checkout` uses this route in a job container without git." | |
| 13226 | + | }, | |
| 13227 | + | "download_zipball_archive": { | |
| 13228 | + | "params": { | |
| 13229 | + | "owner": "flagon-io", | |
| 13230 | + | "name": "hello", | |
| 13231 | + | "ref": "v1.2.0" | |
| 13232 | + | }, | |
| 13233 | + | "response": { | |
| 13234 | + | "url": "https://g1t.sh/flagon-io/hello/archive/7c1e9a4b2d6f80135ac9e2b7d4f6a8c0e1b3d5f7.zip?token=…", | |
| 13235 | + | "commit": "7c1e9a4b2d6f80135ac9e2b7d4f6a8c0e1b3d5f7", | |
| 13236 | + | "expires_at": "2026-10-12T15:13:00.000Z" | |
| 13237 | + | }, | |
| 13238 | + | "notes": "Over REST the answer is `302 Found` with the link in `Location`, as GitHub's is, so `curl -L -o hello.zip …/zipball/v1.2.0` saves the file, with one folder, `flagon-io-hello-7c1e9a4/`; the body above is what the MCP `repository` tool's `zipball` returns. The link needs no token and stops working after 10 minutes." | |
| 13239 | + | }, | |
| 13214 | 13240 | "delete_artifact": { | |
| 13215 | 13241 | "params": { | |
| 13216 | 13242 | "owner": "flagon-io", |
| 119 | 119 | Op::Deployments(_) => return as_is, | |
| 120 | 120 | // Artifacts are shaped by the API itself, in `snake_case`. | |
| 121 | 121 | Op::Artifacts(_) => return as_is, | |
| 122 | + | // So are archive links. | |
| 123 | + | Op::Archives(_) => return as_is, | |
| 122 | 124 | // Deploy keys travel in `snake_case` from identity. | |
| 123 | 125 | Op::DeployKeys(DeployKeysOp::ListDeployKeys) => return through::<Vec<g1t_contracts::deploy_keys::DeployKey>>(op, as_is), | |
| 124 | 126 | Op::DeployKeys(DeployKeysOp::GetDeployKey | DeployKeysOp::CreateDeployKey) => { |
| 4 | 4 | ||
| 5 | 5 | use crate::about::AboutOp; | |
| 6 | 6 | use crate::run_artifacts::ArtifactsOp; | |
| 7 | + | use crate::archives::ArchiveOp; | |
| 7 | 8 | use crate::customize::CustomizeOp; | |
| 8 | 9 | use crate::deploy_keys::DeployKeysOp; | |
| 9 | 10 | use crate::mirrors::MirrorsOp; | |
| ⋯ | |||
| 274 | 275 | ), | |
| 275 | 276 | route("POST", "/repos", Op::CreateRepo, &[]), | |
| 276 | 277 | route("GET", "/repos/:owner/:name", Op::GetRepo, &[]), | |
| 278 | + | // A branch, tag or commit as one archive: a redirect to a link good for | |
| 279 | + | // ten minutes, as GitHub's go to codeload. | |
| 280 | + | route("GET", "/repos/:owner/:name/tarball/:ref", Op::Archives(ArchiveOp::DownloadTarball), &[]), | |
| 281 | + | route("GET", "/repos/:owner/:name/zipball/:ref", Op::Archives(ArchiveOp::DownloadZipball), &[]), | |
| 277 | 282 | route("PATCH", "/repos/:owner/:name", Op::UpdateRepo, &[]), | |
| 278 | 283 | route("POST", "/repos/:owner/:name/transfer", Op::TransferRepo, &[]), | |
| 279 | 284 | route("DELETE", "/repos/:owner/:name", Op::DeleteRepo, &[]), | |
| ⋯ | |||
| 1437 | 1442 | } | |
| 1438 | 1443 | ||
| 1439 | 1444 | #[test] | |
| 1445 | + | fn archives_are_at_githubs_addresses_and_redirect() { | |
| 1446 | + | let sha = "4807077b296e6edbf410d55e72749d3e1170c291"; | |
| 1447 | + | let (route, input) = resolve("GET", &format!("/repos/acme/web/tarball/{sha}"), &[], Value::Null).unwrap(); | |
| 1448 | + | assert_eq!(route.op, Op::Archives(ArchiveOp::DownloadTarball)); | |
| 1449 | + | assert_eq!(input, json!({ "ref": sha, "repo": "acme/web" })); | |
| 1450 | + | // A branch with slashes, URL-encoded as octokit sends it. | |
| 1451 | + | let (route, input) = resolve("GET", "/repos/acme/web/zipball/release%2F1.x", &[], Value::Null).unwrap(); | |
| 1452 | + | assert_eq!(route.op, Op::Archives(ArchiveOp::DownloadZipball)); | |
| 1453 | + | assert_eq!(input, json!({ "ref": "release/1.x", "repo": "acme/web" })); | |
| 1454 | + | // Both answer with a redirect to their link, as the artifact download does. | |
| 1455 | + | assert!(route.op.redirects()); | |
| 1456 | + | assert!(Op::Archives(ArchiveOp::DownloadTarball).redirects()); | |
| 1457 | + | assert!(Op::Artifacts(ArtifactsOp::DownloadArtifact).redirects()); | |
| 1458 | + | assert!(!Op::GetRepo.redirects()); | |
| 1459 | + | // Anyone may ask for a public repository's. | |
| 1460 | + | assert!(!route.op.needs_user()); | |
| 1461 | + | } | |
| 1462 | + | ||
| 1463 | + | #[test] | |
| 1440 | 1464 | fn query_parameters_are_renamed() { | |
| 1441 | 1465 | let query = [ | |
| 1442 | 1466 | ("q".to_owned(), "parser".to_owned()), | |
| 20 | 20 | ||
| 21 | 21 | use crate::about::AboutOp; | |
| 22 | 22 | use crate::run_artifacts::ArtifactsOp; | |
| 23 | + | use crate::archives::ArchiveOp; | |
| 23 | 24 | use crate::customize::CustomizeOp; | |
| 24 | 25 | use crate::deploy_keys::DeployKeysOp; | |
| 25 | 26 | use crate::mirrors::MirrorsOp; | |
| ⋯ | |||
| 75 | 76 | actions: &[ | |
| 76 | 77 | a("list", Op::ListRepos, "Repositories you can see"), | |
| 77 | 78 | a("get", Op::GetRepo, "One repository"), | |
| 79 | + | a("tarball", Op::Archives(ArchiveOp::DownloadTarball), "A 10-minute link to a branch, tag or commit as a .tar.gz"), | |
| 80 | + | a("zipball", Op::Archives(ArchiveOp::DownloadZipball), "A 10-minute link to a branch, tag or commit as a .zip"), | |
| 78 | 81 | a("create", Op::CreateRepo, "Create one, empty or copied from a public git URL"), | |
| 79 | 82 | a("update", Op::UpdateRepo, "Change description, website, topics, default branch, protection"), | |
| 80 | 83 | a("get_settings", Op::GetRepoSettings, "How pull requests merge, and the default branch's protection as its rules stack"), | |
| 273 | 273 | register with `--no-job-docker`. | |
| 274 | 274 | - **Status.** Not scheduled. | |
| 275 | 275 | ||
| 276 | − | ### `actions/checkout` without git, or with Git LFS | |
| 276 | + | ### `actions/checkout` with Git LFS, or without git on a large repository | |
| 277 | 277 | ||
| 278 | − | `actions/checkout` is the action itself. In a job container whose image | |
| 279 | − | has no git, it downloads the repository as a tarball from | |
| 280 | − | `GET /repos/{owner}/{repo}/tarball/{ref}`, which g1t does not serve yet, | |
| 281 | − | and fails. With `lfs: true`, it finds no LFS files: g1t keeps none. | |
| 278 | + | `actions/checkout` is the action itself. With `lfs: true`, it finds no | |
| 279 | + | LFS files: g1t keeps none. In a job container whose image has no git, it | |
| 280 | + | downloads the commit as a tarball from | |
| 281 | + | `GET /repos/{owner}/{repo}/tarball/{ref}`, as on GitHub. g1t makes that | |
| 282 | + | tarball when it is asked, of up to 10,000 files and 24 MB, so the | |
| 283 | + | checkout of a larger repository fails there. | |
| 282 | 284 | ||
| 283 | − | - **Instead.** Use an image with git, or install git in a step before the | |
| 284 | − | checkout. Keep large files out of Git LFS for now. | |
| 285 | + | - **Why.** The tarball is made in a Worker, which holds the files and the | |
| 286 | + | archive in memory at once. | |
| 287 | + | - **Instead.** Keep large files out of Git LFS for now. For a large | |
| 288 | + | repository, use an image with git, or install git in a step before the | |
| 289 | + | checkout. | |
| 285 | 290 | - **Status.** Planned. | |
| 286 | 291 | ||
| 287 | 292 | ### No `gh` command in jobs |
| 66 | 66 | | JavaScript actions (`uses: owner/repo@v7`, `owner/repo/path@v7`) | Fetched from that repository on g1t when g1t has it and your repository may use it, otherwise from GitHub, and run as they are, on Node 24, the runtime current actions declare. An action that declares `node20` runs on Node 20 where the machine has one. See [actions and workflows from other repositories](#actions-and-workflows-from-other-repositories) and [an action's pre and post](#an-actions-pre-and-post). | | |
| 67 | 67 | | Composite actions | The same. | | |
| 68 | 68 | | Reusable workflows (`jobs.<id>.uses: ./.g1t/workflows/build.yml`, or `owner/repo/.g1t/workflows/build.yml@v1` in another repository) | The same: `with:` inputs checked against their `type`, `secrets:` by name or `secrets: inherit`, `on.workflow_call` outputs, `strategy.matrix` on the calling job (one call for each combination), `concurrency` on the calling job, and up to ten levels of workflows with at most 50 different reusable workflows in a run. `.github/workflows/…` finds the workflow under `.g1t/` after the move. See [actions and workflows from other repositories](#actions-and-workflows-from-other-repositories). | | |
| 69 | − | | `actions/checkout` | The action itself, at the version the step names, fetching from g1t with the job's token: every input and output, its post step that removes the credential, and a pull request's [test merge commit](#what-a-pull-requests-run-builds), which g1t keeps in the repository. `lfs` finds no Git LFS files: g1t keeps none. | | |
| 69 | + | | `actions/checkout` | The action itself, at the version the step names, fetching from g1t with the job's token: every input and output, its post step that removes the credential, and a pull request's [test merge commit](#what-a-pull-requests-run-builds), which g1t keeps in the repository. In a job container without git it downloads the commit as a tarball from [`GET /repos/{owner}/{repo}/tarball/{ref}`](/reference/api/repositories/download-tarball-archive/), as on GitHub, up to 10,000 files and 24 MB. `lfs` finds no Git LFS files: g1t keeps none. | | |
| 70 | 70 | | `GITHUB_OUTPUT`, `GITHUB_ENV`, `GITHUB_PATH`, `GITHUB_STATE`, `GITHUB_STEP_SUMMARY` | The same. Step summaries show on the run's page; see [job summaries](#job-summaries). | | |
| 71 | 71 | | `::error::`, `::warning::`, `::notice::`, `::group::`, `::add-mask::`, `::echo::on` | The same: errors and warnings become annotations on the run, with their `file`, `line`, `col`, `endLine` and `endColumn`, and [masked](#masking-secrets) values stay hidden. | | |
| 72 | 72 | | Problem matchers (`::add-matcher::`, `::remove-matcher::`) | The same, so `actions/setup-node`, `setup-go` and `setup-python` turn compiler and linter output into annotations. See [problem matchers](#problem-matchers). | | |
| ⋯ | |||
| 105 | 105 | the `pull_request` types `unassigned`, `locked` and `unlocked`, `edited` | |
| 106 | 106 | for a new title or description, or the `issues` types `unassigned`, | |
| 107 | 107 | `locked`, `unlocked`, `pinned` and `unpinned`. | |
| 108 | − | - **`actions/checkout` in a job container without git.** The action | |
| 109 | − | then downloads a tarball through the API, which g1t does not serve yet. | |
| 110 | − | Use an image with git, or install it in an earlier step. | |
| 111 | 108 | - **Background steps.** A step with `background: true`, or a `wait`, | |
| 112 | 109 | `wait-all`, `cancel` or `parallel` step, is refused: g1t does not run | |
| 113 | 110 | them yet. | |
| 76 | 76 | On a repository's **Files** page, **Code** → **Download ZIP** downloads the | |
| 77 | 77 | branch shown as one zip, its files in a folder named `<repo>-<branch>`. It | |
| 78 | 78 | works for anyone who can see the repository, and for any branch, tag or | |
| 79 | − | commit at `g1t.sh/<workspace>/<repo>/archive/<ref>.zip`. A ZIP holds the | |
| 80 | − | files, not the history; clone for that. A repository with more than 10,000 | |
| 81 | − | files or over 24 MB is too large to download this way, so clone it instead. | |
| 79 | + | commit at `g1t.sh/<workspace>/<repo>/archive/<ref>.zip`, or as a gzipped | |
| 80 | + | tar at `…/archive/<ref>.tar.gz`. Executable files and symlinks stay as | |
| 81 | + | they are in git. An archive holds the files, not the history; clone for | |
| 82 | + | that. A repository with more than 10,000 files or over 24 MB is too large | |
| 83 | + | to download this way, so clone it instead. | |
| 82 | 84 | ||
| 85 | + | The API serves the same archives at GitHub's addresses, | |
| 86 | + | [`GET /repos/{owner}/{repo}/tarball/{ref}`](/reference/api/repositories/download-tarball-archive/) | |
| 87 | + | and [`/zipball/{ref}`](/reference/api/repositories/download-zipball-archive/): | |
| 88 | + | each answers with a redirect to a link that needs no token, good for 10 | |
| 89 | + | minutes, for the commit the ref names. Its files are in one folder named | |
| 90 | + | as GitHub names it, `<workspace>-<repo>-<first 7 of the commit>`, which is | |
| 91 | + | what `actions/checkout` expects when it downloads a repository in a job | |
| 92 | + | container without git. With a token, a private repository's work too: | |
| 93 | + | ||
| 94 | + | ```sh | |
| 95 | + | curl -L -H "Authorization: Bearer $G1T_TOKEN" -o hello.tar.gz \ | |
| 96 | + | https://api.g1t.sh/repos/flagon-io/hello/tarball/main | |
| 97 | + | ``` | |
| 98 | + | ||
| 83 | 99 | ## Raw files | |
| 84 | 100 | ||
| 85 | 101 | **Raw**, above a file on its page, opens the file as it is, with nothing | |
| ⋯ | |||
| 197 | 213 | made, whoever asked for it. Pushes to a fork | |
| 198 | 214 | update the pull request's head commit on its page. | |
| 199 | 215 | ||
| 216 | + | ## Pull request refs | |
| 217 | + | ||
| 218 | + | g1t keeps `refs/pull/<number>/head` and `refs/pull/<number>/merge` in a | |
| 219 | + | repository for each open pull request (see | |
| 220 | + | [What a pull request's run builds](/guides/actions/#what-a-pull-requests-run-builds)). You can | |
| 221 | + | fetch them like any branch, and a `git clone --mirror` gets them, as on | |
| 222 | + | GitHub: | |
| 223 | + | ||
| 224 | + | ```sh | |
| 225 | + | git fetch origin refs/pull/12/head | |
| 226 | + | ``` | |
| 227 | + | ||
| 228 | + | They are hidden refs, as on GitHub: only g1t moves them. A push leaves | |
| 229 | + | them out of what it reads from the repository, and a push that names one | |
| 230 | + | is refused for that ref alone, with the rest of the push applied. So | |
| 231 | + | `git push --mirror` of a repository you cloned with `--mirror` from | |
| 232 | + | GitHub moves every branch and tag and reports only the pull request refs: | |
| 233 | + | ||
| 234 | + | ```text | |
| 235 | + | ! [remote rejected] refs/pull/1/head -> refs/pull/1/head (deny updating a hidden ref) | |
| 236 | + | ``` | |
| 237 | + | ||
| 238 | + | Git then says some refs failed to push; the branches and tags are there. | |
| 239 | + | To leave them out, push branches and tags alone: | |
| 240 | + | `git push origin --all && git push origin --tags`. A push with `--atomic` | |
| 241 | + | that names one is refused whole. | |
| 242 | + | ||
| 200 | 243 | ## Limits | |
| 201 | 244 | ||
| 202 | 245 | ### Size limits | |
| 247 | 247 | | --- | --- | --- | --- | | |
| 248 | 248 | | [`list`](/reference/api/repositories/list-repos/) | Repositories you can see, optionally filtered by `query`. | None | `repo:read` | | |
| 249 | 249 | | [`get`](/reference/api/repositories/get-repo/) | One repository's details. | `repo` | `repo:read` | | |
| 250 | + | | [`tarball`](/reference/api/repositories/download-tarball-archive/) | A link to a branch, tag or commit (`ref`, the default branch when left out) as a `.tar.gz` that needs no token, good for 10 minutes, as `url`, with the `commit` it is of. | `repo` | `code:read` | | |
| 251 | + | | [`zipball`](/reference/api/repositories/download-zipball-archive/) | The same as a `.zip`. | `repo` | `code:read` | | |
| 250 | 252 | | [`create`](/reference/api/repositories/create-repo/) | Create a repository in one of your workspaces, empty or as a copy of a public git repository (`import_url`). `workspace` may be left out if you belong to exactly one. | `name` | `repo:write` | | |
| 251 | 253 | | [`update`](/reference/api/repositories/update-repo/) | Change its `description`, `website`, `topics` and `default_branch`, whether its default branch is `protected`, and whether it is `private`. Maintain role; `protected`, `private` and `default_branch` need Admin, and `private` the [member privileges](/guides/workspaces/#member-privileges) unless you are an owner. | `repo` | `repo:write` | | |
| 252 | 254 | | [`get_settings`](/reference/api/repositories/get-repo-settings/) | How it handles pull requests: how g1t's agents are reviewed, revised and merged, and the default branch's required checks, approvals, bypassing checks, being up to date and merge queue as its [rulesets](/guides/rules/) stack. | `repo` | `repo:read` | |
Binary or large file; its contents are not shown.
| 1 | + | import assert from "node:assert/strict"; | |
| 2 | + | import { execFileSync } from "node:child_process"; | |
| 3 | + | import { lstatSync, mkdtempSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs"; | |
| 4 | + | import { tmpdir } from "node:os"; | |
| 5 | + | import { join } from "node:path"; | |
| 6 | + | import { test } from "node:test"; | |
| 7 | + | import { gunzipSync } from "node:zlib"; | |
| 8 | + | ||
| 9 | + | import { paxRecord, splitPath, tar, tarGz } from "./tar.ts"; | |
| 10 | + | ||
| 11 | + | const text = (value: string) => new TextEncoder().encode(value); | |
| 12 | + | const SHA = "4807077b296e6edbf410d55e72749d3e1170c291"; | |
| 13 | + | const ROOT = "acme-rocket-4807077"; | |
| 14 | + | const LONG = `${ROOT}/${"deep/".repeat(40)}${"n".repeat(120)}.txt`; | |
| 15 | + | ||
| 16 | + | const entries = [ | |
| 17 | + | { path: `${ROOT}/README.md`, data: text("hello\n") }, | |
| 18 | + | { path: `${ROOT}/bin/run`, data: text("#!/bin/sh\necho hi\n"), kind: "exec" as const }, | |
| 19 | + | { path: `${ROOT}/latest`, data: text("README.md"), kind: "symlink" as const }, | |
| 20 | + | { path: `${ROOT}/empty`, data: new Uint8Array() }, | |
| 21 | + | { path: LONG, data: text("far down\n") }, | |
| 22 | + | { path: `${ROOT}/${"p".repeat(90)}/${"q".repeat(90)}.md`, data: text("split\n") }, | |
| 23 | + | ]; | |
| 24 | + | ||
| 25 | + | /** Whether `tar` can be run here. */ | |
| 26 | + | function hasTar(): boolean { | |
| 27 | + | try { | |
| 28 | + | execFileSync("tar", ["--version"], { stdio: "ignore" }); | |
| 29 | + | return true; | |
| 30 | + | } catch { | |
| 31 | + | return false; | |
| 32 | + | } | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | test("a pax record's length counts itself", () => { | |
| 36 | + | const record = new TextDecoder().decode(paxRecord("comment", text(SHA))); | |
| 37 | + | assert.equal(record, `52 comment=${SHA}\n`); | |
| 38 | + | assert.equal(record.length, 52); | |
| 39 | + | // Where adding the length's digits adds a digit. | |
| 40 | + | const value = "x".repeat(91); | |
| 41 | + | const near = new TextDecoder().decode(paxRecord("path", text(value))); | |
| 42 | + | assert.equal(Number(near.split(" ")[0]), near.length); | |
| 43 | + | }); | |
| 44 | + | ||
| 45 | + | test("a long path splits into ustar's prefix and name when it can", () => { | |
| 46 | + | assert.deepEqual(splitPath(text("a/b")), { name: text("a/b"), prefix: new Uint8Array() }); | |
| 47 | + | const split = splitPath(text(`${"p".repeat(90)}/${"q".repeat(90)}`))!; | |
| 48 | + | assert.equal(new TextDecoder().decode(split.prefix), "p".repeat(90)); | |
| 49 | + | assert.equal(new TextDecoder().decode(split.name), "q".repeat(90)); | |
| 50 | + | assert.equal(splitPath(text(`a/${"n".repeat(120)}`)), null); | |
| 51 | + | assert.equal(splitPath(text(`${"p".repeat(200)}/b`)), null); | |
| 52 | + | }); | |
| 53 | + | ||
| 54 | + | test("headers are ustar, with the commit in a pax global header", () => { | |
| 55 | + | const out = tar(entries, { mtime: 1_700_000_000, comment: SHA }); | |
| 56 | + | assert.equal(out.length % 512, 0); | |
| 57 | + | const decoder = new TextDecoder(); | |
| 58 | + | // The global header first, as git archive writes it. | |
| 59 | + | assert.equal(decoder.decode(out.subarray(0, 17)), "pax_global_header"); | |
| 60 | + | assert.equal(String.fromCharCode(out[156]!), "g"); | |
| 61 | + | assert.equal(decoder.decode(out.subarray(257, 263)), "ustar\0"); | |
| 62 | + | assert.equal(decoder.decode(out.subarray(512, 564)), `52 comment=${SHA}\n`); | |
| 63 | + | // Every header's checksum holds. | |
| 64 | + | const second = out.subarray(1024, 1536); | |
| 65 | + | const stored = parseInt(decoder.decode(second.subarray(148, 154)), 8); | |
| 66 | + | let sum = 0; | |
| 67 | + | for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : second[i]!; | |
| 68 | + | assert.equal(stored, sum); | |
| 69 | + | // Ends in two zero blocks. | |
| 70 | + | assert.ok(out.subarray(out.length - 1024).every((byte) => byte === 0)); | |
| 71 | + | }); | |
| 72 | + | ||
| 73 | + | test("tar reads the archive back: files, folders, modes, a symlink and long paths", { skip: !hasTar() && "no tar here" }, async () => { | |
| 74 | + | const dir = mkdtempSync(join(tmpdir(), "g1t-tar-")); | |
| 75 | + | try { | |
| 76 | + | const file = join(dir, "archive.tar.gz"); | |
| 77 | + | const archive = await tarGz(entries, { mtime: 1_700_000_000, comment: SHA }); | |
| 78 | + | // A gzip stream around the tar. | |
| 79 | + | assert.equal(gunzipSync(archive).length % 512, 0); | |
| 80 | + | writeFileSync(file, archive); | |
| 81 | + | const listed = execFileSync("tar", ["-tzf", file], { encoding: "utf8" }).trim().split("\n"); | |
| 82 | + | assert.equal(listed[0], `${ROOT}/`); | |
| 83 | + | for (const entry of entries) assert.ok(listed.includes(entry.path), entry.path); | |
| 84 | + | assert.ok(listed.includes(`${ROOT}/bin/`)); | |
| 85 | + | // One top-level folder, as actions/checkout expects. | |
| 86 | + | assert.deepEqual([...new Set(listed.map((path) => path.split("/")[0]))], [ROOT]); | |
| 87 | + | const out = join(dir, "out"); | |
| 88 | + | execFileSync("mkdir", [out]); | |
| 89 | + | execFileSync("tar", ["-xzf", file, "-C", out]); | |
| 90 | + | assert.equal(readFileSync(join(out, ROOT, "README.md"), "utf8"), "hello\n"); | |
| 91 | + | assert.equal(readFileSync(join(out, LONG), "utf8"), "far down\n"); | |
| 92 | + | assert.equal(readFileSync(join(out, ROOT, "empty"), "utf8"), ""); | |
| 93 | + | assert.equal(lstatSync(join(out, ROOT, "bin/run")).mode & 0o111, 0o111); | |
| 94 | + | assert.equal(lstatSync(join(out, ROOT, "README.md")).mode & 0o111, 0); | |
| 95 | + | assert.ok(lstatSync(join(out, ROOT, "latest")).isSymbolicLink()); | |
| 96 | + | assert.equal(readlinkSync(join(out, ROOT, "latest")), "README.md"); | |
| 97 | + | // Dated by the commit. | |
| 98 | + | assert.equal(Math.round(lstatSync(join(out, ROOT, "README.md")).mtimeMs / 1000), 1_700_000_000); | |
| 99 | + | // git's own reader finds the commit. | |
| 100 | + | try { | |
| 101 | + | const id = execFileSync("sh", ["-c", `gzip -dc "${file}" | git get-tar-commit-id`], { encoding: "utf8" }).trim(); | |
| 102 | + | assert.equal(id, SHA); | |
| 103 | + | } catch (error) { | |
| 104 | + | if ((error as { status?: number }).status === 127) return; | |
| 105 | + | throw error; | |
| 106 | + | } | |
| 107 | + | } finally { | |
| 108 | + | rmSync(dir, { recursive: true, force: true }); | |
| 109 | + | } | |
| 110 | + | }); |
| 1 | + | /** | |
| 2 | + | * A gzipped tar archive of files, made in memory, as `git archive` makes | |
| 3 | + | * one: POSIX ustar headers, a pax global header carrying the commit | |
| 4 | + | * (`comment=<sha>`, which `git get-tar-commit-id` reads), a directory | |
| 5 | + | * entry before each folder's files, executable files and symlinks kept as | |
| 6 | + | * they are in git, and a pax extended header for a path or link target too | |
| 7 | + | * long for ustar. Everything is owned by root and dated by the commit. | |
| 8 | + | * Enough for downloading a commit: each file stays under 8 GB (the caller | |
| 9 | + | * caps them far below that). | |
| 10 | + | */ | |
| 11 | + | ||
| 12 | + | export type TarEntry = { | |
| 13 | + | /** The path in the archive, folders separated by `/`. */ | |
| 14 | + | path: string; | |
| 15 | + | /** The file's bytes; for a symlink, where it points. */ | |
| 16 | + | data: Uint8Array; | |
| 17 | + | kind?: "blob" | "exec" | "symlink"; | |
| 18 | + | }; | |
| 19 | + | ||
| 20 | + | export type TarOptions = { | |
| 21 | + | /** Seconds since the epoch, what every entry is dated. */ | |
| 22 | + | mtime: number; | |
| 23 | + | /** The commit, kept in a pax global header as `git archive` keeps it. */ | |
| 24 | + | comment?: string; | |
| 25 | + | }; | |
| 26 | + | ||
| 27 | + | const BLOCK = 512; | |
| 28 | + | const encoder = new TextEncoder(); | |
| 29 | + | ||
| 30 | + | /** `git archive`'s modes, with its default umask of 002. */ | |
| 31 | + | const MODES = { blob: 0o664, exec: 0o775, symlink: 0o777, dir: 0o775 } as const; | |
| 32 | + | ||
| 33 | + | /** `value` in octal, zero-padded to fill `width` bytes with a NUL at the end. */ | |
| 34 | + | function octal(value: number, width: number): string { | |
| 35 | + | return value.toString(8).padStart(width - 1, "0") + "\0"; | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | function put(header: Uint8Array, at: number, width: number, value: string | Uint8Array) { | |
| 39 | + | const bytes = typeof value === "string" ? encoder.encode(value) : value; | |
| 40 | + | header.set(bytes.subarray(0, width), at); | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | /** | |
| 44 | + | * `path` as ustar's `name` (100 bytes) and `prefix` (155), split at a | |
| 45 | + | * slash; null when it does not fit. | |
| 46 | + | */ | |
| 47 | + | export function splitPath(path: Uint8Array): { name: Uint8Array; prefix: Uint8Array } | null { | |
| 48 | + | if (path.length <= 100) return { name: path, prefix: new Uint8Array() }; | |
| 49 | + | // The last slash that leaves at most 100 bytes after it and 155 before. | |
| 50 | + | for (let at = Math.min(path.length - 1, 155); at > 0; at--) { | |
| 51 | + | if (path[at] !== 0x2f) continue; | |
| 52 | + | if (path.length - at - 1 > 100) return null; | |
| 53 | + | if (path.length - at - 1 === 0) continue; | |
| 54 | + | return { name: path.subarray(at + 1), prefix: path.subarray(0, at) }; | |
| 55 | + | } | |
| 56 | + | return null; | |
| 57 | + | } | |
| 58 | + | ||
| 59 | + | type Header = { name: Uint8Array; prefix?: Uint8Array; mode: number; size: number; mtime: number; type: string; link?: Uint8Array }; | |
| 60 | + | ||
| 61 | + | function header(entry: Header): Uint8Array { | |
| 62 | + | const out = new Uint8Array(BLOCK); | |
| 63 | + | put(out, 0, 100, entry.name); | |
| 64 | + | put(out, 100, 8, octal(entry.mode, 8)); | |
| 65 | + | put(out, 108, 8, octal(0, 8)); | |
| 66 | + | put(out, 116, 8, octal(0, 8)); | |
| 67 | + | put(out, 124, 12, octal(entry.size, 12)); | |
| 68 | + | put(out, 136, 12, octal(entry.mtime, 12)); | |
| 69 | + | put(out, 148, 8, " "); | |
| 70 | + | put(out, 156, 1, entry.type); | |
| 71 | + | if (entry.link) put(out, 157, 100, entry.link); | |
| 72 | + | put(out, 257, 6, "ustar\0"); | |
| 73 | + | put(out, 263, 2, "00"); | |
| 74 | + | put(out, 265, 32, "root"); | |
| 75 | + | put(out, 297, 32, "root"); | |
| 76 | + | put(out, 329, 8, octal(0, 8)); | |
| 77 | + | put(out, 337, 8, octal(0, 8)); | |
| 78 | + | if (entry.prefix) put(out, 345, 155, entry.prefix); | |
| 79 | + | let sum = 0; | |
| 80 | + | for (const byte of out) sum += byte; | |
| 81 | + | put(out, 148, 8, sum.toString(8).padStart(6, "0") + "\0 "); | |
| 82 | + | return out; | |
| 83 | + | } | |
| 84 | + | ||
| 85 | + | /** One pax record, `<length> <key>=<value>\n`, its length counting itself. */ | |
| 86 | + | export function paxRecord(key: string, value: Uint8Array): Uint8Array { | |
| 87 | + | const rest = encoder.encode(` ${key}=`).length + value.length + 1; | |
| 88 | + | let length = rest + String(rest).length; | |
| 89 | + | if (String(length).length !== String(rest).length) length = rest + String(length).length; | |
| 90 | + | const out = new Uint8Array(length); | |
| 91 | + | const head = encoder.encode(`${length} ${key}=`); | |
| 92 | + | out.set(head); | |
| 93 | + | out.set(value, head.length); | |
| 94 | + | out[length - 1] = 0x0a; | |
| 95 | + | return out; | |
| 96 | + | } | |
| 97 | + | ||
| 98 | + | function padded(data: Uint8Array): Uint8Array[] { | |
| 99 | + | const rest = data.length % BLOCK; | |
| 100 | + | return rest === 0 ? [data] : [data, new Uint8Array(BLOCK - rest)]; | |
| 101 | + | } | |
| 102 | + | ||
| 103 | + | function concat(parts: Uint8Array[]): Uint8Array<ArrayBuffer> { | |
| 104 | + | const out = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0)); | |
| 105 | + | let at = 0; | |
| 106 | + | for (const part of parts) { | |
| 107 | + | out.set(part, at); | |
| 108 | + | at += part.length; | |
| 109 | + | } | |
| 110 | + | return out; | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | /** The tar archive of `entries` (uncompressed), each folder's entry before its files. */ | |
| 114 | + | export function tar(entries: TarEntry[], options: TarOptions): Uint8Array<ArrayBuffer> { | |
| 115 | + | const parts: Uint8Array[] = []; | |
| 116 | + | const mtime = Math.max(0, Math.floor(options.mtime)); | |
| 117 | + | if (options.comment) { | |
| 118 | + | const record = paxRecord("comment", encoder.encode(options.comment)); | |
| 119 | + | parts.push(header({ name: encoder.encode("pax_global_header"), mode: 0o666, size: record.length, mtime, type: "g" }), ...padded(record)); | |
| 120 | + | } | |
| 121 | + | const add = (path: string, kind: "blob" | "exec" | "symlink" | "dir", data: Uint8Array) => { | |
| 122 | + | const fullPath = encoder.encode(kind === "dir" ? `${path}/` : path); | |
| 123 | + | const link = kind === "symlink" ? data : undefined; | |
| 124 | + | const body = kind === "blob" || kind === "exec" ? data : new Uint8Array(); | |
| 125 | + | const split = splitPath(fullPath); | |
| 126 | + | const records: Uint8Array[] = []; | |
| 127 | + | if (!split) records.push(paxRecord("path", fullPath)); | |
| 128 | + | if (link && link.length > 100) records.push(paxRecord("linkpath", link)); | |
| 129 | + | if (records.length > 0) { | |
| 130 | + | const pax = concat(records); | |
| 131 | + | const name = encoder.encode("././@PaxHeader"); | |
| 132 | + | parts.push(header({ name, mode: 0o666, size: pax.length, mtime, type: "x" }), ...padded(pax)); | |
| 133 | + | } | |
| 134 | + | const type = kind === "dir" ? "5" : kind === "symlink" ? "2" : "0"; | |
| 135 | + | parts.push( | |
| 136 | + | header({ | |
| 137 | + | name: split?.name ?? fullPath.subarray(0, 100), | |
| 138 | + | prefix: split?.prefix, | |
| 139 | + | mode: MODES[kind], | |
| 140 | + | size: body.length, | |
| 141 | + | mtime, | |
| 142 | + | type, | |
| 143 | + | link: link && link.length <= 100 ? link : link?.subarray(0, 100), | |
| 144 | + | }), | |
| 145 | + | ); | |
| 146 | + | if (body.length > 0) parts.push(...padded(body)); | |
| 147 | + | }; | |
| 148 | + | const folders = new Set<string>(); | |
| 149 | + | for (const entry of entries) { | |
| 150 | + | const segments = entry.path.split("/").filter(Boolean); | |
| 151 | + | for (let depth = 1; depth < segments.length; depth++) { | |
| 152 | + | const folder = segments.slice(0, depth).join("/"); | |
| 153 | + | if (folders.has(folder)) continue; | |
| 154 | + | folders.add(folder); | |
| 155 | + | add(folder, "dir", new Uint8Array()); | |
| 156 | + | } | |
| 157 | + | add(segments.join("/"), entry.kind ?? "blob", entry.data); | |
| 158 | + | } | |
| 159 | + | parts.push(new Uint8Array(BLOCK * 2)); | |
| 160 | + | return concat(parts); | |
| 161 | + | } | |
| 162 | + | ||
| 163 | + | /** {@link tar}, gzipped. */ | |
| 164 | + | export async function tarGz(entries: TarEntry[], options: TarOptions): Promise<Uint8Array<ArrayBuffer>> { | |
| 165 | + | const stream = new Blob([tar(entries, options)]).stream().pipeThrough(new CompressionStream("gzip")); | |
| 166 | + | return new Uint8Array(await new Response(stream).arrayBuffer()); | |
| 167 | + | } |
| 2 | 2 | import { test } from "node:test"; | |
| 3 | 3 | import { inflateRawSync } from "node:zlib"; | |
| 4 | 4 | ||
| 5 | − | import { crc32, zip } from "./zip.ts"; | |
| 5 | + | import { ZIP_MODES, crc32, zip } from "./zip.ts"; | |
| 6 | 6 | ||
| 7 | 7 | test("crc32 matches the standard check value", () => { | |
| 8 | 8 | assert.equal(crc32(new TextEncoder().encode("123456789")), 0xcbf43926); | |
| ⋯ | |||
| 38 | 38 | assert.equal(read["repo-main/a.txt"], "x"); | |
| 39 | 39 | assert.equal(read["repo-main/empty"], ""); | |
| 40 | 40 | }); | |
| 41 | + | ||
| 42 | + | test("each file keeps its Unix mode: executables and symlinks", async () => { | |
| 43 | + | const out = await zip([ | |
| 44 | + | { path: "repo/run", data: new TextEncoder().encode("#!/bin/sh\n"), mode: ZIP_MODES.exec }, | |
| 45 | + | { path: "repo/link", data: new TextEncoder().encode("run"), mode: ZIP_MODES.symlink }, | |
| 46 | + | { path: "repo/plain", data: new TextEncoder().encode("x") }, | |
| 47 | + | ]); | |
| 48 | + | const view = new DataView(out.buffer, out.byteOffset, out.byteLength); | |
| 49 | + | const end = out.length - 22; | |
| 50 | + | let at = view.getUint32(end + 16, true); | |
| 51 | + | const modes: Record<string, number> = {}; | |
| 52 | + | for (let i = 0; i < 3; i++) { | |
| 53 | + | assert.equal(view.getUint32(at, true), 0x02014b50); | |
| 54 | + | const nameLength = view.getUint16(at + 28, true); | |
| 55 | + | const name = new TextDecoder().decode(out.subarray(at + 46, at + 46 + nameLength)); | |
| 56 | + | modes[name] = view.getUint32(at + 38, true) >>> 16; | |
| 57 | + | at += 46 + nameLength; | |
| 58 | + | } | |
| 59 | + | assert.deepEqual(modes, { "repo/run": 0o100755, "repo/link": 0o120777, "repo/plain": 0o100644 }); | |
| 60 | + | }); | |
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…
Loading the diff…