| 1 | //! The Composer registry: `g1t.sh/-/composer/<workspace>/`, one per |
| 2 | //! workspace, built from its repositories (composer.rs says how). |
| 3 | //! |
| 4 | //! ```sh |
| 5 | //! composer config repositories.acme composer https://g1t.sh/-/composer/acme/ |
| 6 | //! composer config --global --auth http-basic.g1t.sh <you> <g1t token> |
| 7 | //! composer require acme/lib |
| 8 | //! ``` |
| 9 | //! |
| 10 | //! Nothing is uploaded. A repository's versions are read again when it is |
| 11 | //! pushed to (`git.push`), restored, renamed or moved, and once for every |
| 12 | //! repository by the backfill; a request only reads what that left. A zip |
| 13 | //! of a commit is made the first time it is asked for, and kept by its |
| 14 | //! digest like every file here. |
| 15 | |
| 16 | use std::collections::{HashMap, HashSet}; |
| 17 | |
| 18 | use base64::Engine; |
| 19 | use base64::engine::general_purpose::STANDARD; |
| 20 | use g1t_contracts::User; |
| 21 | use g1t_contracts::audit::AuditActor; |
| 22 | use g1t_contracts::events::PackageEvent; |
| 23 | use g1t_contracts::new_id; |
| 24 | use g1t_contracts::repos::{ |
| 25 | AllIdsArgs, FileList, IdPage, ListFilesArgs, MAX_LISTED_FILES, MAX_READ_BLOBS, RawBlob, RawBlobsArgs, RawFile, RawFileArgs, RefsArgs, |
| 26 | RepoRefs, |
| 27 | }; |
| 28 | use g1t_kit::now_ms; |
| 29 | use serde_json::{Value, json}; |
| 30 | use worker::{Context, Headers, Method, Request, Response, Result, Url}; |
| 31 | |
| 32 | use crate::access::{self, Action}; |
| 33 | use crate::composer::{self, Origin, Version}; |
| 34 | use crate::db::{NewVersion, PackageRow}; |
| 35 | use crate::digest::Digest; |
| 36 | use crate::oci::{Credentials, origin}; |
| 37 | use crate::store::BlobStore; |
| 38 | use crate::{Caller, Packages, TargetOf}; |
| 39 | |
| 40 | const COMPOSER: &str = "composer"; |
| 41 | /// The biggest `composer.json` or README read. |
| 42 | const MAX_FILE_BYTES: u32 = 1024 * 1024; |
| 43 | /// The most branches and tags a repository's package lists. |
| 44 | const MAX_BRANCHES: usize = 50; |
| 45 | const MAX_TAGS: usize = 300; |
| 46 | /// The most a zip may hold before it is made, in all and per file. |
| 47 | const MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024; |
| 48 | const MAX_ARCHIVED_FILE: u32 = 32 * 1024 * 1024; |
| 49 | /// Repositories the backfill reads each hour. |
| 50 | const BACKFILL_PAGE: u32 = 25; |
| 51 | const README_NAMES: [&str; 4] = ["README.md", "readme.md", "README.markdown", "README"]; |
| 52 | |
| 53 | fn error(status: u16, message: impl Into<String>) -> Result<Response> { |
| 54 | let mut response = Response::from_json(&json!({ "status": "error", "message": message.into() }))?.with_status(status); |
| 55 | if status == 401 { |
| 56 | response.headers_mut().set("www-authenticate", "Basic realm=\"g1t\"")?; |
| 57 | } |
| 58 | Ok(response) |
| 59 | } |
| 60 | |
| 61 | /// One of the registry's endpoints, under `/-/composer/<workspace>/`. |
| 62 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 63 | pub enum ComposerRoute { |
| 64 | Root { workspace: String }, |
| 65 | /// `p2/<vendor>/<name>.json`, or `~dev.json` for the branches. |
| 66 | Metadata { workspace: String, name: String, dev: bool }, |
| 67 | Dist { workspace: String, name: String, commit: String }, |
| 68 | Downloads { workspace: String }, |
| 69 | } |
| 70 | |
| 71 | pub fn route(path: &str) -> Option<ComposerRoute> { |
| 72 | let rest = path.strip_prefix("/-/composer/")?; |
| 73 | let (workspace, rest) = rest.split_once('/')?; |
| 74 | let workspace = workspace.to_ascii_lowercase(); |
| 75 | if rest == "packages.json" || rest.is_empty() { |
| 76 | return Some(ComposerRoute::Root { workspace }); |
| 77 | } |
| 78 | if rest == "downloads" { |
| 79 | return Some(ComposerRoute::Downloads { workspace }); |
| 80 | } |
| 81 | if let Some(file) = rest.strip_prefix("p2/") { |
| 82 | let (name, dev) = match file.strip_suffix("~dev.json") { |
| 83 | Some(name) => (name, true), |
| 84 | None => (file.strip_suffix(".json")?, false), |
| 85 | }; |
| 86 | return composer::valid_name(name).then(|| ComposerRoute::Metadata { workspace, name: name.to_owned(), dev }); |
| 87 | } |
| 88 | let file = rest.strip_prefix("dist/")?; |
| 89 | let (name, zip) = file.rsplit_once('/')?; |
| 90 | let commit = zip.strip_suffix(".zip")?; |
| 91 | (composer::valid_name(name) && commit.len() == 40 && commit.bytes().all(|b| b.is_ascii_hexdigit())).then(|| ComposerRoute::Dist { |
| 92 | workspace, |
| 93 | name: name.to_owned(), |
| 94 | commit: commit.to_ascii_lowercase(), |
| 95 | }) |
| 96 | } |
| 97 | |
| 98 | /// What a version keeps of its ref, to make its entry from on each read. |
| 99 | fn stored_metadata(composer_json: &Value, version: &Version, git_ref: &str, default_branch: bool) -> Value { |
| 100 | json!({ |
| 101 | "composer": composer_json, |
| 102 | "version_normalized": version.normalized, |
| 103 | "ref": git_ref, |
| 104 | "default_branch": default_branch, |
| 105 | }) |
| 106 | } |
| 107 | |
| 108 | /// The versions a repository's refs make: `(version, commit, ref, default)`. |
| 109 | fn wanted_versions(refs: &[g1t_contracts::repos::GitRefEntry], default_branch: &str) -> Vec<(Version, String, String, bool)> { |
| 110 | let mut branches = Vec::new(); |
| 111 | let mut tags = Vec::new(); |
| 112 | for entry in refs { |
| 113 | if let Some(branch) = entry.name.strip_prefix("refs/heads/") { |
| 114 | branches.push((composer::branch_version(branch), entry.commit.clone(), entry.name.clone(), branch == default_branch)); |
| 115 | } else if let Some(tag) = entry.name.strip_prefix("refs/tags/") |
| 116 | && let Some(version) = composer::tag_version(tag) |
| 117 | { |
| 118 | tags.push((version, entry.commit.clone(), entry.name.clone(), false)); |
| 119 | } |
| 120 | } |
| 121 | // The default branch first, then the newest tags. |
| 122 | branches.sort_by_key(|(_, _, _, default)| !*default); |
| 123 | branches.truncate(MAX_BRANCHES); |
| 124 | tags.sort_by(|a, b| b.0.normalized.cmp(&a.0.normalized)); |
| 125 | tags.truncate(MAX_TAGS); |
| 126 | let mut seen = HashSet::new(); |
| 127 | branches.into_iter().chain(tags).filter(|(v, ..)| seen.insert(v.version.clone())).collect() |
| 128 | } |
| 129 | |
| 130 | impl Packages { |
| 131 | pub async fn composer(&self, request: Request, ctx: &Context) -> Result<Response> { |
| 132 | let url = request.url()?; |
| 133 | let Some(route) = route(url.path()) else { |
| 134 | return error(404, "There is nothing at this address."); |
| 135 | }; |
| 136 | match self.composer_route(request, &url, route, ctx).await { |
| 137 | Ok(response) => Ok(response), |
| 138 | Err(problem) => { |
| 139 | worker::console_error!("packages: composer {}: {problem}", url.path()); |
| 140 | error(500, "Something went wrong on our side. Try again in a moment.") |
| 141 | } |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | async fn composer_route(&self, mut request: Request, url: &Url, route: ComposerRoute, ctx: &Context) -> Result<Response> { |
| 146 | let credentials = self.credentials(&request).await?; |
| 147 | if matches!(request.method(), Method::Get | Method::Head) |
| 148 | && let Some(refused) = self.limited(&request, &credentials, "http-basic credentials").await? |
| 149 | { |
| 150 | return Ok(refused); |
| 151 | } |
| 152 | let viewer = match credentials { |
| 153 | Credentials::Viewer(viewer) => viewer, |
| 154 | Credentials::None => None, |
| 155 | Credentials::Token(_) | Credentials::Bad => { |
| 156 | return error(401, "The username or token is not right. Use a g1t access token: composer config --auth http-basic.g1t.sh <you> <token>"); |
| 157 | } |
| 158 | }; |
| 159 | match route { |
| 160 | ComposerRoute::Root { workspace } => self.composer_root(&workspace, viewer.as_ref()).await, |
| 161 | ComposerRoute::Metadata { workspace, name, dev } => self.composer_metadata(url, &workspace, &name, dev, viewer.as_ref()).await, |
| 162 | ComposerRoute::Dist { workspace, name, commit } => self.composer_dist(&workspace, &name, &commit, viewer.as_ref(), ctx).await, |
| 163 | ComposerRoute::Downloads { workspace } => { |
| 164 | let body: Value = request.json().await.unwrap_or_default(); |
| 165 | let names: HashSet<&str> = body["downloads"] |
| 166 | .as_array() |
| 167 | .map(|list| list.iter().filter_map(|d| d["name"].as_str()).collect()) |
| 168 | .unwrap_or_default(); |
| 169 | for name in names.into_iter().take(50) { |
| 170 | if let Some(package) = self.composer_package(&workspace, name).await? { |
| 171 | self.count_download(&package.id, ctx); |
| 172 | } |
| 173 | } |
| 174 | Ok(Response::empty()?.with_status(204)) |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | async fn composer_package(&self, workspace: &str, name: &str) -> Result<Option<PackageRow>> { |
| 180 | Ok(self.db.package(workspace, COMPOSER, name).await?.filter(|p| !p.hidden())) |
| 181 | } |
| 182 | |
| 183 | /// The answer when `viewer` may not pull `package`, if they may not. |
| 184 | fn composer_check(&self, viewer: Option<&User>, package: &PackageRow) -> Option<Result<Response>> { |
| 185 | let target = TargetOf::package(package); |
| 186 | if access::decide(viewer, &target.view(), Action::Pull).allowed { |
| 187 | return None; |
| 188 | } |
| 189 | Some(if viewer.is_none() { |
| 190 | error(401, "Sign in to install this package: composer config --auth http-basic.g1t.sh <you> <g1t token>") |
| 191 | } else { |
| 192 | error(404, "Not found: no such package, or you cannot see it.") |
| 193 | }) |
| 194 | } |
| 195 | |
| 196 | async fn composer_root(&self, workspace: &str, viewer: Option<&User>) -> Result<Response> { |
| 197 | let rows = self.db.list(workspace, Some(COMPOSER), None, None, 1000).await?; |
| 198 | let available: Vec<String> = rows |
| 199 | .iter() |
| 200 | .filter(|row| access::decide(viewer, &TargetOf::package(&row.package).view(), Action::Pull).allowed) |
| 201 | .map(|row| row.package.name.clone()) |
| 202 | .collect(); |
| 203 | Response::from_json(&composer::root(workspace, &available)) |
| 204 | } |
| 205 | |
| 206 | async fn composer_metadata(&self, url: &Url, workspace: &str, name: &str, dev: bool, viewer: Option<&User>) -> Result<Response> { |
| 207 | let Some(package) = self.composer_package(workspace, name).await? else { |
| 208 | return error(404, format!("There is no package {name} in {workspace}.")); |
| 209 | }; |
| 210 | if let Some(refusal) = self.composer_check(viewer, &package) { |
| 211 | return refusal; |
| 212 | } |
| 213 | let base = origin(url); |
| 214 | let repo = package.repo_name.clone().unwrap_or_default(); |
| 215 | let git_url = format!("{base}/{}/{repo}.git", package.workspace); |
| 216 | let mut entries = Vec::new(); |
| 217 | for row in self.db.versions(&package.id, 1000).await? { |
| 218 | let meta = row.meta(); |
| 219 | let version = Version { |
| 220 | version: row.version.clone(), |
| 221 | normalized: meta["version_normalized"].as_str().unwrap_or(&row.version).to_owned(), |
| 222 | }; |
| 223 | if version.is_dev() != dev { |
| 224 | continue; |
| 225 | } |
| 226 | let dist_url = format!("{base}/-/composer/{}/dist/{name}/{}.zip", package.workspace, row.digest); |
| 227 | let origin = Origin { |
| 228 | git_url: &git_url, |
| 229 | dist_url: &dist_url, |
| 230 | commit: &row.digest, |
| 231 | default_branch: meta["default_branch"].as_bool().unwrap_or(false), |
| 232 | }; |
| 233 | entries.push(composer::version_entry(&meta["composer"], name, &version, &origin)); |
| 234 | } |
| 235 | let mut response = Response::from_json(&composer::p2(name, &entries))?; |
| 236 | response.headers_mut().set("last-modified", &package.updated_at)?; |
| 237 | Ok(response) |
| 238 | } |
| 239 | |
| 240 | async fn composer_dist(&self, workspace: &str, name: &str, commit: &str, viewer: Option<&User>, ctx: &Context) -> Result<Response> { |
| 241 | let Some(package) = self.composer_package(workspace, name).await? else { |
| 242 | return error(404, format!("There is no package {name} in {workspace}.")); |
| 243 | }; |
| 244 | if let Some(refusal) = self.composer_check(viewer, &package) { |
| 245 | return refusal; |
| 246 | } |
| 247 | // Only the commits of its versions: a zip is never made of any |
| 248 | // other commit of the repository. |
| 249 | if self.db.version_by_digest(&package.id, commit).await?.is_none() { |
| 250 | return error(404, format!("{commit} is not a version of {name}.")); |
| 251 | } |
| 252 | let blob = match self.db.dist_for_commit(&package.id, commit).await? { |
| 253 | Some(blob) => blob, |
| 254 | None => match self.build_dist(&package, commit).await? { |
| 255 | Ok(blob) => blob, |
| 256 | Err(refusal) => return refusal, |
| 257 | }, |
| 258 | }; |
| 259 | let Some(got) = self.store.get(&blob.object_key, None).await? else { |
| 260 | return error(404, "The archive is missing. Try again."); |
| 261 | }; |
| 262 | self.count_download(&package.id, ctx); |
| 263 | let headers = Headers::new(); |
| 264 | headers.set("content-type", "application/zip")?; |
| 265 | headers.set("content-length", &blob.size.to_string())?; |
| 266 | headers.set("cache-control", "max-age=31536000")?; |
| 267 | Ok(Response::from_body(got.body)?.with_headers(headers)) |
| 268 | } |
| 269 | |
| 270 | /// Makes the zip of a commit: its files but those `.gitattributes` |
| 271 | /// marks `export-ignore`, as `git archive` would leave them out. |
| 272 | async fn build_dist(&self, package: &PackageRow, commit: &str) -> Result<std::result::Result<crate::db::BlobRow, Result<Response>>> { |
| 273 | let Some(repo_id) = package.repo_id.clone() else { |
| 274 | return Ok(Err(error(404, "This package has no repository."))); |
| 275 | }; |
| 276 | let listed: FileList = g1t_kit::call( |
| 277 | &self.repos, |
| 278 | "list_files", |
| 279 | &ListFilesArgs { repo_id: repo_id.clone(), git_ref: Some(commit.to_owned()), skip_dirs: Vec::new(), limit: MAX_LISTED_FILES }, |
| 280 | ) |
| 281 | .await?; |
| 282 | if listed.truncated { |
| 283 | return Ok(Err(error(507, format!("The commit has more than {MAX_LISTED_FILES} files, too many for an archive. Install from source: composer install --prefer-source")))); |
| 284 | } |
| 285 | let attributes = self.repo_file(&repo_id, commit, ".gitattributes").await?; |
| 286 | let ignores = attributes.map(|text| composer::export_ignores(&String::from_utf8_lossy(&text))).unwrap_or_default(); |
| 287 | let files: Vec<(String, String)> = listed |
| 288 | .files |
| 289 | .into_iter() |
| 290 | .filter_map(|file| Some((file.path, file.hash?))) |
| 291 | .filter(|(path, _)| !composer::ignored(&ignores, path)) |
| 292 | .collect(); |
| 293 | let mut bytes_of: HashMap<String, Vec<u8>> = HashMap::new(); |
| 294 | let unique: Vec<String> = files.iter().map(|(_, hash)| hash.clone()).collect::<HashSet<_>>().into_iter().collect(); |
| 295 | let mut total = 0u64; |
| 296 | for chunk in unique.chunks(MAX_READ_BLOBS) { |
| 297 | let read: Vec<RawBlob> = g1t_kit::call( |
| 298 | &self.repos, |
| 299 | "raw_blobs", |
| 300 | &RawBlobsArgs { repo_id: repo_id.clone(), hashes: chunk.to_vec(), max_bytes: MAX_ARCHIVED_FILE }, |
| 301 | ) |
| 302 | .await?; |
| 303 | for blob in read { |
| 304 | total += blob.size; |
| 305 | if total > MAX_ARCHIVE_BYTES || (blob.data.is_none() && blob.size > 0) { |
| 306 | return Ok(Err(error( |
| 307 | 507, |
| 308 | format!("The commit is too large for an archive (over {} MB). Install from source: composer install --prefer-source", MAX_ARCHIVE_BYTES / 1_048_576), |
| 309 | ))); |
| 310 | } |
| 311 | let data = blob.data.as_deref().map(|d| STANDARD.decode(d).unwrap_or_default()).unwrap_or_default(); |
| 312 | bytes_of.insert(blob.hash, data); |
| 313 | } |
| 314 | } |
| 315 | let entries: Vec<(String, Vec<u8>)> = files |
| 316 | .into_iter() |
| 317 | .map(|(path, hash)| { |
| 318 | let data = bytes_of.get(&hash).cloned().unwrap_or_default(); |
| 319 | (path, data) |
| 320 | }) |
| 321 | .collect(); |
| 322 | let zip = composer::zip(&entries); |
| 323 | let digest = Digest::of(&zip); |
| 324 | let size = zip.len() as u64; |
| 325 | let now = now_ms(); |
| 326 | if self.db.blob(&digest).await?.is_none() { |
| 327 | self.store.put(&digest.object_key(), zip).await?; |
| 328 | } |
| 329 | self.db.keep_blob(&package.id, &digest, size, Some("application/zip"), &digest.object_key(), now).await?; |
| 330 | self.db.add_dist(&package.id, commit, &digest, size).await?; |
| 331 | self.db.measure(&package.workspace).await?; |
| 332 | Ok(Ok(crate::db::BlobRow { digest: digest.to_string(), size, media_type: Some("application/zip".into()), object_key: digest.object_key() })) |
| 333 | } |
| 334 | |
| 335 | /// A file of a repository at a commit, if it is there and not large. |
| 336 | async fn repo_file(&self, repo_id: &str, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>> { |
| 337 | let file: Option<RawFile> = g1t_kit::call( |
| 338 | &self.repos, |
| 339 | "raw_file", |
| 340 | &RawFileArgs { repo_id: repo_id.to_owned(), git_ref: git_ref.to_owned(), path: path.to_owned(), max_bytes: MAX_FILE_BYTES }, |
| 341 | ) |
| 342 | .await?; |
| 343 | Ok(file.and_then(|file| STANDARD.decode(file.data).ok())) |
| 344 | } |
| 345 | |
| 346 | /// Reads a repository's Composer package again from its refs: makes it |
| 347 | /// when its default branch gained a `composer.json`, records new and |
| 348 | /// moved versions, lets go of deleted ones, and deletes the package |
| 349 | /// when the repository stopped being one. Says whether it is one. |
| 350 | pub(crate) async fn sync_composer(&self, repo_id: &str) -> Result<bool> { |
| 351 | let found: Option<RepoRefs> = g1t_kit::call(&self.repos, "refs", &RefsArgs { repo_id: repo_id.to_owned() }).await?; |
| 352 | let existing = self.db.package_for_repo(repo_id, COMPOSER).await?; |
| 353 | let Some(RepoRefs { repo, refs }) = found else { |
| 354 | if let Some(package) = existing { |
| 355 | self.drop_composer(&package).await?; |
| 356 | } |
| 357 | return Ok(false); |
| 358 | }; |
| 359 | let workspace = repo.namespace.to_lowercase(); |
| 360 | if self.db.workspace_hidden(&workspace).await? { |
| 361 | return Ok(false); |
| 362 | } |
| 363 | let default = refs.iter().find(|r| r.name == format!("refs/heads/{}", repo.default_branch)).map(|r| r.commit.clone()); |
| 364 | let manifest = match &default { |
| 365 | Some(commit) => self.composer_json(repo_id, commit).await?, |
| 366 | None => None, |
| 367 | }; |
| 368 | let Some((name, root_manifest)) = manifest else { |
| 369 | if let Some(package) = existing { |
| 370 | self.drop_composer(&package).await?; |
| 371 | } |
| 372 | return Ok(false); |
| 373 | }; |
| 374 | // A repository moved to another workspace takes its package along. |
| 375 | let existing = match existing { |
| 376 | Some(package) if package.workspace != workspace => { |
| 377 | self.drop_composer(&package).await?; |
| 378 | None |
| 379 | } |
| 380 | other => other, |
| 381 | }; |
| 382 | let now = now_ms(); |
| 383 | let package = match existing { |
| 384 | Some(package) if package.name == name => package, |
| 385 | Some(package) => { |
| 386 | if self.db.package(&workspace, COMPOSER, &name).await?.is_some() { |
| 387 | worker::console_error!("packages: {workspace}/{} names {name}, which another repository has", repo.name); |
| 388 | package |
| 389 | } else { |
| 390 | self.db.rename_package(&package.id, &name, now).await?; |
| 391 | PackageRow { name: name.clone(), ..package } |
| 392 | } |
| 393 | } |
| 394 | None => { |
| 395 | if let Some(other) = self.db.package(&workspace, COMPOSER, &name).await? |
| 396 | && other.repo_id.as_deref() != Some(repo_id) |
| 397 | { |
| 398 | worker::console_error!("packages: {workspace}/{} names {name}, which another repository has", repo.name); |
| 399 | return Ok(false); |
| 400 | } |
| 401 | self.db |
| 402 | .create_package(&new_id("pkg", now), &workspace, COMPOSER, &name, Some((&repo.id, &repo.name, repo.is_private)), "g1t", now) |
| 403 | .await? |
| 404 | } |
| 405 | }; |
| 406 | |
| 407 | // Versions follow git, so g1t records them: what made them is the |
| 408 | // push, already in the log as `git.push`. |
| 409 | let caller = Caller { actor: Some(AuditActor::system()) }; |
| 410 | let wanted = wanted_versions(&refs, &repo.default_branch); |
| 411 | let stored = self.db.versions(&package.id, 1000).await?; |
| 412 | let mut manifests: HashMap<String, Option<Value>> = HashMap::new(); |
| 413 | if let Some(commit) = &default { |
| 414 | manifests.insert(commit.clone(), Some(root_manifest.clone())); |
| 415 | } |
| 416 | let mut changed = false; |
| 417 | for (version, commit, git_ref, is_default) in &wanted { |
| 418 | let current = stored.iter().find(|row| row.version == version.version); |
| 419 | if let Some(row) = current |
| 420 | && row.digest == *commit |
| 421 | && row.meta()["default_branch"].as_bool().unwrap_or(false) == *is_default |
| 422 | { |
| 423 | continue; |
| 424 | } |
| 425 | if !manifests.contains_key(commit) { |
| 426 | let read = self.composer_json(repo_id, commit).await?.map(|(_, json)| json); |
| 427 | manifests.insert(commit.clone(), read); |
| 428 | } |
| 429 | // A ref without a composer.json of its own is not a version. |
| 430 | let Some(Some(json)) = manifests.get(commit) else { continue }; |
| 431 | self.db |
| 432 | .replace_version( |
| 433 | &NewVersion { |
| 434 | id: new_id("ver", now), |
| 435 | package_id: package.id.clone(), |
| 436 | version: version.version.clone(), |
| 437 | digest: commit.clone(), |
| 438 | size: 0, |
| 439 | metadata: stored_metadata(json, version, git_ref, *is_default).to_string(), |
| 440 | subject: None, |
| 441 | published_by: None, |
| 442 | files: Vec::new(), |
| 443 | }, |
| 444 | now, |
| 445 | ) |
| 446 | .await?; |
| 447 | changed = true; |
| 448 | if current.is_none() { |
| 449 | let event = PackageEvent { version: Some(version.version.clone()), digest: Some(commit.clone()), ..self.event_of(&package) }; |
| 450 | self.announce("package.published", &package, event, &caller).await; |
| 451 | self.audit(&caller, "package.publish", &package, Some(&format!("{name}@{}", version.version)), None).await; |
| 452 | } |
| 453 | } |
| 454 | let kept: HashSet<&str> = wanted.iter().map(|(v, ..)| v.version.as_str()).collect(); |
| 455 | for row in stored.iter().filter(|row| !kept.contains(row.version.as_str())) { |
| 456 | self.db.delete_version(&row.id).await?; |
| 457 | let event = PackageEvent { version: Some(row.version.clone()), digest: Some(row.digest.clone()), ..self.event_of(&package) }; |
| 458 | self.announce("package.version_deleted", &package, event, &caller).await; |
| 459 | self.audit(&caller, "package.delete_version", &package, Some(&format!("{}@{}", package.name, row.version)), None).await; |
| 460 | changed = true; |
| 461 | } |
| 462 | if let Some(commit) = &default { |
| 463 | self.composer_readme(&package, repo_id, commit, &root_manifest, now).await?; |
| 464 | } |
| 465 | if changed { |
| 466 | self.db.measure(&workspace).await?; |
| 467 | } |
| 468 | Ok(true) |
| 469 | } |
| 470 | |
| 471 | /// The package's README and description, from the default branch. |
| 472 | async fn composer_readme(&self, package: &PackageRow, repo_id: &str, commit: &str, manifest: &Value, now: u64) -> Result<()> { |
| 473 | let mut readme = None; |
| 474 | for name in README_NAMES { |
| 475 | if let Some(bytes) = self.repo_file(repo_id, commit, name).await? { |
| 476 | readme = Some(bytes); |
| 477 | break; |
| 478 | } |
| 479 | } |
| 480 | let digest = match readme.filter(|b| !b.is_empty()) { |
| 481 | Some(bytes) => { |
| 482 | let digest = Digest::of(&bytes); |
| 483 | if self.db.blob(&digest).await?.is_none() { |
| 484 | self.store.put(&digest.object_key(), bytes.clone()).await?; |
| 485 | } |
| 486 | self.db.keep_blob(&package.id, &digest, bytes.len() as u64, Some("text/markdown"), &digest.object_key(), now).await?; |
| 487 | Some(digest.to_string()) |
| 488 | } |
| 489 | None => None, |
| 490 | }; |
| 491 | self.db.set_readme(&package.id, digest.as_deref(), manifest["description"].as_str(), now).await |
| 492 | } |
| 493 | |
| 494 | /// A commit's `composer.json`, when it has one naming a valid package. |
| 495 | async fn composer_json(&self, repo_id: &str, commit: &str) -> Result<Option<(String, Value)>> { |
| 496 | let Some(bytes) = self.repo_file(repo_id, commit, "composer.json").await? else { |
| 497 | return Ok(None); |
| 498 | }; |
| 499 | let Ok(json) = serde_json::from_slice::<Value>(&bytes) else { |
| 500 | return Ok(None); |
| 501 | }; |
| 502 | let Some(name) = json["name"].as_str().map(str::to_lowercase).filter(|n| composer::valid_name(n)) else { |
| 503 | return Ok(None); |
| 504 | }; |
| 505 | Ok(Some((name, json))) |
| 506 | } |
| 507 | |
| 508 | async fn drop_composer(&self, package: &PackageRow) -> Result<()> { |
| 509 | self.db.delete_package(&package.id).await?; |
| 510 | self.db.measure(&package.workspace).await?; |
| 511 | let caller = Caller { actor: Some(AuditActor::system()) }; |
| 512 | self.announce("package.deleted", package, self.event_of(package), &caller).await; |
| 513 | self.audit(&caller, "package.delete", package, Some(&package.name), None).await; |
| 514 | Ok(()) |
| 515 | } |
| 516 | |
| 517 | /// Deletes the Composer package built from a deleted repository. |
| 518 | pub(crate) async fn composer_repo_gone(&self, repo_id: &str) -> Result<()> { |
| 519 | if let Some(package) = self.db.package_for_repo(repo_id, COMPOSER).await? { |
| 520 | self.drop_composer(&package).await?; |
| 521 | } |
| 522 | Ok(()) |
| 523 | } |
| 524 | |
| 525 | /// Reads a page of repositories the backfill has not yet, until it has |
| 526 | /// read them all once. Says how many were packages. |
| 527 | pub(crate) async fn composer_backfill(&self) -> Result<u32> { |
| 528 | let (after, finished) = self.db.backfill().await?; |
| 529 | if finished { |
| 530 | return Ok(0); |
| 531 | } |
| 532 | let page: IdPage = g1t_kit::call(&self.repos, "all_ids", &AllIdsArgs { after: after.clone(), limit: BACKFILL_PAGE }).await?; |
| 533 | let mut found = 0; |
| 534 | for id in &page.ids { |
| 535 | match self.sync_composer(id).await { |
| 536 | Ok(true) => found += 1, |
| 537 | Ok(false) => {} |
| 538 | Err(error) => worker::console_error!("packages: composer backfill of {id}: {error}"), |
| 539 | } |
| 540 | } |
| 541 | let last = page.ids.last().cloned().or(after); |
| 542 | self.db.set_backfill(last.as_deref(), page.next.is_none(), now_ms()).await?; |
| 543 | Ok(found) |
| 544 | } |
| 545 | } |
| 546 | |
| 547 | #[cfg(test)] |
| 548 | mod tests { |
| 549 | use super::*; |
| 550 | use g1t_contracts::repos::GitRefEntry; |
| 551 | |
| 552 | #[test] |
| 553 | fn every_endpoint_is_routed() { |
| 554 | let commit = "a".repeat(40); |
| 555 | assert_eq!(route("/-/composer/acme/packages.json"), Some(ComposerRoute::Root { workspace: "acme".into() })); |
| 556 | assert_eq!(route("/-/composer/acme/"), Some(ComposerRoute::Root { workspace: "acme".into() })); |
| 557 | assert_eq!( |
| 558 | route("/-/composer/acme/p2/acme/lib.json"), |
| 559 | Some(ComposerRoute::Metadata { workspace: "acme".into(), name: "acme/lib".into(), dev: false }) |
| 560 | ); |
| 561 | assert_eq!( |
| 562 | route("/-/composer/acme/p2/acme/lib~dev.json"), |
| 563 | Some(ComposerRoute::Metadata { workspace: "acme".into(), name: "acme/lib".into(), dev: true }) |
| 564 | ); |
| 565 | assert_eq!( |
| 566 | route(&format!("/-/composer/acme/dist/acme/lib/{commit}.zip")), |
| 567 | Some(ComposerRoute::Dist { workspace: "acme".into(), name: "acme/lib".into(), commit: commit.clone() }) |
| 568 | ); |
| 569 | assert_eq!(route("/-/composer/acme/downloads"), Some(ComposerRoute::Downloads { workspace: "acme".into() })); |
| 570 | assert_eq!(route("/-/composer/acme/p2/Acme/lib.json"), None); |
| 571 | assert_eq!(route("/-/composer/acme/dist/acme/lib/short.zip"), None); |
| 572 | assert_eq!(route("/-/composer/acme"), None); |
| 573 | } |
| 574 | |
| 575 | #[test] |
| 576 | fn a_repositorys_refs_make_its_versions() { |
| 577 | let entry = |name: &str, commit: &str| GitRefEntry { name: name.into(), commit: commit.into() }; |
| 578 | let refs = [ |
| 579 | entry("refs/heads/feature", "f"), |
| 580 | entry("refs/heads/main", "m"), |
| 581 | entry("refs/tags/v1.0.0", "a"), |
| 582 | entry("refs/tags/v1.1.0", "b"), |
| 583 | entry("refs/tags/nightly", "n"), |
| 584 | ]; |
| 585 | let wanted = wanted_versions(&refs, "main"); |
| 586 | let names: Vec<(&str, &str, bool)> = wanted.iter().map(|(v, c, _, d)| (v.version.as_str(), c.as_str(), *d)).collect(); |
| 587 | assert_eq!( |
| 588 | names, |
| 589 | [("dev-main", "m", true), ("dev-feature", "f", false), ("v1.1.0", "b", false), ("v1.0.0", "a", false)], |
| 590 | "the default branch first, newest tags next, tags that are not versions left out" |
| 591 | ); |
| 592 | } |
| 593 | } |