Sign in

flagon-io/g1t

public

g1t/services/packages/src/composer_http.rs

593 lines28,009 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Composer from the workspace's own repositories, and go get from g1t.sh1//! The Composer registry: `g1t.sh/-/composer/<workspace>/`, one per
2//! workspace, built from its repositories (composer.rs says how).
3//!
4//! ```sh
5//! composer config repositories.acme composer https://g1t.sh/-/composer/acme/
6//! composer config --global --auth http-basic.g1t.sh <you> <g1t token>
7//! composer require acme/lib
8//! ```
9//!
10//! Nothing is uploaded. A repository's versions are read again when it is
11//! pushed to (`git.push`), restored, renamed or moved, and once for every
12//! repository by the backfill; a request only reads what that left. A zip
13//! of a commit is made the first time it is asked for, and kept by its
14//! digest like every file here.
15
16use std::collections::{HashMap, HashSet};
17
18use base64::Engine;
19use base64::engine::general_purpose::STANDARD;
20use g1t_contracts::User;
A public import copies every branch and tag, so an imported library keeps its releases21use g1t_contracts::audit::AuditActor;
Composer from the workspace's own repositories, and go get from g1t.sh22use g1t_contracts::events::PackageEvent;
23use g1t_contracts::new_id;
24use g1t_contracts::repos::{
25 AllIdsArgs, FileList, IdPage, ListFilesArgs, MAX_LISTED_FILES, MAX_READ_BLOBS, RawBlob, RawBlobsArgs, RawFile, RawFileArgs, RefsArgs,
26 RepoRefs,
27};
28use g1t_kit::now_ms;
29use serde_json::{Value, json};
30use worker::{Context, Headers, Method, Request, Response, Result, Url};
31
32use crate::access::{self, Action};
33use crate::composer::{self, Origin, Version};
34use crate::db::{NewVersion, PackageRow};
35use crate::digest::Digest;
36use crate::oci::{Credentials, origin};
37use crate::store::BlobStore;
38use crate::{Caller, Packages, TargetOf};
39
40const COMPOSER: &str = "composer";
41/// The biggest `composer.json` or README read.
42const MAX_FILE_BYTES: u32 = 1024 * 1024;
43/// The most branches and tags a repository's package lists.
44const MAX_BRANCHES: usize = 50;
45const MAX_TAGS: usize = 300;
46/// The most a zip may hold before it is made, in all and per file.
47const MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024;
48const MAX_ARCHIVED_FILE: u32 = 32 * 1024 * 1024;
49/// Repositories the backfill reads each hour.
50const BACKFILL_PAGE: u32 = 25;
51const README_NAMES: [&str; 4] = ["README.md", "readme.md", "README.markdown", "README"];
52
53fn error(status: u16, message: impl Into<String>) -> Result<Response> {
54 let mut response = Response::from_json(&json!({ "status": "error", "message": message.into() }))?.with_status(status);
55 if status == 401 {
56 response.headers_mut().set("www-authenticate", "Basic realm=\"g1t\"")?;
57 }
58 Ok(response)
59}
60
61/// One of the registry's endpoints, under `/-/composer/<workspace>/`.
62#[derive(Clone, Debug, PartialEq, Eq)]
63pub enum ComposerRoute {
64 Root { workspace: String },
65 /// `p2/<vendor>/<name>.json`, or `~dev.json` for the branches.
66 Metadata { workspace: String, name: String, dev: bool },
67 Dist { workspace: String, name: String, commit: String },
68 Downloads { workspace: String },
69}
70
71pub fn route(path: &str) -> Option<ComposerRoute> {
72 let rest = path.strip_prefix("/-/composer/")?;
73 let (workspace, rest) = rest.split_once('/')?;
74 let workspace = workspace.to_ascii_lowercase();
75 if rest == "packages.json" || rest.is_empty() {
76 return Some(ComposerRoute::Root { workspace });
77 }
78 if rest == "downloads" {
79 return Some(ComposerRoute::Downloads { workspace });
80 }
81 if let Some(file) = rest.strip_prefix("p2/") {
82 let (name, dev) = match file.strip_suffix("~dev.json") {
83 Some(name) => (name, true),
84 None => (file.strip_suffix(".json")?, false),
85 };
86 return composer::valid_name(name).then(|| ComposerRoute::Metadata { workspace, name: name.to_owned(), dev });
87 }
88 let file = rest.strip_prefix("dist/")?;
89 let (name, zip) = file.rsplit_once('/')?;
90 let commit = zip.strip_suffix(".zip")?;
91 (composer::valid_name(name) && commit.len() == 40 && commit.bytes().all(|b| b.is_ascii_hexdigit())).then(|| ComposerRoute::Dist {
92 workspace,
93 name: name.to_owned(),
94 commit: commit.to_ascii_lowercase(),
95 })
96}
97
98/// What a version keeps of its ref, to make its entry from on each read.
99fn stored_metadata(composer_json: &Value, version: &Version, git_ref: &str, default_branch: bool) -> Value {
100 json!({
101 "composer": composer_json,
102 "version_normalized": version.normalized,
103 "ref": git_ref,
104 "default_branch": default_branch,
105 })
106}
107
108/// The versions a repository's refs make: `(version, commit, ref, default)`.
109fn wanted_versions(refs: &[g1t_contracts::repos::GitRefEntry], default_branch: &str) -> Vec<(Version, String, String, bool)> {
110 let mut branches = Vec::new();
111 let mut tags = Vec::new();
112 for entry in refs {
113 if let Some(branch) = entry.name.strip_prefix("refs/heads/") {
114 branches.push((composer::branch_version(branch), entry.commit.clone(), entry.name.clone(), branch == default_branch));
115 } else if let Some(tag) = entry.name.strip_prefix("refs/tags/")
116 && let Some(version) = composer::tag_version(tag)
117 {
118 tags.push((version, entry.commit.clone(), entry.name.clone(), false));
119 }
120 }
121 // The default branch first, then the newest tags.
122 branches.sort_by_key(|(_, _, _, default)| !*default);
123 branches.truncate(MAX_BRANCHES);
124 tags.sort_by(|a, b| b.0.normalized.cmp(&a.0.normalized));
125 tags.truncate(MAX_TAGS);
126 let mut seen = HashSet::new();
127 branches.into_iter().chain(tags).filter(|(v, ..)| seen.insert(v.version.clone())).collect()
128}
129
130impl Packages {
131 pub async fn composer(&self, request: Request, ctx: &Context) -> Result<Response> {
132 let url = request.url()?;
133 let Some(route) = route(url.path()) else {
134 return error(404, "There is nothing at this address.");
135 };
136 match self.composer_route(request, &url, route, ctx).await {
137 Ok(response) => Ok(response),
138 Err(problem) => {
139 worker::console_error!("packages: composer {}: {problem}", url.path());
140 error(500, "Something went wrong on our side. Try again in a moment.")
141 }
142 }
143 }
144
145 async fn composer_route(&self, mut request: Request, url: &Url, route: ComposerRoute, ctx: &Context) -> Result<Response> {
146 let credentials = self.credentials(&request).await?;
147 if matches!(request.method(), Method::Get | Method::Head)
148 && let Some(refused) = self.limited(&request, &credentials, "http-basic credentials").await?
149 {
150 return Ok(refused);
151 }
152 let viewer = match credentials {
153 Credentials::Viewer(viewer) => viewer,
154 Credentials::None => None,
155 Credentials::Token(_) | Credentials::Bad => {
156 return error(401, "The username or token is not right. Use a g1t access token: composer config --auth http-basic.g1t.sh <you> <token>");
157 }
158 };
159 match route {
160 ComposerRoute::Root { workspace } => self.composer_root(&workspace, viewer.as_ref()).await,
161 ComposerRoute::Metadata { workspace, name, dev } => self.composer_metadata(url, &workspace, &name, dev, viewer.as_ref()).await,
162 ComposerRoute::Dist { workspace, name, commit } => self.composer_dist(&workspace, &name, &commit, viewer.as_ref(), ctx).await,
163 ComposerRoute::Downloads { workspace } => {
164 let body: Value = request.json().await.unwrap_or_default();
165 let names: HashSet<&str> = body["downloads"]
166 .as_array()
167 .map(|list| list.iter().filter_map(|d| d["name"].as_str()).collect())
168 .unwrap_or_default();
169 for name in names.into_iter().take(50) {
170 if let Some(package) = self.composer_package(&workspace, name).await? {
171 self.count_download(&package.id, ctx);
172 }
173 }
174 Ok(Response::empty()?.with_status(204))
175 }
176 }
177 }
178
179 async fn composer_package(&self, workspace: &str, name: &str) -> Result<Option<PackageRow>> {
180 Ok(self.db.package(workspace, COMPOSER, name).await?.filter(|p| !p.hidden()))
181 }
182
183 /// The answer when `viewer` may not pull `package`, if they may not.
184 fn composer_check(&self, viewer: Option<&User>, package: &PackageRow) -> Option<Result<Response>> {
185 let target = TargetOf::package(package);
186 if access::decide(viewer, &target.view(), Action::Pull).allowed {
187 return None;
188 }
189 Some(if viewer.is_none() {
190 error(401, "Sign in to install this package: composer config --auth http-basic.g1t.sh <you> <g1t token>")
191 } else {
192 error(404, "Not found: no such package, or you cannot see it.")
193 })
194 }
195
196 async fn composer_root(&self, workspace: &str, viewer: Option<&User>) -> Result<Response> {
197 let rows = self.db.list(workspace, Some(COMPOSER), None, None, 1000).await?;
198 let available: Vec<String> = rows
199 .iter()
200 .filter(|row| access::decide(viewer, &TargetOf::package(&row.package).view(), Action::Pull).allowed)
201 .map(|row| row.package.name.clone())
202 .collect();
203 Response::from_json(&composer::root(workspace, &available))
204 }
205
206 async fn composer_metadata(&self, url: &Url, workspace: &str, name: &str, dev: bool, viewer: Option<&User>) -> Result<Response> {
207 let Some(package) = self.composer_package(workspace, name).await? else {
208 return error(404, format!("There is no package {name} in {workspace}."));
209 };
210 if let Some(refusal) = self.composer_check(viewer, &package) {
211 return refusal;
212 }
213 let base = origin(url);
214 let repo = package.repo_name.clone().unwrap_or_default();
215 let git_url = format!("{base}/{}/{repo}.git", package.workspace);
216 let mut entries = Vec::new();
217 for row in self.db.versions(&package.id, 1000).await? {
218 let meta = row.meta();
219 let version = Version {
220 version: row.version.clone(),
221 normalized: meta["version_normalized"].as_str().unwrap_or(&row.version).to_owned(),
222 };
223 if version.is_dev() != dev {
224 continue;
225 }
226 let dist_url = format!("{base}/-/composer/{}/dist/{name}/{}.zip", package.workspace, row.digest);
227 let origin = Origin {
228 git_url: &git_url,
229 dist_url: &dist_url,
230 commit: &row.digest,
231 default_branch: meta["default_branch"].as_bool().unwrap_or(false),
232 };
233 entries.push(composer::version_entry(&meta["composer"], name, &version, &origin));
234 }
235 let mut response = Response::from_json(&composer::p2(name, &entries))?;
236 response.headers_mut().set("last-modified", &package.updated_at)?;
237 Ok(response)
238 }
239
240 async fn composer_dist(&self, workspace: &str, name: &str, commit: &str, viewer: Option<&User>, ctx: &Context) -> Result<Response> {
241 let Some(package) = self.composer_package(workspace, name).await? else {
242 return error(404, format!("There is no package {name} in {workspace}."));
243 };
244 if let Some(refusal) = self.composer_check(viewer, &package) {
245 return refusal;
246 }
247 // Only the commits of its versions: a zip is never made of any
248 // other commit of the repository.
249 if self.db.version_by_digest(&package.id, commit).await?.is_none() {
250 return error(404, format!("{commit} is not a version of {name}."));
251 }
252 let blob = match self.db.dist_for_commit(&package.id, commit).await? {
253 Some(blob) => blob,
254 None => match self.build_dist(&package, commit).await? {
255 Ok(blob) => blob,
256 Err(refusal) => return refusal,
257 },
258 };
259 let Some(got) = self.store.get(&blob.object_key, None).await? else {
260 return error(404, "The archive is missing. Try again.");
261 };
262 self.count_download(&package.id, ctx);
263 let headers = Headers::new();
264 headers.set("content-type", "application/zip")?;
265 headers.set("content-length", &blob.size.to_string())?;
266 headers.set("cache-control", "max-age=31536000")?;
267 Ok(Response::from_body(got.body)?.with_headers(headers))
268 }
269
270 /// Makes the zip of a commit: its files but those `.gitattributes`
271 /// marks `export-ignore`, as `git archive` would leave them out.
272 async fn build_dist(&self, package: &PackageRow, commit: &str) -> Result<std::result::Result<crate::db::BlobRow, Result<Response>>> {
273 let Some(repo_id) = package.repo_id.clone() else {
274 return Ok(Err(error(404, "This package has no repository.")));
275 };
276 let listed: FileList = g1t_kit::call(
277 &self.repos,
278 "list_files",
279 &ListFilesArgs { repo_id: repo_id.clone(), git_ref: Some(commit.to_owned()), skip_dirs: Vec::new(), limit: MAX_LISTED_FILES },
280 )
281 .await?;
282 if listed.truncated {
283 return Ok(Err(error(507, format!("The commit has more than {MAX_LISTED_FILES} files, too many for an archive. Install from source: composer install --prefer-source"))));
284 }
285 let attributes = self.repo_file(&repo_id, commit, ".gitattributes").await?;
286 let ignores = attributes.map(|text| composer::export_ignores(&String::from_utf8_lossy(&text))).unwrap_or_default();
287 let files: Vec<(String, String)> = listed
288 .files
289 .into_iter()
290 .filter_map(|file| Some((file.path, file.hash?)))
291 .filter(|(path, _)| !composer::ignored(&ignores, path))
292 .collect();
293 let mut bytes_of: HashMap<String, Vec<u8>> = HashMap::new();
294 let unique: Vec<String> = files.iter().map(|(_, hash)| hash.clone()).collect::<HashSet<_>>().into_iter().collect();
295 let mut total = 0u64;
296 for chunk in unique.chunks(MAX_READ_BLOBS) {
297 let read: Vec<RawBlob> = g1t_kit::call(
298 &self.repos,
299 "raw_blobs",
300 &RawBlobsArgs { repo_id: repo_id.clone(), hashes: chunk.to_vec(), max_bytes: MAX_ARCHIVED_FILE },
301 )
302 .await?;
303 for blob in read {
304 total += blob.size;
305 if total > MAX_ARCHIVE_BYTES || (blob.data.is_none() && blob.size > 0) {
306 return Ok(Err(error(
307 507,
308 format!("The commit is too large for an archive (over {} MB). Install from source: composer install --prefer-source", MAX_ARCHIVE_BYTES / 1_048_576),
309 )));
310 }
311 let data = blob.data.as_deref().map(|d| STANDARD.decode(d).unwrap_or_default()).unwrap_or_default();
312 bytes_of.insert(blob.hash, data);
313 }
314 }
315 let entries: Vec<(String, Vec<u8>)> = files
316 .into_iter()
317 .map(|(path, hash)| {
318 let data = bytes_of.get(&hash).cloned().unwrap_or_default();
319 (path, data)
320 })
321 .collect();
322 let zip = composer::zip(&entries);
323 let digest = Digest::of(&zip);
324 let size = zip.len() as u64;
325 let now = now_ms();
326 if self.db.blob(&digest).await?.is_none() {
327 self.store.put(&digest.object_key(), zip).await?;
328 }
329 self.db.keep_blob(&package.id, &digest, size, Some("application/zip"), &digest.object_key(), now).await?;
330 self.db.add_dist(&package.id, commit, &digest, size).await?;
331 self.db.measure(&package.workspace).await?;
332 Ok(Ok(crate::db::BlobRow { digest: digest.to_string(), size, media_type: Some("application/zip".into()), object_key: digest.object_key() }))
333 }
334
335 /// A file of a repository at a commit, if it is there and not large.
336 async fn repo_file(&self, repo_id: &str, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>> {
337 let file: Option<RawFile> = g1t_kit::call(
338 &self.repos,
339 "raw_file",
340 &RawFileArgs { repo_id: repo_id.to_owned(), git_ref: git_ref.to_owned(), path: path.to_owned(), max_bytes: MAX_FILE_BYTES },
341 )
342 .await?;
343 Ok(file.and_then(|file| STANDARD.decode(file.data).ok()))
344 }
345
346 /// Reads a repository's Composer package again from its refs: makes it
347 /// when its default branch gained a `composer.json`, records new and
348 /// moved versions, lets go of deleted ones, and deletes the package
349 /// when the repository stopped being one. Says whether it is one.
350 pub(crate) async fn sync_composer(&self, repo_id: &str) -> Result<bool> {
351 let found: Option<RepoRefs> = g1t_kit::call(&self.repos, "refs", &RefsArgs { repo_id: repo_id.to_owned() }).await?;
352 let existing = self.db.package_for_repo(repo_id, COMPOSER).await?;
353 let Some(RepoRefs { repo, refs }) = found else {
354 if let Some(package) = existing {
355 self.drop_composer(&package).await?;
356 }
357 return Ok(false);
358 };
359 let workspace = repo.namespace.to_lowercase();
360 if self.db.workspace_hidden(&workspace).await? {
361 return Ok(false);
362 }
363 let default = refs.iter().find(|r| r.name == format!("refs/heads/{}", repo.default_branch)).map(|r| r.commit.clone());
364 let manifest = match &default {
365 Some(commit) => self.composer_json(repo_id, commit).await?,
366 None => None,
367 };
368 let Some((name, root_manifest)) = manifest else {
369 if let Some(package) = existing {
370 self.drop_composer(&package).await?;
371 }
372 return Ok(false);
373 };
374 // A repository moved to another workspace takes its package along.
375 let existing = match existing {
376 Some(package) if package.workspace != workspace => {
377 self.drop_composer(&package).await?;
378 None
379 }
380 other => other,
381 };
382 let now = now_ms();
383 let package = match existing {
384 Some(package) if package.name == name => package,
385 Some(package) => {
386 if self.db.package(&workspace, COMPOSER, &name).await?.is_some() {
387 worker::console_error!("packages: {workspace}/{} names {name}, which another repository has", repo.name);
388 package
389 } else {
390 self.db.rename_package(&package.id, &name, now).await?;
391 PackageRow { name: name.clone(), ..package }
392 }
393 }
394 None => {
395 if let Some(other) = self.db.package(&workspace, COMPOSER, &name).await?
396 && other.repo_id.as_deref() != Some(repo_id)
397 {
398 worker::console_error!("packages: {workspace}/{} names {name}, which another repository has", repo.name);
399 return Ok(false);
400 }
401 self.db
402 .create_package(&new_id("pkg", now), &workspace, COMPOSER, &name, Some((&repo.id, &repo.name, repo.is_private)), "g1t", now)
403 .await?
404 }
405 };
406
A public import copies every branch and tag, so an imported library keeps its releases407 // Versions follow git, so g1t records them: what made them is the
408 // push, already in the log as `git.push`.
409 let caller = Caller { actor: Some(AuditActor::system()) };
Composer from the workspace's own repositories, and go get from g1t.sh410 let wanted = wanted_versions(&refs, &repo.default_branch);
411 let stored = self.db.versions(&package.id, 1000).await?;
412 let mut manifests: HashMap<String, Option<Value>> = HashMap::new();
413 if let Some(commit) = &default {
414 manifests.insert(commit.clone(), Some(root_manifest.clone()));
415 }
416 let mut changed = false;
417 for (version, commit, git_ref, is_default) in &wanted {
418 let current = stored.iter().find(|row| row.version == version.version);
419 if let Some(row) = current
420 && row.digest == *commit
421 && row.meta()["default_branch"].as_bool().unwrap_or(false) == *is_default
422 {
423 continue;
424 }
425 if !manifests.contains_key(commit) {
426 let read = self.composer_json(repo_id, commit).await?.map(|(_, json)| json);
427 manifests.insert(commit.clone(), read);
428 }
429 // A ref without a composer.json of its own is not a version.
430 let Some(Some(json)) = manifests.get(commit) else { continue };
431 self.db
432 .replace_version(
433 &NewVersion {
434 id: new_id("ver", now),
435 package_id: package.id.clone(),
436 version: version.version.clone(),
437 digest: commit.clone(),
438 size: 0,
439 metadata: stored_metadata(json, version, git_ref, *is_default).to_string(),
440 subject: None,
441 published_by: None,
442 files: Vec::new(),
443 },
444 now,
445 )
446 .await?;
447 changed = true;
448 if current.is_none() {
449 let event = PackageEvent { version: Some(version.version.clone()), digest: Some(commit.clone()), ..self.event_of(&package) };
450 self.announce("package.published", &package, event, &caller).await;
A public import copies every branch and tag, so an imported library keeps its releases451 self.audit(&caller, "package.publish", &package, Some(&format!("{name}@{}", version.version)), None).await;
Composer from the workspace's own repositories, and go get from g1t.sh452 }
453 }
454 let kept: HashSet<&str> = wanted.iter().map(|(v, ..)| v.version.as_str()).collect();
455 for row in stored.iter().filter(|row| !kept.contains(row.version.as_str())) {
456 self.db.delete_version(&row.id).await?;
457 let event = PackageEvent { version: Some(row.version.clone()), digest: Some(row.digest.clone()), ..self.event_of(&package) };
458 self.announce("package.version_deleted", &package, event, &caller).await;
A public import copies every branch and tag, so an imported library keeps its releases459 self.audit(&caller, "package.delete_version", &package, Some(&format!("{}@{}", package.name, row.version)), None).await;
Composer from the workspace's own repositories, and go get from g1t.sh460 changed = true;
461 }
462 if let Some(commit) = &default {
463 self.composer_readme(&package, repo_id, commit, &root_manifest, now).await?;
464 }
465 if changed {
466 self.db.measure(&workspace).await?;
467 }
468 Ok(true)
469 }
470
471 /// The package's README and description, from the default branch.
472 async fn composer_readme(&self, package: &PackageRow, repo_id: &str, commit: &str, manifest: &Value, now: u64) -> Result<()> {
473 let mut readme = None;
474 for name in README_NAMES {
475 if let Some(bytes) = self.repo_file(repo_id, commit, name).await? {
476 readme = Some(bytes);
477 break;
478 }
479 }
480 let digest = match readme.filter(|b| !b.is_empty()) {
481 Some(bytes) => {
482 let digest = Digest::of(&bytes);
483 if self.db.blob(&digest).await?.is_none() {
484 self.store.put(&digest.object_key(), bytes.clone()).await?;
485 }
486 self.db.keep_blob(&package.id, &digest, bytes.len() as u64, Some("text/markdown"), &digest.object_key(), now).await?;
487 Some(digest.to_string())
488 }
489 None => None,
490 };
491 self.db.set_readme(&package.id, digest.as_deref(), manifest["description"].as_str(), now).await
492 }
493
494 /// A commit's `composer.json`, when it has one naming a valid package.
495 async fn composer_json(&self, repo_id: &str, commit: &str) -> Result<Option<(String, Value)>> {
496 let Some(bytes) = self.repo_file(repo_id, commit, "composer.json").await? else {
497 return Ok(None);
498 };
499 let Ok(json) = serde_json::from_slice::<Value>(&bytes) else {
500 return Ok(None);
501 };
502 let Some(name) = json["name"].as_str().map(str::to_lowercase).filter(|n| composer::valid_name(n)) else {
503 return Ok(None);
504 };
505 Ok(Some((name, json)))
506 }
507
508 async fn drop_composer(&self, package: &PackageRow) -> Result<()> {
509 self.db.delete_package(&package.id).await?;
510 self.db.measure(&package.workspace).await?;
A public import copies every branch and tag, so an imported library keeps its releases511 let caller = Caller { actor: Some(AuditActor::system()) };
512 self.announce("package.deleted", package, self.event_of(package), &caller).await;
513 self.audit(&caller, "package.delete", package, Some(&package.name), None).await;
Composer from the workspace's own repositories, and go get from g1t.sh514 Ok(())
515 }
516
517 /// Deletes the Composer package built from a deleted repository.
518 pub(crate) async fn composer_repo_gone(&self, repo_id: &str) -> Result<()> {
519 if let Some(package) = self.db.package_for_repo(repo_id, COMPOSER).await? {
520 self.drop_composer(&package).await?;
521 }
522 Ok(())
523 }
524
525 /// Reads a page of repositories the backfill has not yet, until it has
526 /// read them all once. Says how many were packages.
527 pub(crate) async fn composer_backfill(&self) -> Result<u32> {
528 let (after, finished) = self.db.backfill().await?;
529 if finished {
530 return Ok(0);
531 }
532 let page: IdPage = g1t_kit::call(&self.repos, "all_ids", &AllIdsArgs { after: after.clone(), limit: BACKFILL_PAGE }).await?;
533 let mut found = 0;
534 for id in &page.ids {
535 match self.sync_composer(id).await {
536 Ok(true) => found += 1,
537 Ok(false) => {}
538 Err(error) => worker::console_error!("packages: composer backfill of {id}: {error}"),
539 }
540 }
541 let last = page.ids.last().cloned().or(after);
542 self.db.set_backfill(last.as_deref(), page.next.is_none(), now_ms()).await?;
543 Ok(found)
544 }
545}
546
547#[cfg(test)]
548mod tests {
549 use super::*;
550 use g1t_contracts::repos::GitRefEntry;
551
552 #[test]
553 fn every_endpoint_is_routed() {
554 let commit = "a".repeat(40);
555 assert_eq!(route("/-/composer/acme/packages.json"), Some(ComposerRoute::Root { workspace: "acme".into() }));
556 assert_eq!(route("/-/composer/acme/"), Some(ComposerRoute::Root { workspace: "acme".into() }));
557 assert_eq!(
558 route("/-/composer/acme/p2/acme/lib.json"),
559 Some(ComposerRoute::Metadata { workspace: "acme".into(), name: "acme/lib".into(), dev: false })
560 );
561 assert_eq!(
562 route("/-/composer/acme/p2/acme/lib~dev.json"),
563 Some(ComposerRoute::Metadata { workspace: "acme".into(), name: "acme/lib".into(), dev: true })
564 );
565 assert_eq!(
566 route(&format!("/-/composer/acme/dist/acme/lib/{commit}.zip")),
567 Some(ComposerRoute::Dist { workspace: "acme".into(), name: "acme/lib".into(), commit: commit.clone() })
568 );
569 assert_eq!(route("/-/composer/acme/downloads"), Some(ComposerRoute::Downloads { workspace: "acme".into() }));
570 assert_eq!(route("/-/composer/acme/p2/Acme/lib.json"), None);
571 assert_eq!(route("/-/composer/acme/dist/acme/lib/short.zip"), None);
572 assert_eq!(route("/-/composer/acme"), None);
573 }
574
575 #[test]
576 fn a_repositorys_refs_make_its_versions() {
577 let entry = |name: &str, commit: &str| GitRefEntry { name: name.into(), commit: commit.into() };
578 let refs = [
579 entry("refs/heads/feature", "f"),
580 entry("refs/heads/main", "m"),
581 entry("refs/tags/v1.0.0", "a"),
582 entry("refs/tags/v1.1.0", "b"),
583 entry("refs/tags/nightly", "n"),
584 ];
585 let wanted = wanted_versions(&refs, "main");
586 let names: Vec<(&str, &str, bool)> = wanted.iter().map(|(v, c, _, d)| (v.version.as_str(), c.as_str(), *d)).collect();
587 assert_eq!(
588 names,
589 [("dev-main", "m", true), ("dev-feature", "f", false), ("v1.1.0", "b", false), ("v1.0.0", "a", false)],
590 "the default branch first, newest tags next, tags that are not versions left out"
591 );
592 }
593}