Sign in

flagon-io/g1t

public

g1t/services/packages/src/cargo_http.rs

462 lines21,522 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Cargo on g1t.sh: a sparse registry per workspace, cargo publish/add/yank/search with a g1t token; npm rows show their version1//! The Cargo registry: `g1t.sh/-/cargo/<workspace>/`, a sparse registry
2//! for each workspace. `.cargo/config.toml` names it, and `cargo login`
3//! keeps a g1t token for it:
4//!
5//! ```toml
6//! [registries.acme]
7//! index = "sparse+https://g1t.sh/-/cargo/acme/index/"
8//! ```
9//!
10//! Cargo sends the token as the whole `Authorization` header, with no
11//! scheme. The index is made from the versions on each read; a `.crate`
12//! is stored once, by its SHA-256, which is also its index `cksum`.
13
14use g1t_contracts::User;
15use g1t_contracts::audit::AuditActor;
16use g1t_contracts::events::PackageEvent;
17use g1t_contracts::new_id;
18use g1t_kit::now_ms;
19use serde_json::{Value, json};
20use worker::{Context, Headers, Method, Request, Response, ResponseBody, Result, Url};
21
22use crate::access::{self, Action};
23use crate::cargo::{self, CargoRoute};
24use crate::db::{NewFile, NewVersion, PackageRow, VersionRow};
25use crate::digest::Digest;
26use crate::npm;
27use crate::oci::{Credentials, origin, published_by};
28use crate::store::BlobStore;
29use crate::{Caller, Packages, TargetOf, token};
30
31const CARGO: &str = "cargo";
32/// The most versions an index file lists.
33const MAX_VERSIONS: u32 = 5000;
34/// The longest README kept for a crate's page.
35const MAX_README_BYTES: usize = 1024 * 1024;
36/// The most crates `cargo search` is answered with.
37const MAX_SEARCH: u32 = 100;
38const DOCS: &str = "https://docs.g1t.sh/guides/cargo/";
39const TOKENS: &str = "https://g1t.sh/settings/tokens";
40
41/// Cargo's error shape: `{"errors": [{"detail": "..."}]}`, which it prints.
42/// A 401 says where to get a token, which cargo shows beside its own hint.
43fn error(status: u16, message: impl Into<String>) -> Result<Response> {
44 let mut response = Response::from_json(&json!({ "errors": [{ "detail": message.into() }] }))?.with_status(status);
45 if status == 401 {
46 response.headers_mut().set("www-authenticate", &format!("Cargo login_url=\"{TOKENS}\""))?;
47 }
48 Ok(response)
49}
50
51fn ok() -> Result<Response> {
52 Response::from_json(&json!({ "ok": true }))
53}
54
55fn not_found() -> Result<Response> {
56 error(404, "Not found: no such crate, or you cannot see it. Private crates need a token: cargo login --registry <workspace>.")
57}
58
59fn sign_in(workspace: &str) -> String {
60 format!("Sign in to use this registry: cargo login --registry {workspace}, with a g1t access token from {TOKENS}")
61}
62
63impl Packages {
64 /// Answers a Cargo request.
65 pub async fn cargo(&self, request: Request, ctx: &Context) -> Result<Response> {
66 let url = request.url()?;
67 let Some((workspace, route)) = cargo::route(url.path()) else {
68 return error(404, "There is nothing at this address.");
69 };
70 match self.cargo_route(request, &url, &workspace, route, ctx).await {
71 Ok(response) => Ok(response),
72 Err(problem) => {
73 worker::console_error!("packages: cargo {}: {problem}", url.path());
74 error(500, "Something went wrong on our side. Try again in a moment.")
75 }
76 }
77 }
78
79 /// Who the request is from: cargo's bare token, a `Bearer` one, or
80 /// Basic credentials with a g1t token as the password.
81 async fn cargo_credentials(&self, request: &Request) -> Result<Credentials> {
82 let Some(header) = request.headers().get("authorization")? else {
83 return Ok(Credentials::None);
84 };
85 let viewer = if let Some((username, secret)) = token::basic(&header) {
86 self.viewer_for(&username, &secret).await?
87 } else if let Some(token) = cargo::token(&header) {
88 self.viewer_for("token", token).await?
89 } else {
90 None
91 };
92 Ok(match viewer {
93 Some(user) => Credentials::Viewer(Some(user)),
94 None => Credentials::Bad,
95 })
96 }
97
98 async fn cargo_route(&self, mut request: Request, url: &Url, workspace: &str, route: CargoRoute, ctx: &Context) -> Result<Response> {
99 let method = request.method();
100 let credentials = self.cargo_credentials(&request).await?;
101 if matches!(method, Method::Get | Method::Head)
102 && let Some(refused) = self.limited(&request, &credentials, &format!("a token (cargo login --registry {workspace})")).await?
103 {
104 return Ok(refused);
105 }
106 let viewer = match credentials {
107 Credentials::Viewer(viewer) => viewer,
108 Credentials::None => None,
109 Credentials::Token(_) | Credentials::Bad => {
110 return error(
111 401,
112 format!("The token is not right, or has expired. Make an access token at {TOKENS}, then: cargo login --registry {workspace}"),
113 );
114 }
115 };
116 let viewer = viewer.as_ref();
117 let read = matches!(method, Method::Get | Method::Head);
118 match route {
119 CargoRoute::Config if read => self.cargo_config(url, workspace, viewer).await,
120 CargoRoute::Index { name } if read => self.cargo_index(&request, workspace, &name, viewer).await,
121 CargoRoute::Download { name, version } if read => self.crate_download(workspace, &name, &version, viewer, method == Method::Head, ctx).await,
122 CargoRoute::Search if read => self.cargo_search(url, workspace, viewer).await,
123 CargoRoute::Publish if method == Method::Put => self.cargo_publish(&mut request, workspace, viewer).await,
124 CargoRoute::Yank { name, version } if method == Method::Delete => self.cargo_yank(workspace, &name, &version, true, viewer).await,
125 CargoRoute::Unyank { name, version } if method == Method::Put => self.cargo_yank(workspace, &name, &version, false, viewer).await,
126 CargoRoute::Owners { .. } => error(
127 400,
128 "Crate owners are not kept here: who may publish a crate is decided by its repository's roles, or the workspace's. See https://docs.g1t.sh/guides/packages/#who-can-see-and-publish-a-package",
129 ),
130 _ => error(405, "Not a method this address takes."),
131 }
132 }
133
134 /// The crate, by its name in any case, if its workspace is not deleted.
135 async fn crate_package(&self, workspace: &str, name: &str) -> Result<Option<PackageRow>> {
136 Ok(self.db.package_any_case(workspace, CARGO, name).await?.filter(|p| !p.hidden()))
137 }
138
139 /// Whether `viewer` may `action` the crate, as the answer when not: 401
140 /// for someone not signed in who may not read it, 404 for anyone else
141 /// who may not read it, and 403 with the reason for one who may.
142 fn cargo_check(&self, viewer: Option<&User>, package: &PackageRow, action: Action) -> Option<Result<Response>> {
143 let target = TargetOf::package(package);
144 let decision = access::decide(viewer, &target.view(), action);
145 if decision.allowed {
146 return None;
147 }
148 let readable = action != Action::Pull && access::decide(viewer, &target.view(), Action::Pull).allowed;
149 if !readable && viewer.is_none() {
150 return Some(error(401, sign_in(&package.workspace)));
151 }
152 if !readable {
153 return Some(not_found());
154 }
155 Some(error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned())))
156 }
157
158 /// `index/config.json`. Signed in, cargo is told to send its token with
159 /// every request. Anonymous requests to a workspace that has private
160 /// crates get a 401, which makes cargo ask again with its token; one
161 /// with only public crates is open to anyone.
162 async fn cargo_config(&self, url: &Url, workspace: &str, viewer: Option<&User>) -> Result<Response> {
163 if viewer.is_none() && self.db.has_private(workspace, CARGO).await? {
164 return error(401, sign_in(workspace));
165 }
166 let base = format!("{}/-/cargo/{workspace}", origin(url));
167 let mut response = Response::from_json(&cargo::config(&base, viewer.is_some()))?;
168 response.headers_mut().set("cache-control", "no-cache")?;
169 Ok(response)
170 }
171
172 /// A crate's index file: one line per version, oldest first.
173 async fn cargo_index(&self, request: &Request, workspace: &str, name: &str, viewer: Option<&User>) -> Result<Response> {
174 let Some(package) = self.crate_package(workspace, name).await? else {
175 return not_found();
176 };
177 if let Some(refusal) = self.cargo_check(viewer, &package, Action::Pull) {
178 return refusal;
179 }
180 let mut versions = self.db.versions(&package.id, MAX_VERSIONS).await?;
181 if versions.is_empty() {
182 return not_found();
183 }
184 versions.reverse();
185 let mut body = String::new();
186 for version in &versions {
187 body.push_str(&cargo::index_line(&version.meta(), version.is_yanked()));
188 body.push('\n');
189 }
190 let etag = format!("\"{}\"", &Digest::of(body.as_bytes()).hex()[..32]);
191 let headers = Headers::new();
192 headers.set("content-type", "text/plain; charset=utf-8")?;
193 headers.set("cache-control", "no-cache")?;
194 headers.set("etag", &etag)?;
195 if request.headers().get("if-none-match")?.is_some_and(|sent| sent == etag) {
196 return Ok(Response::empty()?.with_status(304).with_headers(headers));
197 }
198 Ok(Response::ok(body)?.with_headers(headers))
199 }
200
201 async fn crate_download(&self, workspace: &str, name: &str, version: &str, viewer: Option<&User>, head: bool, ctx: &Context) -> Result<Response> {
202 let Some(package) = self.crate_package(workspace, name).await? else {
203 return not_found();
204 };
205 if let Some(refusal) = self.cargo_check(viewer, &package, Action::Pull) {
206 return refusal;
207 }
208 let gone = || error(404, format!("{name}@{version} is not there."));
209 let Some(row) = self.db.version_named(&package.id, version).await? else {
210 return gone();
211 };
212 let Some(digest) = Digest::parse(&row.digest) else {
213 return gone();
214 };
215 let Some(blob) = self.db.package_blob(&package.id, &digest).await? else {
216 return gone();
217 };
218 let headers = Headers::new();
219 headers.set("content-type", "application/gzip")?;
220 headers.set("content-length", &blob.size.to_string())?;
221 headers.set("cache-control", "max-age=31536000")?;
222 if head {
223 return Ok(Response::from_body(ResponseBody::Empty)?.with_headers(headers));
224 }
225 let Some(got) = self.store.get(&blob.object_key, None).await? else {
226 return gone();
227 };
228 self.count_download(&package.id, ctx);
229 Ok(Response::from_body(got.body)?.with_headers(headers))
230 }
231
232 /// `cargo search`: the workspace's crates the viewer may see, by name.
233 async fn cargo_search(&self, url: &Url, workspace: &str, viewer: Option<&User>) -> Result<Response> {
234 let query = url.query_pairs().find(|(k, _)| k == "q").map(|(_, v)| v.into_owned()).unwrap_or_default();
235 let per_page = url
236 .query_pairs()
237 .find(|(k, _)| k == "per_page")
238 .and_then(|(_, v)| v.parse::<u32>().ok())
239 .unwrap_or(10)
240 .clamp(1, MAX_SEARCH);
241 let rows = self.db.list(workspace, Some(CARGO), None, Some(&query), MAX_SEARCH).await?;
242 let visible: Vec<_> = rows
243 .iter()
244 .filter(|row| access::decide(viewer, &TargetOf::package(&row.package).view(), Action::Pull).allowed)
245 .collect();
246 let crates: Vec<Value> = visible
247 .iter()
248 .take(per_page as usize)
249 .map(|row| {
250 json!({
251 "name": row.package.name,
252 "max_version": crate::db::latest_shown(row).unwrap_or_default(),
253 "description": row.package.description,
254 })
255 })
256 .collect();
257 Response::from_json(&json!({ "crates": crates, "meta": { "total": visible.len() } }))
258 }
259
260 /// The package publishing makes, linked to the repository the crate's
261 /// `repository` names on g1t, or else the one named like it.
262 async fn cargo_target(&self, workspace: &str, name: &str, metadata: &Value) -> Result<TargetOf> {
263 let named = npm::repository_of(&metadata["repository"], &self.host)
264 .filter(|(owner, _)| owner == workspace)
265 .map(|(_, repo)| repo);
266 let lower = name.to_ascii_lowercase();
267 let dashed = lower.replace('_', "-");
268 let mut repo = None;
269 for candidate in named.iter().map(String::as_str).chain([lower.as_str(), dashed.as_str()]) {
270 if let Some(found) = self.repo_by_name(workspace, candidate).await? {
271 repo = Some(found);
272 break;
273 }
274 }
275 Ok(TargetOf {
276 workspace: workspace.to_owned(),
277 repo: repo.map(|r| (r.id, r.name, r.is_private)),
278 public: false,
279 })
280 }
281
282 /// `cargo publish`: the metadata and the `.crate` in one body.
283 async fn cargo_publish(&self, request: &mut Request, workspace: &str, viewer: Option<&User>) -> Result<Response> {
284 let declared = request.headers().get("content-length")?.and_then(|n| n.parse::<u64>().ok());
285 let too_large = || {
286 let mb = self.max_request / 1_000_000;
287 error(413, format!("A publish may be at most {mb} MB, the .crate file and its metadata together. See {DOCS}#size"))
288 };
289 if declared.is_some_and(|n| n > self.max_request) {
290 return too_large();
291 }
292 let bytes = request.bytes().await?;
293 if bytes.len() as u64 > self.max_request {
294 return too_large();
295 }
296 let (metadata, krate) = match cargo::parse_publish(&bytes) {
297 Ok(parts) => parts,
298 Err(message) => return error(400, message),
299 };
300 let name = metadata["name"].as_str().unwrap_or("").to_owned();
301 if let Err(message) = cargo::valid_name(&name) {
302 return error(400, message);
303 }
304 let version = metadata["vers"].as_str().unwrap_or("").to_owned();
305 if !npm::valid_version(&version) {
306 return error(400, format!("{version} is not a semver version."));
307 }
308 if krate.is_empty() {
309 return error(400, "The .crate file is empty.");
310 }
311 let digest = Digest::of(krate);
312 let entry = match cargo::index_entry(&metadata, digest.hex()) {
313 Ok(entry) => entry,
314 Err(message) => return error(400, message),
315 };
316
317 // A name is taken whatever its case, and `-` and `_` are one.
318 let found = self.db.package_folded(workspace, CARGO, &cargo::folded(&name)).await?;
319 if let Some(found) = &found {
320 if found.hidden() {
321 return error(403, format!("The workspace {workspace} is deleted; nothing can be published to it."));
322 }
323 if found.name != name {
324 return error(400, format!("The name {name} is taken by the crate {}. Publish it under that name.", found.name));
325 }
326 } else if self.db.workspace_hidden(workspace).await? {
327 return error(403, format!("The workspace {workspace} is deleted; nothing can be published to it."));
328 }
329 let target = match &found {
330 Some(package) => TargetOf::package(package),
331 None => self.cargo_target(workspace, &name, &metadata).await?,
332 };
333 let decision = access::decide(viewer, &target.view(), Action::Push);
334 if !decision.allowed {
335 if viewer.is_none() {
336 return error(401, sign_in(workspace));
337 }
338 let readable = found.is_none() || access::decide(viewer, &target.view(), Action::Pull).allowed;
339 if !readable {
340 return not_found();
341 }
342 return error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()));
343 }
344 let caller = Caller { actor: viewer.map(AuditActor::of) };
345 let package = match found {
346 Some(package) => package,
347 None => {
348 self.db
349 .create_package(
350 &new_id("pkg", now_ms()),
351 workspace,
352 CARGO,
353 &name,
354 target.repo.as_ref().map(|(id, repo, private)| (id.as_str(), repo.as_str(), *private)),
355 caller.actor.as_ref().map_or("", |actor| actor.actor_id.as_str()),
356 now_ms(),
357 )
358 .await?
359 }
360 };
361 let existing = self.db.versions(&package.id, MAX_VERSIONS).await?;
362 if let Some(taken) = existing.iter().find(|v| cargo::without_build(&v.version) == cargo::without_build(&version)) {
363 return error(
364 400,
365 format!("{name}@{} is already published, and a version is published once. Bump the version in Cargo.toml.", taken.version),
366 );
367 }
368
369 let size = krate.len() as u64;
370 if let Some(refusal) = self.storage_refusal(&package, &[(digest.to_string(), size)]).await? {
371 return error(403, refusal);
372 }
373 let now = now_ms();
374 let stored = match self.db.blob(&digest).await? {
375 Some(blob) => self.store.head(&blob.object_key).await?.is_some(),
376 None => false,
377 };
378 if !stored {
379 self.store.put(&digest.object_key(), krate.to_vec()).await?;
380 }
381 self.db
382 .keep_blob(&package.id, &digest, size, Some("application/gzip"), &digest.object_key(), now)
383 .await?;
384 self.db
385 .publish(
386 NewVersion {
387 id: new_id("ver", now),
388 package_id: package.id.clone(),
389 version: version.clone(),
390 digest: digest.to_string(),
391 size,
392 metadata: entry.to_string(),
393 subject: None,
394 published_by: published_by(&caller),
395 files: vec![NewFile {
396 name: "crate".to_owned(),
397 digest: digest.to_string(),
398 size,
399 media_type: Some("application/gzip".to_owned()),
400 }],
401 },
402 None,
403 now,
404 )
405 .await?;
406 // The README and description the crate's page shows: the highest
407 // stable version's, so a pre-release does not replace them.
408 let highest = std::iter::once(version.as_str())
409 .chain(existing.iter().map(|v| v.version.as_str()))
410 .collect::<Vec<_>>()
411 .join("\n");
412 if crate::db::newest_version(&highest).as_deref() == Some(version.as_str()) {
413 let readme = metadata["readme"].as_str().unwrap_or("").trim();
414 let readme_digest = if readme.is_empty() || readme.len() > MAX_README_BYTES {
415 None
416 } else {
417 let bytes = readme.as_bytes().to_vec();
418 let digest = Digest::of(&bytes);
419 if self.db.blob(&digest).await?.is_none() {
420 self.store.put(&digest.object_key(), bytes.clone()).await?;
421 }
422 self.db
423 .keep_blob(&package.id, &digest, bytes.len() as u64, Some("text/markdown"), &digest.object_key(), now)
424 .await?;
425 Some(digest.to_string())
426 };
427 self.db.set_readme(&package.id, readme_digest.as_deref(), metadata["description"].as_str(), now).await?;
428 }
429 self.db.measure(&package.workspace).await?;
430 let event = PackageEvent {
431 version: Some(version.clone()),
432 digest: Some(digest.to_string()),
433 size: Some(size),
434 ..self.event_of(&package)
435 };
436 self.announce("package.published", &package, event, &caller).await;
437 self.audit(&caller, "package.publish", &package, Some(&format!("{workspace}/{name}@{version}")), None).await;
438 Response::from_json(&json!({ "warnings": { "invalid_categories": [], "invalid_badges": [], "other": [] } }))
439 }
440
441 /// `cargo yank` and `cargo yank --undo`: the version stays, for
442 /// lockfiles that name it, but is no longer picked for new ones.
443 async fn cargo_yank(&self, workspace: &str, name: &str, version: &str, yank: bool, viewer: Option<&User>) -> Result<Response> {
444 let Some(package) = self.crate_package(workspace, name).await? else {
445 return not_found();
446 };
447 if let Some(refusal) = self.cargo_check(viewer, &package, Action::Push) {
448 return refusal;
449 }
450 let Some(row): Option<VersionRow> = self.db.version_named(&package.id, version).await? else {
451 return error(404, format!("{name}@{version} is not there."));
452 };
453 if row.is_yanked() != yank {
454 self.db.set_yanked(&row.id, yank).await?;
455 self.db.touch_package(&package.id, now_ms()).await?;
456 let caller = Caller { actor: viewer.map(AuditActor::of) };
457 let action = if yank { "package.yank" } else { "package.unyank" };
458 self.audit(&caller, action, &package, Some(&format!("{workspace}/{}@{version}", package.name)), None).await;
459 }
460 ok()
461 }
462}