Sign in

flagon-io/g1t

public

g1t/services/packages/src/nuget_http.rs

495 lines25,400 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a6a121745e81f639f'1//! The NuGet feed: `g1t.sh/-/nuget/<workspace>/v3/index.json`, a v3 feed
2//! for each workspace. `dotnet nuget push` sends a g1t token as its API key
3//! (`X-NuGet-ApiKey`); restores send Basic credentials (any username, a g1t
4//! token as the password) from `nuget.config`, after the feed answers a
5//! private request with a `401`.
6//!
7//! A `.nupkg` is stored once, by its SHA-256, with its `.nuspec` beside it;
8//! the flat container, registration and search documents are made from the
9//! versions on each read. `dotnet nuget delete` unlists a version, as
10//! nuget.org does: it is still downloaded by those who name it.
11
12use g1t_contracts::User;
13use g1t_contracts::audit::AuditActor;
14use g1t_contracts::events::PackageEvent;
15use g1t_contracts::new_id;
16use g1t_kit::now_ms;
17use serde_json::{Value, json};
18use worker::{Context, Headers, Method, Request, Response, ResponseBody, Result, Url};
19
20use crate::access::{self, Action};
21use crate::db::{NewFile, NewVersion, PackageRow, VersionRow};
22use crate::digest::Digest;
23use crate::npm;
24use crate::nuget::{self, Content, Listed, NugetRoute};
25use crate::oci::{Credentials, origin, published_by};
26use crate::store::BlobStore;
27use crate::{Caller, Packages, TargetOf, token};
28
29const NUGET: &str = "nuget";
30/// The most versions a package's documents list.
31const MAX_VERSIONS: u32 = 5000;
32/// The longest README kept for a package's page.
33const MAX_README_BYTES: usize = 1024 * 1024;
34/// The most packages one search answers with.
35const MAX_SEARCH: u32 = 100;
36const DOCS: &str = "https://docs.g1t.sh/guides/nuget/";
37const TOKENS: &str = "https://g1t.sh/settings/tokens";
38
39/// A plain-text answer, which `dotnet` prints after the status.
40fn error(status: u16, message: impl Into<String>) -> Result<Response> {
41 let mut response = Response::ok(message.into())?.with_status(status);
42 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
43 if status == 401 {
44 response.headers_mut().set("www-authenticate", "Basic realm=\"g1t\"")?;
45 }
46 Ok(response)
47}
48
49fn sign_in() -> String {
50 format!("Sign in to use this feed: give the source a username and a g1t access token from {TOKENS} as its password. See {DOCS}")
51}
52
53fn json_response(value: &Value, head: bool) -> Result<Response> {
54 let mut response = if head { Response::empty()? } else { Response::from_json(value)? };
55 response.headers_mut().set("content-type", "application/json")?;
56 response.headers_mut().set("cache-control", "no-cache")?;
57 Ok(response)
58}
59
60impl Packages {
61 /// Answers a NuGet request.
62 pub async fn nuget(&self, request: Request, ctx: &Context) -> Result<Response> {
63 let url = request.url()?;
64 let Some((workspace, route)) = nuget::route(url.path()) else {
65 return error(404, "There is nothing at this address.");
66 };
67 match self.nuget_route(request, &url, &workspace, route, ctx).await {
68 Ok(response) => Ok(response),
69 Err(problem) => {
70 worker::console_error!("packages: nuget {}: {problem}", url.path());
71 error(500, "Something went wrong on our side. Try again in a moment.")
72 }
73 }
74 }
75
76 /// Who the request is from: the push's API key, or Basic credentials
77 /// (or a `Bearer` token) from the source's settings.
78 async fn nuget_credentials(&self, request: &Request) -> Result<Credentials> {
79 if let Some(key) = request.headers().get("x-nuget-apikey")?.map(|k| k.trim().to_owned()).filter(|k| !k.is_empty()) {
80 return Ok(match self.viewer_for("token", &key).await? {
81 Some(user) => Credentials::Viewer(Some(user)),
82 None => Credentials::Bad,
83 });
84 }
85 let Some(header) = request.headers().get("authorization")? else {
86 return Ok(Credentials::None);
87 };
88 let viewer = if let Some((username, secret)) = token::basic(&header) {
89 self.viewer_for(&username, &secret).await?
90 } else if let Some(bearer) = token::bearer(&header) {
91 self.viewer_for("token", bearer).await?
92 } else {
93 None
94 };
95 Ok(match viewer {
96 Some(user) => Credentials::Viewer(Some(user)),
97 None => Credentials::Bad,
98 })
99 }
100
101 async fn nuget_route(&self, mut request: Request, url: &Url, workspace: &str, route: NugetRoute, ctx: &Context) -> Result<Response> {
102 let method = request.method();
103 let credentials = self.nuget_credentials(&request).await?;
104 let read = matches!(method, Method::Get | Method::Head);
105 if read && let Some(refused) = self.limited(&request, &credentials, "a g1t token in the source's credentials").await? {
106 return Ok(refused);
107 }
108 let viewer = match credentials {
109 Credentials::Viewer(viewer) => viewer,
110 Credentials::None => None,
111 Credentials::Token(_) | Credentials::Bad => {
112 return error(401, format!("The token is not right, or has expired. Make an access token at {TOKENS}."));
113 }
114 };
115 let viewer = viewer.as_ref();
116 let base = format!("{}/-/nuget/{workspace}", origin(url));
117 let head = method == Method::Head;
118 match route {
119 NugetRoute::Index if read => {
120 if viewer.is_none() && self.db.has_private(workspace, NUGET).await? {
121 return error(401, sign_in());
122 }
123 json_response(&nuget::service_index(&base), head)
124 }
125 NugetRoute::Versions { id } if read => self.nuget_versions(workspace, &id, viewer, head).await,
126 NugetRoute::Content { id, version, file } if read => self.nuget_content(workspace, &id, &version, file, viewer, head, ctx).await,
127 NugetRoute::Registration { id } if read => self.nuget_registration(&base, workspace, &id, None, viewer, head).await,
128 NugetRoute::Leaf { id, version } if read => self.nuget_registration(&base, workspace, &id, Some(&version), viewer, head).await,
129 NugetRoute::Search if read => self.nuget_search(url, &base, workspace, viewer).await,
130 NugetRoute::Push if method == Method::Put => self.nuget_push(&mut request, workspace, viewer).await,
131 NugetRoute::Listing { id, version } if method == Method::Delete => self.nuget_listing(workspace, &id, &version, false, viewer).await,
132 NugetRoute::Listing { id, version } if method == Method::Post => self.nuget_listing(workspace, &id, &version, true, viewer).await,
133 _ => error(405, "Not a method this address takes."),
134 }
135 }
136
137 /// The package, by its id in any case, if its workspace is not deleted.
138 async fn nuget_package(&self, workspace: &str, id: &str) -> Result<Option<PackageRow>> {
139 Ok(self.db.package_any_case(workspace, NUGET, id).await?.filter(|p| !p.hidden()))
140 }
141
142 /// The answer for something not there: a `401` to someone not signed
143 /// in when the workspace has private packages, so the client sends its
144 /// credentials and a private package looks like a missing one.
145 async fn nuget_absent(&self, workspace: &str, viewer: Option<&User>) -> Result<Response> {
146 if viewer.is_none() && self.db.has_private(workspace, NUGET).await? {
147 return error(401, sign_in());
148 }
149 error(404, "Not found: no such package or version, or you cannot see it.")
150 }
151
152 async fn nuget_check(&self, viewer: Option<&User>, package: &PackageRow, action: Action) -> Result<Option<Response>> {
153 let target = TargetOf::package(package);
154 let decision = access::decide(viewer, &target.view(), action);
155 if decision.allowed {
156 return Ok(None);
157 }
158 let readable = action != Action::Pull && access::decide(viewer, &target.view(), Action::Pull).allowed;
159 if !readable && viewer.is_none() {
160 return Ok(Some(error(401, sign_in())?));
161 }
162 if !readable {
163 return Ok(Some(self.nuget_absent(&package.workspace, viewer).await?));
164 }
165 Ok(Some(error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()))?))
166 }
167
168 /// The package and its versions, oldest first, when the viewer may read it.
169 async fn nuget_readable(&self, workspace: &str, id: &str, viewer: Option<&User>) -> Result<std::result::Result<(PackageRow, Vec<VersionRow>), Response>> {
170 let Some(package) = self.nuget_package(workspace, id).await? else {
171 return Ok(Err(self.nuget_absent(workspace, viewer).await?));
172 };
173 if let Some(refusal) = self.nuget_check(viewer, &package, Action::Pull).await? {
174 return Ok(Err(refusal));
175 }
176 let mut versions = self.db.versions(&package.id, MAX_VERSIONS).await?;
177 if versions.is_empty() {
178 return Ok(Err(self.nuget_absent(workspace, viewer).await?));
179 }
180 versions.sort_by(|a, b| nuget::compare(&a.version, &b.version));
181 Ok(Ok((package, versions)))
182 }
183
184 /// The flat container's version list: every version, unlisted ones too.
185 async fn nuget_versions(&self, workspace: &str, id: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
186 let (_, versions) = match self.nuget_readable(workspace, id, viewer).await? {
187 Ok(found) => found,
188 Err(refused) => return Ok(refused),
189 };
190 let listed: Vec<String> = versions.iter().map(|v| v.version.to_ascii_lowercase()).collect();
191 json_response(&json!({ "versions": listed }), head)
192 }
193
194 /// A version's `.nupkg` or `.nuspec`.
195 #[allow(clippy::too_many_arguments)]
196 async fn nuget_content(&self, workspace: &str, id: &str, version: &str, file: Content, viewer: Option<&User>, head: bool, ctx: &Context) -> Result<Response> {
197 let (package, versions) = match self.nuget_readable(workspace, id, viewer).await? {
198 Ok(found) => found,
199 Err(refused) => return Ok(refused),
200 };
201 let wanted = nuget::normalize(version).unwrap_or_default().to_ascii_lowercase();
202 let Some(row) = versions.iter().find(|v| v.version.to_ascii_lowercase() == wanted) else {
203 return self.nuget_absent(workspace, viewer).await;
204 };
205 let name = match file {
206 Content::Nupkg => "nupkg",
207 Content::Nuspec => "nuspec",
208 };
209 let Some(kept) = self.db.file(&row.id, name).await? else {
210 return self.nuget_absent(workspace, viewer).await;
211 };
212 let Some(digest) = Digest::parse(&kept.digest) else {
213 return self.nuget_absent(workspace, viewer).await;
214 };
215 let Some(blob) = self.db.package_blob(&package.id, &digest).await? else {
216 return self.nuget_absent(workspace, viewer).await;
217 };
218 let headers = Headers::new();
219 headers.set("content-type", if file == Content::Nupkg { "application/octet-stream" } else { "application/xml" })?;
220 headers.set("content-length", &blob.size.to_string())?;
221 headers.set("cache-control", "max-age=31536000")?;
222 if head {
223 return Ok(Response::from_body(ResponseBody::Empty)?.with_headers(headers));
224 }
225 let Some(got) = self.store.get(&blob.object_key, None).await? else {
226 return self.nuget_absent(workspace, viewer).await;
227 };
228 if file == Content::Nupkg {
229 self.count_download(&package.id, ctx);
230 }
231 Ok(Response::from_body(got.body)?.with_headers(headers))
232 }
233
234 /// A package's registration index, or one version's leaf.
235 async fn nuget_registration(&self, base: &str, workspace: &str, id: &str, version: Option<&str>, viewer: Option<&User>, head: bool) -> Result<Response> {
236 let (package, versions) = match self.nuget_readable(workspace, id, viewer).await? {
237 Ok(found) => found,
238 Err(refused) => return Ok(refused),
239 };
240 let metadata: Vec<Value> = versions.iter().map(VersionRow::meta).collect();
241 let listed: Vec<Listed<'_>> = versions
242 .iter()
243 .zip(&metadata)
244 .map(|(row, metadata)| Listed { version: &row.version, metadata, published: &row.published_at, listed: !row.is_yanked(), downloads: 0 })
245 .collect();
246 match version {
247 None => json_response(&nuget::registration(base, &package.name, &listed), head),
248 Some(version) => {
249 let wanted = nuget::normalize(version).unwrap_or_default().to_ascii_lowercase();
250 let Some(one) = listed.iter().find(|v| v.version.to_ascii_lowercase() == wanted) else {
251 return self.nuget_absent(workspace, viewer).await;
252 };
253 json_response(&nuget::leaf(base, &package.name, one), head)
254 }
255 }
256 }
257
258 /// Search: the workspace's packages the viewer may see whose id or
259 /// description holds the query, with their listed versions.
260 async fn nuget_search(&self, url: &Url, base: &str, workspace: &str, viewer: Option<&User>) -> Result<Response> {
261 let query = |key: &str| url.query_pairs().find(|(k, _)| k == key).map(|(_, v)| v.into_owned());
262 let q = query("q").unwrap_or_default().trim().to_ascii_lowercase();
263 let skip = query("skip").and_then(|v| v.parse::<usize>().ok()).unwrap_or(0);
264 let take = query("take").and_then(|v| v.parse::<usize>().ok()).unwrap_or(20).min(MAX_SEARCH as usize);
265 let prerelease = query("prerelease").is_some_and(|v| v.eq_ignore_ascii_case("true"));
266 if viewer.is_none() && self.db.has_private(workspace, NUGET).await? {
267 return error(401, sign_in());
268 }
269 let packages = self.db.packages_of(workspace, NUGET, 1000).await?;
270 let versions = self.db.ecosystem_versions(workspace, NUGET, 20_000).await?;
271 let mut found = Vec::new();
272 for package in &packages {
273 if !access::decide(viewer, &TargetOf::package(package).view(), Action::Pull).allowed {
274 continue;
275 }
276 let rows: Vec<&VersionRow> = versions
277 .iter()
278 .filter(|v| v.package_id == package.id && (prerelease || !nuget::is_prerelease(&v.version)))
279 .collect();
280 let metadata: Vec<Value> = rows.iter().map(|v| v.meta()).collect();
281 let matches = q.is_empty()
282 || package.name.to_ascii_lowercase().contains(&q)
283 || package.description.as_deref().is_some_and(|d| d.to_ascii_lowercase().contains(&q));
284 if !matches {
285 continue;
286 }
287 let mut listed: Vec<Listed<'_>> = rows
288 .iter()
289 .zip(&metadata)
290 .map(|(row, metadata)| Listed { version: &row.version, metadata, published: &row.published_at, listed: !row.is_yanked(), downloads: 0 })
291 .collect();
292 listed.sort_by(|a, b| nuget::compare(a.version, b.version));
293 if let Some(mut result) = nuget::search_result(base, &package.name, &listed) {
294 result["totalDownloads"] = json!(package.downloads);
295 found.push(result);
296 }
297 }
298 let total = found.len();
299 let data: Vec<Value> = found.into_iter().skip(skip).take(take).collect();
300 json_response(&json!({ "totalHits": total, "data": data }), false)
301 }
302
303 /// The package a first push makes, linked to the repository its
304 /// `.nuspec` names on g1t, or else one named like its id.
305 async fn nuget_target(&self, workspace: &str, id: &str, repository: Option<&str>) -> Result<TargetOf> {
306 let named = repository
307 .and_then(|url| npm::repository_of(&Value::String(url.to_owned()), &self.host))
308 .filter(|(owner, _)| owner == workspace)
309 .map(|(_, repo)| repo);
310 let lower = id.to_ascii_lowercase();
311 let dashed = lower.replace('.', "-");
312 let mut repo = None;
313 for candidate in named.iter().map(String::as_str).chain([lower.as_str(), dashed.as_str()]) {
314 if let Some(found) = self.repo_by_name(workspace, candidate).await? {
315 repo = Some(found);
316 break;
317 }
318 }
319 Ok(TargetOf { workspace: workspace.to_owned(), repo: repo.map(|r| (r.id, r.name, r.is_private)), public: false })
320 }
321
322 /// `dotnet nuget push`: a `PUT` of the `.nupkg`, in a multipart body.
323 async fn nuget_push(&self, request: &mut Request, workspace: &str, viewer: Option<&User>) -> Result<Response> {
324 let declared = request.headers().get("content-length")?.and_then(|n| n.parse::<u64>().ok());
325 let too_large = || {
326 let mb = self.max_request / 1_000_000;
327 error(413, format!("A push may be at most {mb} MB. See {DOCS}#size"))
328 };
329 if declared.is_some_and(|n| n > self.max_request) {
330 return too_large();
331 }
332 let content_type = request.headers().get("content-type")?;
333 let body = request.bytes().await?;
334 if body.len() as u64 > self.max_request {
335 return too_large();
336 }
337 if viewer.is_none() {
338 return error(401, format!("Push with a g1t access token as the API key: dotnet nuget push <file> --api-key <token>. Make one at {TOKENS}."));
339 }
340 let nupkg = match nuget::pushed_file(content_type.as_deref(), &body) {
341 Ok(file) => file,
342 Err(message) => return error(400, message),
343 };
344 let read = match nuget::read_package(nupkg) {
345 Ok(read) => read,
346 Err(message) => return error(400, message),
347 };
348 let spec = &read.nuspec;
349 if !nuget::valid_id(&spec.id) {
350 return error(400, format!("{} is not a valid package id: letters, digits and _, in parts joined by ., - or _.", spec.id));
351 }
352 let Some(version) = nuget::normalize(&spec.version) else {
353 return error(400, format!("{} is not a version NuGet reads.", spec.version));
354 };
355
356 let found = self.db.package_any_case(workspace, NUGET, &spec.id).await?;
357 if found.as_ref().is_some_and(PackageRow::hidden) || (found.is_none() && self.db.workspace_hidden(workspace).await?) {
358 return error(403, format!("The workspace {workspace} is deleted; nothing can be pushed to it."));
359 }
360 let target = match &found {
361 Some(package) => TargetOf::package(package),
362 None => self.nuget_target(workspace, &spec.id, spec.repository_url.as_deref()).await?,
363 };
364 let decision = access::decide(viewer, &target.view(), Action::Push);
365 if !decision.allowed {
366 let readable = found.is_none() || access::decide(viewer, &target.view(), Action::Pull).allowed;
367 if !readable {
368 return error(404, "Not found: no such package, or you cannot see it.");
369 }
370 return error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()));
371 }
372 let caller = Caller { actor: viewer.map(AuditActor::of) };
373 let package = match found {
374 Some(package) => package,
375 None => {
376 self.db
377 .create_package(
378 &new_id("pkg", now_ms()),
379 workspace,
380 NUGET,
381 &spec.id,
382 target.repo.as_ref().map(|(id, repo, private)| (id.as_str(), repo.as_str(), *private)),
383 caller.actor.as_ref().map_or("", |actor| actor.actor_id.as_str()),
384 now_ms(),
385 )
386 .await?
387 }
388 };
389 let existing = self.db.versions(&package.id, MAX_VERSIONS).await?;
390 if let Some(taken) = existing.iter().find(|v| v.version.eq_ignore_ascii_case(&version)) {
391 return error(409, format!("{} {} is already pushed, and a version is pushed once. Bump the version.", package.name, taken.version));
392 }
393
394 let nupkg = nupkg.to_vec();
395 let digest = Digest::of(&nupkg);
396 let size = nupkg.len() as u64;
397 let nuspec_digest = Digest::of(&read.nuspec_bytes);
398 let nuspec_size = read.nuspec_bytes.len() as u64;
399 let files = [(digest.to_string(), size), (nuspec_digest.to_string(), nuspec_size)];
400 if let Some(refusal) = self.storage_refusal(&package, &files).await? {
401 return error(403, refusal);
402 }
403 let now = now_ms();
404 for (digest, bytes, media_type) in [(&digest, nupkg, "application/octet-stream"), (&nuspec_digest, read.nuspec_bytes.clone(), "application/xml")] {
405 let stored = match self.db.blob(digest).await? {
406 Some(blob) => self.store.head(&blob.object_key).await?.is_some(),
407 None => false,
408 };
409 let length = bytes.len() as u64;
410 if !stored {
411 self.store.put(&digest.object_key(), bytes).await?;
412 }
413 self.db.keep_blob(&package.id, digest, length, Some(media_type), &digest.object_key(), now).await?;
414 }
415 self.db
416 .publish(
417 NewVersion {
418 id: new_id("ver", now),
419 package_id: package.id.clone(),
420 version: version.clone(),
421 digest: digest.to_string(),
422 size: size + nuspec_size,
423 metadata: nuget::stored(spec, &version).to_string(),
424 subject: None,
425 published_by: published_by(&caller),
426 files: vec![
427 NewFile { name: "nupkg".to_owned(), digest: digest.to_string(), size, media_type: Some("application/octet-stream".to_owned()) },
428 NewFile {
429 name: "nuspec".to_owned(),
430 digest: nuspec_digest.to_string(),
431 size: nuspec_size,
432 media_type: Some("application/xml".to_owned()),
433 },
434 ],
435 },
436 None,
437 now,
438 )
439 .await?;
440 // The README and description the page shows: the highest stable
441 // version's, so a pre-release does not replace them.
442 let highest = !nuget::is_prerelease(&version)
443 && existing.iter().filter(|v| !nuget::is_prerelease(&v.version)).all(|v| nuget::compare(&v.version, &version).is_lt());
444 let first = nuget::is_prerelease(&version) && existing.is_empty();
445 if highest || first {
446 let readme = read.readme.as_deref().map(str::trim).filter(|r| !r.is_empty() && r.len() <= MAX_README_BYTES);
447 let readme_digest = match readme {
448 Some(readme) => {
449 let bytes = readme.as_bytes().to_vec();
450 let digest = Digest::of(&bytes);
451 if self.db.blob(&digest).await?.is_none() {
452 self.store.put(&digest.object_key(), bytes.clone()).await?;
453 }
454 self.db.keep_blob(&package.id, &digest, bytes.len() as u64, Some("text/markdown"), &digest.object_key(), now).await?;
455 Some(digest.to_string())
456 }
457 None => None,
458 };
459 self.db.set_readme(&package.id, readme_digest.as_deref(), spec.description.as_deref(), now).await?;
460 }
461 self.db.measure(&package.workspace).await?;
462 let event = PackageEvent {
463 version: Some(version.clone()),
464 digest: Some(digest.to_string()),
465 size: Some(size),
466 ..self.event_of(&package)
467 };
468 self.announce("package.published", &package, event, &caller).await;
469 self.audit(&caller, "package.publish", &package, Some(&format!("{workspace}/{}@{version}", package.name)), None).await;
470 error(201, format!("{} {version} was pushed.", package.name))
471 }
472
473 /// `dotnet nuget delete` unlists a version; a `POST` lists it again.
474 async fn nuget_listing(&self, workspace: &str, id: &str, version: &str, listed: bool, viewer: Option<&User>) -> Result<Response> {
475 let Some(package) = self.nuget_package(workspace, id).await? else {
476 return self.nuget_absent(workspace, viewer).await;
477 };
478 if let Some(refusal) = self.nuget_check(viewer, &package, Action::Push).await? {
479 return Ok(refusal);
480 }
481 let wanted = nuget::normalize(version).unwrap_or_default();
482 let versions = self.db.versions(&package.id, MAX_VERSIONS).await?;
483 let Some(row) = versions.iter().find(|v| v.version.eq_ignore_ascii_case(&wanted)) else {
484 return error(404, format!("{} {version} is not there.", package.name));
485 };
486 if row.is_yanked() == listed {
487 self.db.set_yanked(&row.id, !listed).await?;
488 self.db.touch_package(&package.id, now_ms()).await?;
489 let caller = Caller { actor: viewer.map(AuditActor::of) };
490 let action = if listed { "package.relist" } else { "package.unlist" };
491 self.audit(&caller, action, &package, Some(&format!("{workspace}/{}@{}", package.name, row.version)), None).await;
492 }
493 Ok(Response::empty()?.with_status(if listed { 200 } else { 204 }))
494 }
495}

This file's history is long; its oldest lines are credited to the oldest commit read.