Sign in

flagon-io/g1t

public

g1t/services/packages/src/rubygems_http.rs

481 lines24,348 bytesCodeBlame
1//! The RubyGems registry: `g1t.sh/-/rubygems/<workspace>/`, one for each
2//! workspace. `gem push --host` sends a g1t token as the whole
3//! `Authorization` header (`GEM_HOST_API_KEY`, or `~/.gem/credentials`);
4//! Bundler sends Basic credentials (any username, a g1t token as the
5//! password) from `bundle config`.
6//!
7//! Bundler installs from the compact index (`versions`, `info/<gem>`,
8//! `names`), made from the versions on each read, with each file's MD5 as
9//! its `ETag` as Bundler checks it. `gem install --source` and `gem search`
10//! read the full index: `specs.4.8.gz` (and `latest_` and `prerelease_`),
11//! and a version's `quick/Marshal.4.8/<gem>.gemspec.rz`, made from what
12//! each version keeps, in Ruby's Marshal format. A `.gem` is stored once, by its
13//! SHA-256, which is also its index `checksum`. `gem yank` takes a version
14//! out of the index; its file stays for lockfiles that name it.
15
16use g1t_contracts::User;
17use g1t_contracts::audit::AuditActor;
18use g1t_contracts::events::PackageEvent;
19use g1t_contracts::new_id;
20use g1t_kit::now_ms;
21use serde_json::Value;
22use worker::{Context, Headers, Method, Request, Response, ResponseBody, Result, Url};
23
24use crate::access::{self, Action};
25use crate::db::{NewFile, NewVersion, PackageRow, VersionRow};
26use crate::digest::Digest;
27use crate::oci::{Credentials, published_by};
28use crate::rubygems::{self, GemRoute};
29use crate::store::BlobStore;
30use crate::{Caller, Packages, TargetOf, cargo, npm, token};
31
32const RUBYGEMS: &str = "rubygems";
33/// The most versions a workspace's index lists.
34const MAX_VERSIONS: u32 = 20_000;
35/// The most gems a workspace's index lists.
36const MAX_GEMS: u32 = 2000;
37const DOCS: &str = "https://docs.g1t.sh/guides/rubygems/";
38const TOKENS: &str = "https://g1t.sh/settings/tokens";
39
40/// A plain-text answer, which `gem` and Bundler print.
41fn error(status: u16, message: impl Into<String>) -> Result<Response> {
42 let mut response = Response::ok(message.into())?.with_status(status);
43 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
44 if status == 401 {
45 response.headers_mut().set("www-authenticate", "Basic realm=\"g1t\"")?;
46 }
47 Ok(response)
48}
49
50fn sign_in(workspace: &str) -> String {
51 format!(
52 "Sign in to use this registry: bundle config set --global https://g1t.sh/-/rubygems/{workspace}/ <you>:<token>, with a g1t access token from {TOKENS}. See {DOCS}"
53 )
54}
55
56/// An index file, with the quoted MD5 of its body as its `ETag`: Bundler
57/// checks the file it keeps against it, and asks with `If-None-Match`.
58fn index_file(request: &Request, body: String, head: bool) -> Result<Response> {
59 let etag = format!("\"{:x}\"", md5::compute(body.as_bytes()));
60 let headers = Headers::new();
61 headers.set("content-type", "text/plain; charset=utf-8")?;
62 headers.set("cache-control", "no-cache")?;
63 headers.set("etag", &etag)?;
64 if request.headers().get("if-none-match")?.is_some_and(|sent| sent.trim_start_matches("W/") == etag) {
65 return Ok(Response::empty()?.with_status(304).with_headers(headers));
66 }
67 headers.set("content-length", &body.len().to_string())?;
68 let body = if head { ResponseBody::Empty } else { ResponseBody::Body(body.into_bytes()) };
69 Ok(Response::from_body(body)?.with_headers(headers))
70}
71
72/// A full index file or a specification, which `gem` reads as bytes.
73fn binary(bytes: Vec<u8>, head: bool, cache: &str) -> Result<Response> {
74 let headers = Headers::new();
75 headers.set("content-type", "application/octet-stream")?;
76 headers.set("content-length", &bytes.len().to_string())?;
77 headers.set("cache-control", cache)?;
78 let body = if head { ResponseBody::Empty } else { ResponseBody::Body(bytes) };
79 Ok(Response::from_body(body)?.with_headers(headers))
80}
81
82/// A version's line in the index.
83fn line(row: &VersionRow) -> String {
84 let checksum = Digest::parse(&row.digest).map(|d| d.hex().to_owned()).unwrap_or_default();
85 rubygems::info_line(&row.version, &row.meta(), &checksum)
86}
87
88impl Packages {
89 /// Answers a RubyGems request.
90 pub async fn rubygems(&self, request: Request, ctx: &Context) -> Result<Response> {
91 let url = request.url()?;
92 let Some((workspace, route)) = rubygems::route(url.path()) else {
93 return error(404, "There is nothing at this address.");
94 };
95 match self.rubygems_route(request, &url, &workspace, route, ctx).await {
96 Ok(response) => Ok(response),
97 Err(problem) => {
98 worker::console_error!("packages: rubygems {}: {problem}", url.path());
99 error(500, "Something went wrong on our side. Try again in a moment.")
100 }
101 }
102 }
103
104 /// Who the request is from: `gem`'s bare key (or a `Bearer` one), or
105 /// Bundler's Basic credentials.
106 async fn gem_credentials(&self, request: &Request) -> Result<Credentials> {
107 let Some(header) = request.headers().get("authorization")? else {
108 return Ok(Credentials::None);
109 };
110 let viewer = if let Some((username, secret)) = token::basic(&header) {
111 self.viewer_for(&username, &secret).await?
112 } else if let Some(key) = cargo::token(&header) {
113 self.viewer_for("token", key).await?
114 } else {
115 None
116 };
117 Ok(match viewer {
118 Some(user) => Credentials::Viewer(Some(user)),
119 None => Credentials::Bad,
120 })
121 }
122
123 async fn rubygems_route(&self, mut request: Request, url: &Url, workspace: &str, route: GemRoute, ctx: &Context) -> Result<Response> {
124 let method = request.method();
125 let credentials = self.gem_credentials(&request).await?;
126 let read = matches!(method, Method::Get | Method::Head);
127 if read && let Some(refused) = self.limited(&request, &credentials, "Basic credentials in bundle config").await? {
128 return Ok(refused);
129 }
130 let viewer = match credentials {
131 Credentials::Viewer(viewer) => viewer,
132 Credentials::None => None,
133 Credentials::Token(_) | Credentials::Bad => {
134 return error(401, format!("The token is not right, or has expired. Make an access token at {TOKENS}."));
135 }
136 };
137 let viewer = viewer.as_ref();
138 let head = method == Method::Head;
139 match route {
140 GemRoute::Versions if read => self.gem_versions(&request, workspace, viewer, head).await,
141 GemRoute::Names if read => self.gem_names(&request, workspace, viewer, head).await,
142 GemRoute::Info { name } if read => self.gem_info(&request, workspace, &name, viewer, head).await,
143 GemRoute::Gem { stem } if read => self.gem_download(workspace, &stem, viewer, head, ctx).await,
144 GemRoute::Specs(which) if read => self.gem_specs(workspace, which, viewer, head).await,
145 GemRoute::QuickSpec { stem } if read => self.gem_quick_spec(workspace, &stem, viewer, head).await,
146 GemRoute::Push if method == Method::Post => self.gem_push(&mut request, workspace, viewer).await,
147 GemRoute::Yank if method == Method::Delete => self.gem_yank(&mut request, url, workspace, viewer).await,
148 _ => error(405, "Not a method this address takes."),
149 }
150 }
151
152 /// The answer for something not there: a `401` to someone not signed
153 /// in when the workspace has private gems, so Bundler asks for
154 /// credentials and a private gem looks like a missing one.
155 async fn gem_absent(&self, workspace: &str, viewer: Option<&User>) -> Result<Response> {
156 if viewer.is_none() && self.db.has_private(workspace, RUBYGEMS).await? {
157 return error(401, sign_in(workspace));
158 }
159 error(404, "Not found: no such gem or version, or you cannot see it.")
160 }
161
162 async fn gem_check(&self, viewer: Option<&User>, package: &PackageRow, action: Action) -> Result<Option<Response>> {
163 let target = TargetOf::package(package);
164 let decision = access::decide(viewer, &target.view(), action);
165 if decision.allowed {
166 return Ok(None);
167 }
168 let readable = action != Action::Pull && access::decide(viewer, &target.view(), Action::Pull).allowed;
169 if !readable && viewer.is_none() {
170 return Ok(Some(error(401, sign_in(&package.workspace))?));
171 }
172 if !readable {
173 return Ok(Some(self.gem_absent(&package.workspace, viewer).await?));
174 }
175 Ok(Some(error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()))?))
176 }
177
178 /// The workspace's gems the viewer may see, with their versions in the
179 /// index (not yanked), oldest first.
180 async fn gem_index(&self, workspace: &str, viewer: Option<&User>) -> Result<std::result::Result<Vec<(PackageRow, Vec<VersionRow>)>, Response>> {
181 if viewer.is_none() && self.db.has_private(workspace, RUBYGEMS).await? {
182 return Ok(Err(error(401, sign_in(workspace))?));
183 }
184 let packages = self.db.packages_of(workspace, RUBYGEMS, MAX_GEMS).await?;
185 let versions = self.db.ecosystem_versions(workspace, RUBYGEMS, MAX_VERSIONS).await?;
186 Ok(Ok(packages
187 .into_iter()
188 .filter(|p| access::decide(viewer, &TargetOf::package(p).view(), Action::Pull).allowed)
189 .map(|p| {
190 let rows: Vec<VersionRow> = versions.iter().filter(|v| v.package_id == p.id && !v.is_yanked()).cloned().collect();
191 (p, rows)
192 })
193 .filter(|(_, rows)| !rows.is_empty())
194 .collect()))
195 }
196
197 async fn gem_versions(&self, request: &Request, workspace: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
198 let gems = match self.gem_index(workspace, viewer).await? {
199 Ok(gems) => gems,
200 Err(refused) => return Ok(refused),
201 };
202 let created = gems
203 .iter()
204 .flat_map(|(_, rows)| rows.iter().map(|r| r.published_at.as_str()))
205 .min()
206 .map(|at| format!("{}Z", &at[..at.len().min(19)]))
207 .unwrap_or_else(|| "2026-01-01T00:00:00Z".to_owned());
208 let lines: Vec<(String, Vec<String>, String)> = gems
209 .iter()
210 .map(|(package, rows)| {
211 let info = rubygems::info(&rows.iter().map(line).collect::<Vec<_>>());
212 (package.name.clone(), rows.iter().map(|r| r.version.clone()).collect(), format!("{:x}", md5::compute(info.as_bytes())))
213 })
214 .collect();
215 index_file(request, rubygems::versions_file(&created, &lines), head)
216 }
217
218 async fn gem_names(&self, request: &Request, workspace: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
219 let gems = match self.gem_index(workspace, viewer).await? {
220 Ok(gems) => gems,
221 Err(refused) => return Ok(refused),
222 };
223 let names: Vec<String> = gems.into_iter().map(|(p, _)| p.name).collect();
224 index_file(request, rubygems::names_file(&names), head)
225 }
226
227 async fn gem_info(&self, request: &Request, workspace: &str, name: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
228 let Some(package) = self.db.package(workspace, RUBYGEMS, name).await?.filter(|p| !p.hidden()) else {
229 return self.gem_absent(workspace, viewer).await;
230 };
231 if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
232 return Ok(refusal);
233 }
234 let mut rows: Vec<VersionRow> = self.db.versions(&package.id, MAX_VERSIONS).await?.into_iter().filter(|v| !v.is_yanked()).collect();
235 if rows.is_empty() {
236 return self.gem_absent(workspace, viewer).await;
237 }
238 rows.reverse();
239 index_file(request, rubygems::info(&rows.iter().map(line).collect::<Vec<_>>()), head)
240 }
241
242 /// A full index file: the versions in the index of every gem the
243 /// viewer may see, as `[name, Gem::Version, platform]`.
244 async fn gem_specs(&self, workspace: &str, which: rubygems::Specs, viewer: Option<&User>, head: bool) -> Result<Response> {
245 let gems = match self.gem_index(workspace, viewer).await? {
246 Ok(gems) => gems,
247 Err(refused) => return Ok(refused),
248 };
249 let tuples: Vec<rubygems::Tuple> =
250 gems.iter().flat_map(|(package, rows)| rows.iter().map(|row| rubygems::Tuple::of(&package.name, &row.version, &row.meta()))).collect();
251 binary(rubygems::specs_file(which, &tuples), head, "no-cache")
252 }
253
254 /// A version's specification, marshalled and deflated, which `gem
255 /// install` reads before the gem. Yanked versions' too, as their files.
256 async fn gem_quick_spec(&self, workspace: &str, stem: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
257 for (name, key) in rubygems::candidates(stem).into_iter().rev() {
258 let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(|p| !p.hidden()) else {
259 continue;
260 };
261 if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
262 return Ok(refusal);
263 }
264 let Some(row) = self.db.version_named(&package.id, &key).await? else {
265 continue;
266 };
267 let spec = rubygems::quick_spec(&package.name, &row.version, &row.meta(), &row.published_at);
268 return binary(spec, head, "max-age=300");
269 }
270 self.gem_absent(workspace, viewer).await
271 }
272
273 /// A `.gem`, yanked ones too: a lockfile may still name them.
274 async fn gem_download(&self, workspace: &str, stem: &str, viewer: Option<&User>, head: bool, ctx: &Context) -> Result<Response> {
275 for (name, key) in rubygems::candidates(stem).into_iter().rev() {
276 let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(|p| !p.hidden()) else {
277 continue;
278 };
279 if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
280 return Ok(refusal);
281 }
282 let Some(row) = self.db.version_named(&package.id, &key).await? else {
283 continue;
284 };
285 let Some(digest) = Digest::parse(&row.digest) else { continue };
286 let Some(blob) = self.db.package_blob(&package.id, &digest).await? else { continue };
287 let headers = Headers::new();
288 headers.set("content-type", "application/octet-stream")?;
289 headers.set("content-length", &blob.size.to_string())?;
290 headers.set("cache-control", "max-age=31536000")?;
291 if head {
292 return Ok(Response::from_body(ResponseBody::Empty)?.with_headers(headers));
293 }
294 let Some(got) = self.store.get(&blob.object_key, None).await? else { continue };
295 self.count_download(&package.id, ctx);
296 return Ok(Response::from_body(got.body)?.with_headers(headers));
297 }
298 self.gem_absent(workspace, viewer).await
299 }
300
301 /// The package a first push makes, linked to the repository the gem's
302 /// `source_code_uri` or homepage names on g1t, or else one named like it.
303 async fn gem_target(&self, workspace: &str, spec: &rubygems::Gemspec) -> Result<TargetOf> {
304 let named: Vec<String> = [&spec.source_code_uri, &spec.homepage]
305 .into_iter()
306 .flatten()
307 .filter_map(|url| npm::repository_of(&Value::String(url.clone()), &self.host))
308 .filter(|(owner, _)| owner == workspace)
309 .map(|(_, repo)| repo)
310 .collect();
311 let lower = spec.name.to_ascii_lowercase();
312 let dashed = lower.replace('_', "-");
313 let mut repo = None;
314 for candidate in named.iter().map(String::as_str).chain([lower.as_str(), dashed.as_str()]) {
315 if let Some(found) = self.repo_by_name(workspace, candidate).await? {
316 repo = Some(found);
317 break;
318 }
319 }
320 Ok(TargetOf { workspace: workspace.to_owned(), repo: repo.map(|r| (r.id, r.name, r.is_private)), public: false })
321 }
322
323 /// `gem push`: the `.gem` as the body.
324 async fn gem_push(&self, request: &mut Request, workspace: &str, viewer: Option<&User>) -> Result<Response> {
325 let declared = request.headers().get("content-length")?.and_then(|n| n.parse::<u64>().ok());
326 let too_large = || {
327 let mb = self.max_request / 1_000_000;
328 error(413, format!("A gem may be at most {mb} MB. See {DOCS}#size"))
329 };
330 if declared.is_some_and(|n| n > self.max_request) {
331 return too_large();
332 }
333 let gem = request.bytes().await?;
334 if gem.len() as u64 > self.max_request {
335 return too_large();
336 }
337 if viewer.is_none() {
338 return error(401, format!("Push with a g1t access token: GEM_HOST_API_KEY=<token> gem push <file> --host https://g1t.sh/-/rubygems/{workspace}. Make one at {TOKENS}."));
339 }
340 let spec = match rubygems::read_gem(&gem) {
341 Ok(spec) => spec,
342 Err(message) => return error(422, message),
343 };
344 if !rubygems::valid_name(&spec.name) {
345 return error(422, format!("{} is not a valid gem name: letters, digits, ., - and _, with a letter.", spec.name));
346 }
347 if !rubygems::valid_version(&spec.version) {
348 return error(422, format!("{} is not a version RubyGems reads.", spec.version));
349 }
350 let key = rubygems::key(&spec.version, &spec.platform);
351
352 // A name is taken whatever its case.
353 let found = self.db.package_any_case(workspace, RUBYGEMS, &spec.name).await?;
354 if let Some(found) = &found {
355 if found.hidden() {
356 return error(403, format!("The workspace {workspace} is deleted; nothing can be pushed to it."));
357 }
358 if found.name != spec.name {
359 return error(409, format!("The name {} is taken by the gem {}. Push it under that name.", spec.name, found.name));
360 }
361 } else if self.db.workspace_hidden(workspace).await? {
362 return error(403, format!("The workspace {workspace} is deleted; nothing can be pushed to it."));
363 }
364 let target = match &found {
365 Some(package) => TargetOf::package(package),
366 None => self.gem_target(workspace, &spec).await?,
367 };
368 let decision = access::decide(viewer, &target.view(), Action::Push);
369 if !decision.allowed {
370 let readable = found.is_none() || access::decide(viewer, &target.view(), Action::Pull).allowed;
371 if !readable {
372 return error(404, "Not found: no such gem, or you cannot see it.");
373 }
374 return error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()));
375 }
376 let caller = Caller { actor: viewer.map(AuditActor::of) };
377 let package = match found {
378 Some(package) => package,
379 None => {
380 self.db
381 .create_package(
382 &new_id("pkg", now_ms()),
383 workspace,
384 RUBYGEMS,
385 &spec.name,
386 target.repo.as_ref().map(|(id, repo, private)| (id.as_str(), repo.as_str(), *private)),
387 caller.actor.as_ref().map_or("", |actor| actor.actor_id.as_str()),
388 now_ms(),
389 )
390 .await?
391 }
392 };
393 let existing = self.db.versions(&package.id, MAX_VERSIONS).await?;
394 if existing.iter().any(|v| v.version == key) {
395 return error(409, format!("{} ({key}) is already pushed, and a version is pushed once, yanked or not. Bump the version.", package.name));
396 }
397
398 let digest = Digest::of(&gem);
399 let size = gem.len() as u64;
400 if let Some(refusal) = self.storage_refusal(&package, &[(digest.to_string(), size)]).await? {
401 return error(403, refusal);
402 }
403 let now = now_ms();
404 let stored = match self.db.blob(&digest).await? {
405 Some(blob) => self.store.head(&blob.object_key).await?.is_some(),
406 None => false,
407 };
408 if !stored {
409 self.store.put(&digest.object_key(), gem).await?;
410 }
411 self.db.keep_blob(&package.id, &digest, size, Some("application/octet-stream"), &digest.object_key(), now).await?;
412 self.db
413 .publish(
414 NewVersion {
415 id: new_id("ver", now),
416 package_id: package.id.clone(),
417 version: key.clone(),
418 digest: digest.to_string(),
419 size,
420 metadata: rubygems::stored(&spec).to_string(),
421 subject: None,
422 published_by: published_by(&caller),
423 files: vec![NewFile { name: "gem".to_owned(), digest: digest.to_string(), size, media_type: Some("application/octet-stream".to_owned()) }],
424 },
425 None,
426 now,
427 )
428 .await?;
429 // The description the page shows: the highest stable version's.
430 let highest = !rubygems::is_prerelease(&spec.version)
431 && existing.iter().map(|v| v.meta()).filter_map(|m| m["number"].as_str().map(str::to_owned)).all(|other| {
432 rubygems::is_prerelease(&other) || crate::db::newest_version(&format!("{other}\n{}", spec.version)).as_deref() == Some(spec.version.as_str())
433 });
434 if highest || existing.is_empty() {
435 let description = spec.summary.as_deref().or(spec.description.as_deref());
436 self.db.set_readme(&package.id, None, description, now).await?;
437 }
438 self.db.measure(&package.workspace).await?;
439 let event = PackageEvent {
440 version: Some(key.clone()),
441 digest: Some(digest.to_string()),
442 size: Some(size),
443 ..self.event_of(&package)
444 };
445 self.announce("package.published", &package, event, &caller).await;
446 self.audit(&caller, "package.publish", &package, Some(&format!("{workspace}/{}@{key}", package.name)), None).await;
447 let mut response = Response::ok(format!("Successfully registered gem: {} ({key})", package.name))?;
448 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
449 Ok(response)
450 }
451
452 /// `gem yank`: the version leaves the index, its file stays.
453 async fn gem_yank(&self, request: &mut Request, url: &Url, workspace: &str, viewer: Option<&User>) -> Result<Response> {
454 let body = request.text().await.unwrap_or_default();
455 let query = url.query().unwrap_or("").to_owned();
456 let value = |key: &str| rubygems::form_value(&body, key).or_else(|| rubygems::form_value(&query, key));
457 let (Some(name), Some(version)) = (value("gem_name"), value("version")) else {
458 return error(400, "Name the gem and version: gem yank <gem> --version <version>.");
459 };
460 let key = rubygems::key(&version, value("platform").as_deref().unwrap_or("ruby"));
461 let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(|p| !p.hidden()) else {
462 return self.gem_absent(workspace, viewer).await;
463 };
464 if let Some(refusal) = self.gem_check(viewer, &package, Action::Push).await? {
465 return Ok(refusal);
466 }
467 let Some(row) = self.db.version_named(&package.id, &key).await? else {
468 return error(404, format!("{name} ({key}) is not there."));
469 };
470 if row.is_yanked() {
471 return error(422, format!("{name} ({key}) is already yanked."));
472 }
473 self.db.set_yanked(&row.id, true).await?;
474 self.db.touch_package(&package.id, now_ms()).await?;
475 let caller = Caller { actor: viewer.map(AuditActor::of) };
476 self.audit(&caller, "package.yank", &package, Some(&format!("{workspace}/{name}@{key}")), None).await;
477 let mut response = Response::ok(format!("Successfully deleted gem: {name} ({key})"))?;
478 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
479 Ok(response)
480 }
481}