Sign in

flagon-io/g1t

public

g1t/services/packages/src/rubygems_http.rs

435 lines21,739 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a6a121745e81f639f'1//! The RubyGems registry: `g1t.sh/-/rubygems/<workspace>/`, one for each
2//! workspace. `gem push --host` sends a g1t token as the whole
3//! `Authorization` header (`GEM_HOST_API_KEY`, or `~/.gem/credentials`);
4//! Bundler sends Basic credentials (any username, a g1t token as the
5//! password) from `bundle config`.
6//!
7//! Bundler installs from the compact index (`versions`, `info/<gem>`,
8//! `names`), made from the versions on each read, with each file's MD5 as
9//! its `ETag` as Bundler checks it. A `.gem` is stored once, by its
10//! SHA-256, which is also its index `checksum`. `gem yank` takes a version
11//! out of the index; its file stays for lockfiles that name it.
12
13use g1t_contracts::User;
14use g1t_contracts::audit::AuditActor;
15use g1t_contracts::events::PackageEvent;
16use g1t_contracts::new_id;
17use g1t_kit::now_ms;
18use serde_json::Value;
19use worker::{Context, Headers, Method, Request, Response, ResponseBody, Result, Url};
20
21use crate::access::{self, Action};
22use crate::db::{NewFile, NewVersion, PackageRow, VersionRow};
23use crate::digest::Digest;
24use crate::oci::{Credentials, published_by};
25use crate::rubygems::{self, GemRoute};
26use crate::store::BlobStore;
27use crate::{Caller, Packages, TargetOf, cargo, npm, token};
28
29const RUBYGEMS: &str = "rubygems";
30/// The most versions a workspace's index lists.
31const MAX_VERSIONS: u32 = 20_000;
32/// The most gems a workspace's index lists.
33const MAX_GEMS: u32 = 2000;
34const DOCS: &str = "https://docs.g1t.sh/guides/rubygems/";
35const TOKENS: &str = "https://g1t.sh/settings/tokens";
36
37/// A plain-text answer, which `gem` and Bundler print.
38fn error(status: u16, message: impl Into<String>) -> Result<Response> {
39 let mut response = Response::ok(message.into())?.with_status(status);
40 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
41 if status == 401 {
42 response.headers_mut().set("www-authenticate", "Basic realm=\"g1t\"")?;
43 }
44 Ok(response)
45}
46
47fn sign_in(workspace: &str) -> String {
48 format!(
49 "Sign in to use this registry: bundle config set --global https://g1t.sh/-/rubygems/{workspace}/ <you>:<token>, with a g1t access token from {TOKENS}. See {DOCS}"
50 )
51}
52
53/// An index file, with the quoted MD5 of its body as its `ETag`: Bundler
54/// checks the file it keeps against it, and asks with `If-None-Match`.
55fn index_file(request: &Request, body: String, head: bool) -> Result<Response> {
56 let etag = format!("\"{:x}\"", md5::compute(body.as_bytes()));
57 let headers = Headers::new();
58 headers.set("content-type", "text/plain; charset=utf-8")?;
59 headers.set("cache-control", "no-cache")?;
60 headers.set("etag", &etag)?;
61 if request.headers().get("if-none-match")?.is_some_and(|sent| sent.trim_start_matches("W/") == etag) {
62 return Ok(Response::empty()?.with_status(304).with_headers(headers));
63 }
64 headers.set("content-length", &body.len().to_string())?;
65 let body = if head { ResponseBody::Empty } else { ResponseBody::Body(body.into_bytes()) };
66 Ok(Response::from_body(body)?.with_headers(headers))
67}
68
69/// A version's line in the index.
70fn line(row: &VersionRow) -> String {
71 let checksum = Digest::parse(&row.digest).map(|d| d.hex().to_owned()).unwrap_or_default();
72 rubygems::info_line(&row.version, &row.meta(), &checksum)
73}
74
75impl Packages {
76 /// Answers a RubyGems request.
77 pub async fn rubygems(&self, request: Request, ctx: &Context) -> Result<Response> {
78 let url = request.url()?;
79 let Some((workspace, route)) = rubygems::route(url.path()) else {
80 return error(404, "There is nothing at this address.");
81 };
82 match self.rubygems_route(request, &url, &workspace, route, ctx).await {
83 Ok(response) => Ok(response),
84 Err(problem) => {
85 worker::console_error!("packages: rubygems {}: {problem}", url.path());
86 error(500, "Something went wrong on our side. Try again in a moment.")
87 }
88 }
89 }
90
91 /// Who the request is from: `gem`'s bare key (or a `Bearer` one), or
92 /// Bundler's Basic credentials.
93 async fn gem_credentials(&self, request: &Request) -> Result<Credentials> {
94 let Some(header) = request.headers().get("authorization")? else {
95 return Ok(Credentials::None);
96 };
97 let viewer = if let Some((username, secret)) = token::basic(&header) {
98 self.viewer_for(&username, &secret).await?
99 } else if let Some(key) = cargo::token(&header) {
100 self.viewer_for("token", key).await?
101 } else {
102 None
103 };
104 Ok(match viewer {
105 Some(user) => Credentials::Viewer(Some(user)),
106 None => Credentials::Bad,
107 })
108 }
109
110 async fn rubygems_route(&self, mut request: Request, url: &Url, workspace: &str, route: GemRoute, ctx: &Context) -> Result<Response> {
111 let method = request.method();
112 let credentials = self.gem_credentials(&request).await?;
113 let read = matches!(method, Method::Get | Method::Head);
114 if read && let Some(refused) = self.limited(&request, &credentials, "Basic credentials in bundle config").await? {
115 return Ok(refused);
116 }
117 let viewer = match credentials {
118 Credentials::Viewer(viewer) => viewer,
119 Credentials::None => None,
120 Credentials::Token(_) | Credentials::Bad => {
121 return error(401, format!("The token is not right, or has expired. Make an access token at {TOKENS}."));
122 }
123 };
124 let viewer = viewer.as_ref();
125 let head = method == Method::Head;
126 match route {
127 GemRoute::Versions if read => self.gem_versions(&request, workspace, viewer, head).await,
128 GemRoute::Names if read => self.gem_names(&request, workspace, viewer, head).await,
129 GemRoute::Info { name } if read => self.gem_info(&request, workspace, &name, viewer, head).await,
130 GemRoute::Gem { stem } if read => self.gem_download(workspace, &stem, viewer, head, ctx).await,
131 GemRoute::Push if method == Method::Post => self.gem_push(&mut request, workspace, viewer).await,
132 GemRoute::Yank if method == Method::Delete => self.gem_yank(&mut request, url, workspace, viewer).await,
133 _ => error(405, "Not a method this address takes."),
134 }
135 }
136
137 /// The answer for something not there: a `401` to someone not signed
138 /// in when the workspace has private gems, so Bundler asks for
139 /// credentials and a private gem looks like a missing one.
140 async fn gem_absent(&self, workspace: &str, viewer: Option<&User>) -> Result<Response> {
141 if viewer.is_none() && self.db.has_private(workspace, RUBYGEMS).await? {
142 return error(401, sign_in(workspace));
143 }
144 error(404, "Not found: no such gem or version, or you cannot see it.")
145 }
146
147 async fn gem_check(&self, viewer: Option<&User>, package: &PackageRow, action: Action) -> Result<Option<Response>> {
148 let target = TargetOf::package(package);
149 let decision = access::decide(viewer, &target.view(), action);
150 if decision.allowed {
151 return Ok(None);
152 }
153 let readable = action != Action::Pull && access::decide(viewer, &target.view(), Action::Pull).allowed;
154 if !readable && viewer.is_none() {
155 return Ok(Some(error(401, sign_in(&package.workspace))?));
156 }
157 if !readable {
158 return Ok(Some(self.gem_absent(&package.workspace, viewer).await?));
159 }
160 Ok(Some(error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()))?))
161 }
162
163 /// The workspace's gems the viewer may see, with their versions in the
164 /// index (not yanked), oldest first.
165 async fn gem_index(&self, workspace: &str, viewer: Option<&User>) -> Result<std::result::Result<Vec<(PackageRow, Vec<VersionRow>)>, Response>> {
166 if viewer.is_none() && self.db.has_private(workspace, RUBYGEMS).await? {
167 return Ok(Err(error(401, sign_in(workspace))?));
168 }
169 let packages = self.db.packages_of(workspace, RUBYGEMS, MAX_GEMS).await?;
170 let versions = self.db.ecosystem_versions(workspace, RUBYGEMS, MAX_VERSIONS).await?;
171 Ok(Ok(packages
172 .into_iter()
173 .filter(|p| access::decide(viewer, &TargetOf::package(p).view(), Action::Pull).allowed)
174 .map(|p| {
175 let rows: Vec<VersionRow> = versions.iter().filter(|v| v.package_id == p.id && !v.is_yanked()).cloned().collect();
176 (p, rows)
177 })
178 .filter(|(_, rows)| !rows.is_empty())
179 .collect()))
180 }
181
182 async fn gem_versions(&self, request: &Request, workspace: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
183 let gems = match self.gem_index(workspace, viewer).await? {
184 Ok(gems) => gems,
185 Err(refused) => return Ok(refused),
186 };
187 let created = gems
188 .iter()
189 .flat_map(|(_, rows)| rows.iter().map(|r| r.published_at.as_str()))
190 .min()
191 .map(|at| format!("{}Z", &at[..at.len().min(19)]))
192 .unwrap_or_else(|| "2026-01-01T00:00:00Z".to_owned());
193 let lines: Vec<(String, Vec<String>, String)> = gems
194 .iter()
195 .map(|(package, rows)| {
196 let info = rubygems::info(&rows.iter().map(line).collect::<Vec<_>>());
197 (package.name.clone(), rows.iter().map(|r| r.version.clone()).collect(), format!("{:x}", md5::compute(info.as_bytes())))
198 })
199 .collect();
200 index_file(request, rubygems::versions_file(&created, &lines), head)
201 }
202
203 async fn gem_names(&self, request: &Request, workspace: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
204 let gems = match self.gem_index(workspace, viewer).await? {
205 Ok(gems) => gems,
206 Err(refused) => return Ok(refused),
207 };
208 let names: Vec<String> = gems.into_iter().map(|(p, _)| p.name).collect();
209 index_file(request, rubygems::names_file(&names), head)
210 }
211
212 async fn gem_info(&self, request: &Request, workspace: &str, name: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
213 let Some(package) = self.db.package(workspace, RUBYGEMS, name).await?.filter(|p| !p.hidden()) else {
214 return self.gem_absent(workspace, viewer).await;
215 };
216 if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
217 return Ok(refusal);
218 }
219 let mut rows: Vec<VersionRow> = self.db.versions(&package.id, MAX_VERSIONS).await?.into_iter().filter(|v| !v.is_yanked()).collect();
220 if rows.is_empty() {
221 return self.gem_absent(workspace, viewer).await;
222 }
223 rows.reverse();
224 index_file(request, rubygems::info(&rows.iter().map(line).collect::<Vec<_>>()), head)
225 }
226
227 /// A `.gem`, yanked ones too: a lockfile may still name them.
228 async fn gem_download(&self, workspace: &str, stem: &str, viewer: Option<&User>, head: bool, ctx: &Context) -> Result<Response> {
229 for (name, key) in rubygems::candidates(stem).into_iter().rev() {
230 let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(|p| !p.hidden()) else {
231 continue;
232 };
233 if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
234 return Ok(refusal);
235 }
236 let Some(row) = self.db.version_named(&package.id, &key).await? else {
237 continue;
238 };
239 let Some(digest) = Digest::parse(&row.digest) else { continue };
240 let Some(blob) = self.db.package_blob(&package.id, &digest).await? else { continue };
241 let headers = Headers::new();
242 headers.set("content-type", "application/octet-stream")?;
243 headers.set("content-length", &blob.size.to_string())?;
244 headers.set("cache-control", "max-age=31536000")?;
245 if head {
246 return Ok(Response::from_body(ResponseBody::Empty)?.with_headers(headers));
247 }
248 let Some(got) = self.store.get(&blob.object_key, None).await? else { continue };
249 self.count_download(&package.id, ctx);
250 return Ok(Response::from_body(got.body)?.with_headers(headers));
251 }
252 self.gem_absent(workspace, viewer).await
253 }
254
255 /// The package a first push makes, linked to the repository the gem's
256 /// `source_code_uri` or homepage names on g1t, or else one named like it.
257 async fn gem_target(&self, workspace: &str, spec: &rubygems::Gemspec) -> Result<TargetOf> {
258 let named: Vec<String> = [&spec.source_code_uri, &spec.homepage]
259 .into_iter()
260 .flatten()
261 .filter_map(|url| npm::repository_of(&Value::String(url.clone()), &self.host))
262 .filter(|(owner, _)| owner == workspace)
263 .map(|(_, repo)| repo)
264 .collect();
265 let lower = spec.name.to_ascii_lowercase();
266 let dashed = lower.replace('_', "-");
267 let mut repo = None;
268 for candidate in named.iter().map(String::as_str).chain([lower.as_str(), dashed.as_str()]) {
269 if let Some(found) = self.repo_by_name(workspace, candidate).await? {
270 repo = Some(found);
271 break;
272 }
273 }
274 Ok(TargetOf { workspace: workspace.to_owned(), repo: repo.map(|r| (r.id, r.name, r.is_private)), public: false })
275 }
276
277 /// `gem push`: the `.gem` as the body.
278 async fn gem_push(&self, request: &mut Request, workspace: &str, viewer: Option<&User>) -> Result<Response> {
279 let declared = request.headers().get("content-length")?.and_then(|n| n.parse::<u64>().ok());
280 let too_large = || {
281 let mb = self.max_request / 1_000_000;
282 error(413, format!("A gem may be at most {mb} MB. See {DOCS}#size"))
283 };
284 if declared.is_some_and(|n| n > self.max_request) {
285 return too_large();
286 }
287 let gem = request.bytes().await?;
288 if gem.len() as u64 > self.max_request {
289 return too_large();
290 }
291 if viewer.is_none() {
292 return error(401, format!("Push with a g1t access token: GEM_HOST_API_KEY=<token> gem push <file> --host https://g1t.sh/-/rubygems/{workspace}. Make one at {TOKENS}."));
293 }
294 let spec = match rubygems::read_gem(&gem) {
295 Ok(spec) => spec,
296 Err(message) => return error(422, message),
297 };
298 if !rubygems::valid_name(&spec.name) {
299 return error(422, format!("{} is not a valid gem name: letters, digits, ., - and _, with a letter.", spec.name));
300 }
301 if !rubygems::valid_version(&spec.version) {
302 return error(422, format!("{} is not a version RubyGems reads.", spec.version));
303 }
304 let key = rubygems::key(&spec.version, &spec.platform);
305
306 // A name is taken whatever its case.
307 let found = self.db.package_any_case(workspace, RUBYGEMS, &spec.name).await?;
308 if let Some(found) = &found {
309 if found.hidden() {
310 return error(403, format!("The workspace {workspace} is deleted; nothing can be pushed to it."));
311 }
312 if found.name != spec.name {
313 return error(409, format!("The name {} is taken by the gem {}. Push it under that name.", spec.name, found.name));
314 }
315 } else if self.db.workspace_hidden(workspace).await? {
316 return error(403, format!("The workspace {workspace} is deleted; nothing can be pushed to it."));
317 }
318 let target = match &found {
319 Some(package) => TargetOf::package(package),
320 None => self.gem_target(workspace, &spec).await?,
321 };
322 let decision = access::decide(viewer, &target.view(), Action::Push);
323 if !decision.allowed {
324 let readable = found.is_none() || access::decide(viewer, &target.view(), Action::Pull).allowed;
325 if !readable {
326 return error(404, "Not found: no such gem, or you cannot see it.");
327 }
328 return error(403, decision.reason.unwrap_or_else(|| "Not allowed.".to_owned()));
329 }
330 let caller = Caller { actor: viewer.map(AuditActor::of) };
331 let package = match found {
332 Some(package) => package,
333 None => {
334 self.db
335 .create_package(
336 &new_id("pkg", now_ms()),
337 workspace,
338 RUBYGEMS,
339 &spec.name,
340 target.repo.as_ref().map(|(id, repo, private)| (id.as_str(), repo.as_str(), *private)),
341 caller.actor.as_ref().map_or("", |actor| actor.actor_id.as_str()),
342 now_ms(),
343 )
344 .await?
345 }
346 };
347 let existing = self.db.versions(&package.id, MAX_VERSIONS).await?;
348 if existing.iter().any(|v| v.version == key) {
349 return error(409, format!("{} ({key}) is already pushed, and a version is pushed once, yanked or not. Bump the version.", package.name));
350 }
351
352 let digest = Digest::of(&gem);
353 let size = gem.len() as u64;
354 if let Some(refusal) = self.storage_refusal(&package, &[(digest.to_string(), size)]).await? {
355 return error(403, refusal);
356 }
357 let now = now_ms();
358 let stored = match self.db.blob(&digest).await? {
359 Some(blob) => self.store.head(&blob.object_key).await?.is_some(),
360 None => false,
361 };
362 if !stored {
363 self.store.put(&digest.object_key(), gem).await?;
364 }
365 self.db.keep_blob(&package.id, &digest, size, Some("application/octet-stream"), &digest.object_key(), now).await?;
366 self.db
367 .publish(
368 NewVersion {
369 id: new_id("ver", now),
370 package_id: package.id.clone(),
371 version: key.clone(),
372 digest: digest.to_string(),
373 size,
374 metadata: rubygems::stored(&spec).to_string(),
375 subject: None,
376 published_by: published_by(&caller),
377 files: vec![NewFile { name: "gem".to_owned(), digest: digest.to_string(), size, media_type: Some("application/octet-stream".to_owned()) }],
378 },
379 None,
380 now,
381 )
382 .await?;
383 // The description the page shows: the highest stable version's.
384 let highest = !rubygems::is_prerelease(&spec.version)
385 && existing.iter().map(|v| v.meta()).filter_map(|m| m["number"].as_str().map(str::to_owned)).all(|other| {
386 rubygems::is_prerelease(&other) || crate::db::newest_version(&format!("{other}\n{}", spec.version)).as_deref() == Some(spec.version.as_str())
387 });
388 if highest || existing.is_empty() {
389 let description = spec.summary.as_deref().or(spec.description.as_deref());
390 self.db.set_readme(&package.id, None, description, now).await?;
391 }
392 self.db.measure(&package.workspace).await?;
393 let event = PackageEvent {
394 version: Some(key.clone()),
395 digest: Some(digest.to_string()),
396 size: Some(size),
397 ..self.event_of(&package)
398 };
399 self.announce("package.published", &package, event, &caller).await;
400 self.audit(&caller, "package.publish", &package, Some(&format!("{workspace}/{}@{key}", package.name)), None).await;
401 let mut response = Response::ok(format!("Successfully registered gem: {} ({key})", package.name))?;
402 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
403 Ok(response)
404 }
405
406 /// `gem yank`: the version leaves the index, its file stays.
407 async fn gem_yank(&self, request: &mut Request, url: &Url, workspace: &str, viewer: Option<&User>) -> Result<Response> {
408 let body = request.text().await.unwrap_or_default();
409 let query = url.query().unwrap_or("").to_owned();
410 let value = |key: &str| rubygems::form_value(&body, key).or_else(|| rubygems::form_value(&query, key));
411 let (Some(name), Some(version)) = (value("gem_name"), value("version")) else {
412 return error(400, "Name the gem and version: gem yank <gem> --version <version>.");
413 };
414 let key = rubygems::key(&version, value("platform").as_deref().unwrap_or("ruby"));
415 let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(|p| !p.hidden()) else {
416 return self.gem_absent(workspace, viewer).await;
417 };
418 if let Some(refusal) = self.gem_check(viewer, &package, Action::Push).await? {
419 return Ok(refusal);
420 }
421 let Some(row) = self.db.version_named(&package.id, &key).await? else {
422 return error(404, format!("{name} ({key}) is not there."));
423 };
424 if row.is_yanked() {
425 return error(422, format!("{name} ({key}) is already yanked."));
426 }
427 self.db.set_yanked(&row.id, true).await?;
428 self.db.touch_package(&package.id, now_ms()).await?;
429 let caller = Caller { actor: viewer.map(AuditActor::of) };
430 self.audit(&caller, "package.yank", &package, Some(&format!("{workspace}/{name}@{key}")), None).await;
431 let mut response = Response::ok(format!("Successfully deleted gem: {name} ({key})"))?;
432 response.headers_mut().set("content-type", "text/plain; charset=utf-8")?;
433 Ok(response)
434 }
435}

This file's history is long; its oldest lines are credited to the oldest commit read.