g1t/services/packages/src/rubygems.rs

439 lines17,835 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Packages: Maven, NuGet and RubyGems registries for every workspace1//! What the RubyGems registry needs that does not touch the network: gem
2//! names and versions, the registry's paths, the `Gem::Specification` read
3//! from a `.gem` (a tar holding `metadata.gz`), and the compact index
4//! Bundler reads: `versions`, `info/<gem>` and `names`.
5//!
6//! A version is keyed by its number and platform as the compact index
7//! writes it (`1.0.0`, `1.0.0-x86_64-linux`), and keeps what its index
8//! line needs as its metadata, made once when it is pushed.
9
10use serde_json::{Value, json};
11
12use crate::archive;
13use crate::yaml;
14
15/// The longest gem name taken.
16pub const MAX_NAME: usize = 128;
17/// The largest `metadata.gz`, unpacked, read from a gem.
18const MAX_METADATA_BYTES: usize = 4 * 1024 * 1024;
19
20/// A gem's name: letters, digits, `.`, `-` and `_`, with a letter in it.
21pub fn valid_name(name: &str) -> bool {
22 !name.is_empty()
23 && name.len() <= MAX_NAME
24 && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_'))
25 && name.bytes().any(|b| b.is_ascii_alphabetic())
26 && name.as_bytes()[0].is_ascii_alphanumeric()
27}
28
29/// A version as `Gem::Version` takes it: numbers and words joined by dots,
30/// starting with a number (`1.0.0`, `2.0.0.rc1`, `1.0.0-beta.1`).
31pub fn valid_version(version: &str) -> bool {
32 let (core, suffix) = version.split_once('-').map_or((version, None), |(c, s)| (c, Some(s)));
33 let mut parts = core.split('.');
34 let first_ok = parts.next().is_some_and(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()));
35 first_ok
36 && version.len() <= 128
37 && parts.all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric()))
38 && suffix.is_none_or(|s| s.split('.').all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')))
39}
40
41/// A version with a letter in it is a pre-release, as RubyGems decides.
42pub fn is_prerelease(version: &str) -> bool {
43 version.bytes().any(|b| b.is_ascii_alphabetic())
44}
45
46/// The version as the index keys it: `1.0.0`, or `1.0.0-java` for a gem
47/// built for a platform.
48pub fn key(version: &str, platform: &str) -> String {
49 if platform.is_empty() || platform == "ruby" { version.to_owned() } else { format!("{version}-{platform}") }
50}
51
52/// One of the registry's endpoints, under `/-/rubygems/<workspace>/`.
53#[derive(Clone, Debug, PartialEq, Eq)]
54pub enum GemRoute {
55 /// `versions`: every gem and its versions, for Bundler.
56 Versions,
57 /// `info/<gem>`: a gem's versions, dependencies and checksums.
58 Info { name: String },
59 /// `names`: every gem's name.
60 Names,
61 /// `gems/<name>-<version>[-<platform>].gem`; the name and version are
62 /// told apart by the handler, as names may hold `-`.
63 Gem { stem: String },
64 /// `api/v1/gems`: `gem push`.
65 Push,
66 /// `api/v1/gems/yank`: `gem yank`.
67 Yank,
68}
69
70pub fn route(path: &str) -> Option<(String, GemRoute)> {
71 let rest = path.strip_prefix("/-/rubygems/")?;
72 let (workspace, rest) = rest.split_once('/')?;
73 let workspace = workspace.to_ascii_lowercase();
74 if workspace.is_empty() {
75 return None;
76 }
77 let route = match rest.trim_end_matches('/') {
78 "versions" => GemRoute::Versions,
79 "names" => GemRoute::Names,
80 "api/v1/gems" => GemRoute::Push,
81 "api/v1/gems/yank" => GemRoute::Yank,
82 other => {
83 if let Some(name) = other.strip_prefix("info/") {
84 if !valid_name(name) {
85 return None;
86 }
87 GemRoute::Info { name: name.to_owned() }
88 } else {
89 let file = other.strip_prefix("gems/")?;
90 let stem = file.strip_suffix(".gem")?;
91 if stem.contains('/') || candidates(stem).is_empty() {
92 return None;
93 }
94 GemRoute::Gem { stem: stem.to_owned() }
95 }
96 }
97 };
98 Some((workspace, route))
99}
100
101/// The ways a file's stem splits into a name and a version key: at each
102/// `-` followed by a digit, longest name last (`a-b-1.0` is `a-b` `1.0`).
103pub fn candidates(stem: &str) -> Vec<(String, String)> {
104 stem.char_indices()
105 .filter(|&(i, c)| c == '-' && stem[i + 1..].starts_with(|n: char| n.is_ascii_digit()))
106 .map(|(i, _)| (stem[..i].to_owned(), stem[i + 1..].to_owned()))
107 .filter(|(name, _)| valid_name(name))
108 .collect()
109}
110
111/// What is read from a gem's specification.
112#[derive(Clone, Debug, Default, PartialEq, Eq)]
113pub struct Gemspec {
114 pub name: String,
115 pub version: String,
116 pub platform: String,
117 pub summary: Option<String>,
118 pub description: Option<String>,
119 pub homepage: Option<String>,
120 pub source_code_uri: Option<String>,
121 pub licenses: Vec<String>,
122 pub authors: Vec<String>,
123 /// Runtime dependencies: name and requirement (`>= 2.0&< 4`).
124 pub dependencies: Vec<(String, String)>,
125 pub ruby: Option<String>,
126 pub rubygems: Option<String>,
127}
128
129/// A `Gem::Requirement` as the compact index writes it: `>= 2.0&< 4`.
130pub fn requirement(value: &Value) -> String {
131 value["requirements"]
132 .as_array()
133 .map(|list| {
134 list.iter()
135 .filter_map(|pair| {
136 let op = pair.get(0)?.as_str()?;
137 let version = pair.get(1).map(|v| v.get("version").unwrap_or(v)).and_then(Value::as_str)?;
138 Some(format!("{op} {version}"))
139 })
140 .collect::<Vec<_>>()
141 .join("&")
142 })
143 .unwrap_or_default()
144}
145
146/// A requirement that says nothing (`>= 0`), as left out of the index.
147fn anything(requirement: &str) -> bool {
148 requirement.is_empty() || requirement == ">= 0"
149}
150
151fn text(value: &Value) -> Option<String> {
152 value.as_str().map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
153}
154
155fn texts(value: &Value) -> Vec<String> {
156 match value {
157 Value::Array(list) => list.iter().filter_map(text).collect(),
158 Value::String(_) => text(value).into_iter().collect(),
159 _ => Vec::new(),
160 }
161}
162
163/// Reads a specification RubyGems wrote as YAML.
164pub fn read_spec(text_yaml: &str) -> Result<Gemspec, String> {
165 let spec = yaml::parse(text_yaml).map_err(|problem| format!("The gem's metadata is not YAML: {problem}"))?;
166 if !spec.is_object() {
167 return Err("The gem's metadata is not a specification.".to_owned());
168 }
169 let version = &spec["version"];
170 let version = text(version.get("version").unwrap_or(version)).ok_or("The gem's metadata names no version.")?;
171 let platform = match &spec["platform"] {
172 Value::Object(map) => ["cpu", "os", "version"].iter().filter_map(|k| map.get(*k).and_then(text)).collect::<Vec<_>>().join("-"),
173 other => text(other).unwrap_or_else(|| "ruby".to_owned()),
174 };
175 let dependencies = spec["dependencies"]
176 .as_array()
177 .map(|deps| {
178 deps.iter()
179 .filter(|d| d["type"].as_str().is_none_or(|t| t.trim_start_matches(':') == "runtime"))
180 .filter_map(|d| {
181 let name = text(&d["name"])?;
182 let req = d.get("requirement").filter(|r| r.is_object()).or_else(|| d.get("version_requirements")).map(requirement).unwrap_or_default();
183 Some((name, if req.is_empty() { ">= 0".to_owned() } else { req }))
184 })
185 .collect()
186 })
187 .unwrap_or_default();
188 let required = |key: &str| Some(requirement(&spec[key])).filter(|r| !anything(r));
189 Ok(Gemspec {
190 name: text(&spec["name"]).ok_or("The gem's metadata names no gem.")?,
191 version,
192 platform: if platform.is_empty() { "ruby".to_owned() } else { platform },
193 summary: text(&spec["summary"]),
194 description: text(&spec["description"]),
195 homepage: text(&spec["homepage"]),
196 source_code_uri: text(&spec["metadata"]["source_code_uri"]),
197 licenses: texts(&spec["licenses"]),
198 authors: texts(&spec["authors"]),
199 dependencies,
200 ruby: required("required_ruby_version"),
201 rubygems: required("required_rubygems_version"),
202 })
203}
204
205/// Reads a `.gem`: the specification in its `metadata.gz`.
206pub fn read_gem(gem: &[u8]) -> Result<Gemspec, String> {
207 let files = archive::tar_files(gem).map_err(|_| "The file is not a .gem: it is not a tar archive.".to_owned())?;
208 let (_, metadata) = files.iter().find(|(name, _)| name == "metadata.gz").ok_or("The gem has no metadata.gz.")?;
209 let yaml = archive::gunzip(metadata, MAX_METADATA_BYTES)?;
210 read_spec(&String::from_utf8(yaml).map_err(|_| "The gem's metadata is not UTF-8.".to_owned())?)
211}
212
213/// What a version keeps for its index line and the gem's page.
214pub fn stored(spec: &Gemspec) -> Value {
215 json!({
216 "name": spec.name,
217 "number": spec.version,
218 "platform": spec.platform,
219 "summary": spec.summary,
220 "description": spec.description,
221 "homepage": spec.homepage,
222 "source_code_uri": spec.source_code_uri,
223 "licenses": spec.licenses,
224 "authors": spec.authors,
225 "dependencies": spec.dependencies.iter().map(|(name, req)| json!({ "name": name, "requirement": req })).collect::<Vec<_>>(),
226 "ruby": spec.ruby,
227 "rubygems": spec.rubygems,
228 })
229}
230
231/// One line of a gem's info file: `1.0.0 rack:>= 2.0&< 4|checksum:<sha256>,ruby:>= 3.0`.
232pub fn info_line(key: &str, stored: &Value, checksum: &str) -> String {
233 let deps: Vec<String> = stored["dependencies"]
234 .as_array()
235 .map(|deps| {
236 deps.iter()
237 .filter_map(|d| Some(format!("{}:{}", d["name"].as_str()?, d["requirement"].as_str().unwrap_or(">= 0"))))
238 .collect()
239 })
240 .unwrap_or_default();
241 let mut requirements = vec![format!("checksum:{checksum}")];
242 if let Some(ruby) = stored["ruby"].as_str() {
243 requirements.push(format!("ruby:{ruby}"));
244 }
245 if let Some(rubygems) = stored["rubygems"].as_str() {
246 requirements.push(format!("rubygems:{rubygems}"));
247 }
248 format!("{key} {}|{}", deps.join(","), requirements.join(","))
249}
250
251/// A gem's info file, from its versions' lines, oldest first.
252pub fn info(lines: &[String]) -> String {
253 let mut out = String::from("---\n");
254 for line in lines {
255 out.push_str(line);
256 out.push('\n');
257 }
258 out
259}
260
261/// The `versions` file: each gem's versions and its info file's MD5.
262pub fn versions_file(created_at: &str, gems: &[(String, Vec<String>, String)]) -> String {
263 let mut out = format!("created_at: {created_at}\n---\n");
264 for (name, versions, md5) in gems {
265 out.push_str(&format!("{name} {} {md5}\n", versions.join(",")));
266 }
267 out
268}
269
270pub fn names_file(names: &[String]) -> String {
271 let mut out = String::from("---\n");
272 for name in names {
273 out.push_str(name);
274 out.push('\n');
275 }
276 out
277}
278
279/// A value from a form body or query string (`gem_name=hello&version=1.0`).
280pub fn form_value(form: &str, key: &str) -> Option<String> {
281 let url = worker::Url::parse(&format!("http://form.invalid/?{form}")).ok()?;
282 url.query_pairs().find(|(k, _)| k == key).map(|(_, v)| v.into_owned()).filter(|v| !v.is_empty())
283}
284
285#[cfg(test)]
286mod tests {
287 use super::*;
288
289 #[test]
290 fn names_and_versions_follow_rubygems_rules() {
291 for good in ["rails", "hello-world", "net_http2", "a1", "Hello.rb"] {
292 assert!(valid_name(good), "{good}");
293 }
294 for bad in ["", "123", "-a", ".a", "a b", "a/b", &"a".repeat(129)] {
295 assert!(!valid_name(bad), "{bad}");
296 }
297 for good in ["1.0.0", "0.1", "2.0.0.rc1", "1.0.0-beta.1", "3"] {
298 assert!(valid_version(good), "{good}");
299 }
300 for bad in ["", "a.1", "1..0", "1.0 0", "1.0-"] {
301 assert!(!valid_version(bad), "{bad}");
302 }
303 assert!(is_prerelease("2.0.0.rc1") && !is_prerelease("2.0.0"));
304 assert_eq!(key("1.0.0", "ruby"), "1.0.0");
305 assert_eq!(key("1.0.0", "x86_64-linux"), "1.0.0-x86_64-linux");
306 }
307
308 #[test]
309 fn every_endpoint_is_routed() {
310 let at = |route: GemRoute| Some(("acme".to_owned(), route));
311 assert_eq!(route("/-/rubygems/Acme/versions"), at(GemRoute::Versions));
312 assert_eq!(route("/-/rubygems/acme/names"), at(GemRoute::Names));
313 assert_eq!(route("/-/rubygems/acme/info/hello-world"), at(GemRoute::Info { name: "hello-world".into() }));
314 assert_eq!(route("/-/rubygems/acme/gems/hello-world-0.1.0.gem"), at(GemRoute::Gem { stem: "hello-world-0.1.0".into() }));
315 assert_eq!(route("/-/rubygems/acme/api/v1/gems"), at(GemRoute::Push));
316 assert_eq!(route("/-/rubygems/acme/api/v1/gems/yank"), at(GemRoute::Yank));
317 assert_eq!(route("/-/rubygems/acme/gems/hello.gem"), None, "no version");
318 assert_eq!(route("/-/rubygems/acme/info/a b"), None);
319 assert_eq!(route("/-/rubygems/acme/other"), None);
320 assert_eq!(route("/-/rubygems/acme"), None);
321 assert_eq!(
322 candidates("hello-world-0.1.0-x86_64-linux"),
323 [("hello-world".to_owned(), "0.1.0-x86_64-linux".to_owned())],
324 "x86_64 starts with a letter"
325 );
326 assert_eq!(candidates("a-2-1.0"), [("a".to_owned(), "2-1.0".to_owned()), ("a-2".to_owned(), "1.0".to_owned())]);
327 }
328
329 const SPEC: &str = r#"--- !ruby/object:Gem::Specification
330name: hello-world
331version: !ruby/object:Gem::Version
332 version: 0.2.0
333platform: ruby
334authors:
335- Ada
336dependencies:
337- !ruby/object:Gem::Dependency
338 name: rack
339 requirement: !ruby/object:Gem::Requirement
340 requirements:
341 - - "<"
342 - !ruby/object:Gem::Version
343 version: '4'
344 - - ">="
345 - !ruby/object:Gem::Version
346 version: '2.0'
347 type: :runtime
348 prerelease: false
349 version_requirements: !ruby/object:Gem::Requirement
350 requirements:
351 - - "<"
352 - !ruby/object:Gem::Version
353 version: '4'
354- !ruby/object:Gem::Dependency
355 name: json
356 requirement: !ruby/object:Gem::Requirement
357 requirements:
358 - - ">="
359 - !ruby/object:Gem::Version
360 version: '0'
361 type: :runtime
362- !ruby/object:Gem::Dependency
363 name: rspec
364 requirement: !ruby/object:Gem::Requirement
365 requirements:
366 - - "~>"
367 - !ruby/object:Gem::Version
368 version: '3.0'
369 type: :development
370description: Says hello.
371homepage: https://g1t.sh/acme/hello-world
372licenses:
373- MIT
374metadata:
375 source_code_uri: https://g1t.sh/acme/hello-world
376required_ruby_version: !ruby/object:Gem::Requirement
377 requirements:
378 - - ">="
379 - !ruby/object:Gem::Version
380 version: 3.0.0
381required_rubygems_version: !ruby/object:Gem::Requirement
382 requirements:
383 - - ">="
384 - !ruby/object:Gem::Version
385 version: '0'
386summary: Says hello
387"#;
388
389 #[test]
390 fn a_gem_is_read_from_its_metadata() {
391 let mut gz = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3];
392 gz.extend_from_slice(&miniz_oxide::deflate::compress_to_vec(SPEC.as_bytes(), 6));
393 gz.extend_from_slice(&crate::composer::crc32(SPEC.as_bytes()).to_le_bytes());
394 gz.extend_from_slice(&(SPEC.len() as u32).to_le_bytes());
395 let mut gem = Vec::new();
396 for (name, data) in [("metadata.gz", gz.as_slice()), ("data.tar.gz", b"x".as_slice())] {
397 let mut header = [0u8; 512];
398 header[..name.len()].copy_from_slice(name.as_bytes());
399 header[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes());
400 header[156] = b'0';
401 gem.extend_from_slice(&header);
402 gem.extend_from_slice(data);
403 gem.resize(gem.len().div_ceil(512) * 512, 0);
404 }
405 gem.extend_from_slice(&[0; 1024]);
406 let spec = read_gem(&gem).unwrap();
407 assert_eq!((spec.name.as_str(), spec.version.as_str(), spec.platform.as_str()), ("hello-world", "0.2.0", "ruby"));
408 assert_eq!(spec.dependencies, [("rack".to_owned(), "< 4&>= 2.0".to_owned()), ("json".to_owned(), ">= 0".to_owned())], "runtime only");
409 assert_eq!(spec.ruby.as_deref(), Some(">= 3.0.0"));
410 assert_eq!(spec.rubygems, None, ">= 0 says nothing");
411 assert_eq!(spec.source_code_uri.as_deref(), Some("https://g1t.sh/acme/hello-world"));
412 assert_eq!(spec.licenses, ["MIT"]);
413 assert!(read_gem(b"not a gem").is_err());
414
415 let line = info_line(&key(&spec.version, &spec.platform), &stored(&spec), "abc123");
416 assert_eq!(line, "0.2.0 rack:< 4&>= 2.0,json:>= 0|checksum:abc123,ruby:>= 3.0.0");
417 let bare = info_line("1.0.0", &json!({ "dependencies": [] }), "ff");
418 assert_eq!(bare, "1.0.0 |checksum:ff");
419 }
420
421 #[test]
422 fn a_platform_mapping_reads_as_its_name() {
423 let spec = read_spec("name: native\nversion: !ruby/object:Gem::Version\n version: 1.0.0\nplatform: !ruby/object:Gem::Platform\n cpu: x86_64\n os: linux\n version:\n").unwrap();
424 assert_eq!(spec.platform, "x86_64-linux");
425 assert!(read_spec("name: x\n").is_err(), "no version");
426 }
427
428 #[test]
429 fn the_compact_index_is_bundlers_shape() {
430 let info = info(&["0.1.0 |checksum:aa".to_owned(), "0.2.0 rack:>= 2|checksum:bb".to_owned()]);
431 assert_eq!(info, "---\n0.1.0 |checksum:aa\n0.2.0 rack:>= 2|checksum:bb\n");
432 let versions = versions_file("2026-10-06T00:00:00Z", &[("hello".into(), vec!["0.1.0".into(), "0.2.0".into()], "d41d8".into())]);
433 assert_eq!(versions, "created_at: 2026-10-06T00:00:00Z\n---\nhello 0.1.0,0.2.0 d41d8\n");
434 assert_eq!(names_file(&["a".into(), "b".into()]), "---\na\nb\n");
435 assert_eq!(form_value("gem_name=hello-world&version=0.1.0&platform=", "gem_name").as_deref(), Some("hello-world"));
436 assert_eq!(form_value("gem_name=a%2Bb", "gem_name").as_deref(), Some("a+b"));
437 assert_eq!(form_value("version=1", "platform"), None);
438 }
439}