flagon-io/g1t

public

RubyGems: the full index (specs.4.8.gz, latest and prerelease, quick/Marshal.4.8 specifications) in Ruby's Marshal format, so gem install --source and gem search work

syntaqxcommitted 6h agoParent3ecc942Browse files

6 files+594−110/6 viewed

+28−8View BlameViewed

4	4		---
5	5
6	6		Every workspace has a gem registry of its own. `gem push` publishes to it,
7		−	and Bundler installs from it through the compact index (`versions`,
8		−	`info/<gem>`), private gems included. Install with Bundler: the registry
9		−	serves the compact index, not the older full index (`specs.4.8.gz`) that
10		−	`gem install --source` reads.
	7	+	and Bundler and `gem install` install from it, private gems included.
	8	+	It serves both indexes RubyGems reads: the compact index Bundler uses
	9	+	(`versions`, `info/<gem>`), and the full index (`specs.4.8.gz` and each
	10	+	version's specification) that `gem install --source` and `gem search`
	11	+	read.
11	12
12	13		```text
13	14		https://g1t.sh/-/rubygems/<workspace>/
⋯
96	97		`bundle install` then reads the compact index and downloads each `.gem`,
97	98		which it checks against the SHA-256 the index names.
98	99
	100	+	## Install with gem
	101	+
	102	+	To install a gem and what it depends on without a `Gemfile`, name the
	103	+	workspace's source. For private gems, put a username (any works) and a
	104	+	token in its address:
	105	+
	106	+	```sh
	107	+	gem install http-client --source https://ada:<token>@g1t.sh/-/rubygems/acme/
	108	+	```
	109	+
	110	+	Gems it depends on from rubygems.org need that source too: add
	111	+	`--source https://rubygems.org/` after the workspace's. To keep the
	112	+	source, add it once with `gem sources --add <address>`.
	113	+
	114	+	`gem search http --remote --source <address>` lists the workspace's gems,
	115	+	and `gem specification http-client --remote --source <address>` shows one.
	116	+	`--prerelease` includes pre-releases. Yanked versions are in neither.
	117	+
99	118		## Names and versions
100	119
101	120		A gem's name is letters, digits, `.`, `-` and `_`, with at least one
⋯
114	133		GEM_HOST_API_KEY=<token> gem yank http-client --version 0.3.1 --host https://g1t.sh/-/rubygems/acme
115	134		```
116	135
117		−	A yanked version leaves the index, so Bundler no longer resolves to it,
	136	+	A yanked version leaves both indexes, so Bundler and `gem install` no
	137	+	longer resolve to it,
118	138		but its `.gem` is still downloaded for a `Gemfile.lock` that names it.
119	139		Yanking needs what pushing does. The gem's page marks yanked versions, and
120	140		someone with Admin on the linked repository (an owner, for the
⋯
127	147
128	148		\| The workspace's gems \| Without credentials \| With credentials \|
129	149		\| --- \| --- \| --- \|
130		−	\| All public \| Bundler reads the index and downloads them. \| The same; the key is sent to push and yank. \|
131		−	\| Some private \| The registry answers `401`, and Bundler asks for credentials for the source. \| Each gem the credentials' owner may see. \|
	150	+	\| All public \| Bundler and `gem` read the index and download them. \| The same; the key is sent to push and yank. \|
	151	+	\| Some private \| The registry answers `401`: Bundler asks for credentials for the source, and `gem` needs them in the source's address. \| Each gem the credentials' owner may see. \|
132	152
133	153		A private gem you cannot see looks exactly like one that does not exist.
134	154
⋯
163	183
164	184		\| Error \| Means \|
165	185		\| --- \| --- \|
166		−	\| `401` \| No credentials or key, or a wrong or expired token. For Bundler, set the source's credentials with `bundle config set`; for `gem push`, give `GEM_HOST_API_KEY`. \|
	186	+	\| `401` \| No credentials or key, or a wrong or expired token. For Bundler, set the source's credentials with `bundle config set`; for `gem install`, put them in the source's address; for `gem push`, give `GEM_HOST_API_KEY`. \|
167	187		\| `403` \| Signed in, but your role or your token's scopes do not allow it, or the workspace is out of free package storage. The response says which. \|
168	188		\| `404` \| No such gem or version, or a private one you cannot see. \|
169	189		\| `409` \| That version is already pushed, or its name is taken by a gem named in another case. \|

+10−0View BlameViewed

87	87		Ok(data)
88	88		}
89	89
	90	+	/// `data`, gzipped: what RubyGems' full index files are.
	91	+	pub fn gzip(data: &[u8]) -> Vec<u8> {
	92	+	let mut out = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3];
	93	+	out.extend_from_slice(&miniz_oxide::deflate::compress_to_vec(data, 6));
	94	+	out.extend_from_slice(&crc32(data).to_le_bytes());
	95	+	out.extend_from_slice(&(data.len() as u32).to_le_bytes());
	96	+	out
	97	+	}
	98	+
90	99		/// The bytes of a gzip file, inflated, up to `limit`.
91	100		pub fn gunzip(bytes: &[u8], limit: usize) -> Result<Vec<u8>, String> {
92	101		let bad = \|\| "The file is not gzipped.".to_owned();
⋯
200	209		assert_eq!(files[1].1, b"data");
201	210		assert_eq!(gunzip(files[0].1, 1024).unwrap(), b"--- !ruby/object:Gem::Specification\nname: hello\n");
202	211		assert!(gunzip(b"plain text, not gzip at all", 1024).is_err());
	212	+	assert_eq!(gunzip(&super::gzip(b"specs"), 1024).unwrap(), b"specs");
203	213		assert!(tar_files(&gem[..1538]).is_err(), "cut short");
204	214		}
205	215		}

+1−0View BlameViewed

17	17		mod digest;
18	18		mod limits;
19	19		mod manifest;
	20	+	mod marshal;
20	21		mod maven;
21	22		mod maven_http;
22	23		mod names;

+203−0View BlameViewed

1	+	//! Ruby's Marshal format, version 4.8, written (never read): what the
2	+	//! RubyGems full index is made of. `specs.4.8.gz` is a marshalled array of
3	+	//! `[name, Gem::Version, platform]`, and each
4	+	//! `quick/Marshal.4.8/<gem>.gemspec.rz` a marshalled `Gem::Specification`.
5	+	//!
6	+	//! Only what those need is here: nil, booleans, small integers, strings
7	+	//! (UTF-8, or binary), symbols, arrays, hashes, plain objects with
8	+	//! instance variables, and the two kinds of custom dump RubyGems' classes
9	+	//! use (`marshal_dump`, which `Gem::Version` and `Gem::Requirement` use,
10	+	//! and `_dump`, which `Gem::Specification` uses). Symbols already written
11	+	//! are written again as links, as Ruby does; objects never are, which
12	+	//! Ruby reads the same.
13	+
14	+	use std::collections::HashMap;
15	+
16	+	/// A Ruby value to marshal.
17	+	#[derive(Clone, Debug, PartialEq)]
18	+	pub enum Value {
19	+	Nil,
20	+	Bool(bool),
21	+	/// An integer between -2^31 and 2^31, which Marshal writes as a Fixnum.
22	+	Int(i32),
23	+	/// A UTF-8 string.
24	+	Str(String),
25	+	/// A binary (ASCII-8BIT) string.
26	+	Bytes(Vec<u8>),
27	+	Symbol(String),
28	+	Array(Vec<Value>),
29	+	Hash(Vec<(Value, Value)>),
30	+	/// An object of `class` with these instance variables (`@name`).
31	+	Object { class: String, ivars: Vec<(String, Value)> },
32	+	/// What `class#marshal_dump` returned, for `class.marshal_load`.
33	+	UserMarshal { class: String, data: Box<Value> },
34	+	/// The bytes `class#_dump` returned, for `class._load`.
35	+	UserDef { class: String, data: Vec<u8> },
36	+	}
37	+
38	+	impl Value {
39	+	pub fn str(text: impl Into<String>) -> Value {
40	+	Value::Str(text.into())
41	+	}
42	+
43	+	/// A string, or nil for none.
44	+	pub fn opt(text: Option<&str>) -> Value {
45	+	text.map_or(Value::Nil, Value::str)
46	+	}
47	+	}
48	+
49	+	/// `value`, marshalled, with the 4.8 header.
50	+	pub fn dump(value: &Value) -> Vec<u8> {
51	+	let mut writer = Writer { out: vec![4, 8], symbols: HashMap::new() };
52	+	writer.value(value);
53	+	writer.out
54	+	}
55	+
56	+	struct Writer {
57	+	out: Vec<u8>,
58	+	symbols: HashMap<String, usize>,
59	+	}
60	+
61	+	impl Writer {
62	+	/// Marshal's integer: 0 as itself, -123..=122 in one byte offset by
63	+	/// five, else a byte count (negated for a negative) and the bytes,
64	+	/// least first.
65	+	fn long(&mut self, n: i64) {
66	+	if n == 0 {
67	+	self.out.push(0);
68	+	} else if (1..123).contains(&n) {
69	+	self.out.push((n + 5) as u8);
70	+	} else if (-123..0).contains(&n) {
71	+	self.out.push(((n - 5) & 0xff) as u8);
72	+	} else {
73	+	let mut bytes = Vec::new();
74	+	let mut rest = n;
75	+	for _ in 0..4 {
76	+	bytes.push((rest & 0xff) as u8);
77	+	rest >>= 8;
78	+	if (n > 0 && rest == 0) \|\| (n < 0 && rest == -1) {
79	+	break;
80	+	}
81	+	}
82	+	let count = bytes.len() as i64;
83	+	self.out.push(if n > 0 { count as u8 } else { (-count & 0xff) as u8 });
84	+	self.out.extend_from_slice(&bytes);
85	+	}
86	+	}
87	+
88	+	fn bytes(&mut self, bytes: &[u8]) {
89	+	self.long(bytes.len() as i64);
90	+	self.out.extend_from_slice(bytes);
91	+	}
92	+
93	+	fn symbol(&mut self, name: &str) {
94	+	if let Some(&index) = self.symbols.get(name) {
95	+	self.out.push(b';');
96	+	self.long(index as i64);
97	+	return;
98	+	}
99	+	let index = self.symbols.len();
100	+	self.symbols.insert(name.to_owned(), index);
101	+	self.out.push(b':');
102	+	self.bytes(name.as_bytes());
103	+	}
104	+
105	+	fn value(&mut self, value: &Value) {
106	+	match value {
107	+	Value::Nil => self.out.push(b'0'),
108	+	Value::Bool(true) => self.out.push(b'T'),
109	+	Value::Bool(false) => self.out.push(b'F'),
110	+	Value::Int(n) => {
111	+	self.out.push(b'i');
112	+	self.long(i64::from(*n));
113	+	}
114	+	// A string with an encoding is a string with one instance
115	+	// variable, `E`: true for UTF-8.
116	+	Value::Str(text) => {
117	+	self.out.push(b'I');
118	+	self.out.push(b'"');
119	+	self.bytes(text.as_bytes());
120	+	self.long(1);
121	+	self.symbol("E");
122	+	self.out.push(b'T');
123	+	}
124	+	Value::Bytes(bytes) => {
125	+	self.out.push(b'"');
126	+	self.bytes(bytes);
127	+	}
128	+	Value::Symbol(name) => self.symbol(name),
129	+	Value::Array(items) => {
130	+	self.out.push(b'[');
131	+	self.long(items.len() as i64);
132	+	for item in items {
133	+	self.value(item);
134	+	}
135	+	}
136	+	Value::Hash(pairs) => {
137	+	self.out.push(b'{');
138	+	self.long(pairs.len() as i64);
139	+	for (key, item) in pairs {
140	+	self.value(key);
141	+	self.value(item);
142	+	}
143	+	}
144	+	Value::Object { class, ivars } => {
145	+	self.out.push(b'o');
146	+	self.symbol(class);
147	+	self.long(ivars.len() as i64);
148	+	for (name, item) in ivars {
149	+	self.symbol(name);
150	+	self.value(item);
151	+	}
152	+	}
153	+	Value::UserMarshal { class, data } => {
154	+	self.out.push(b'U');
155	+	self.symbol(class);
156	+	self.value(data);
157	+	}
158	+	Value::UserDef { class, data } => {
159	+	self.out.push(b'u');
160	+	self.symbol(class);
161	+	self.bytes(data);
162	+	}
163	+	}
164	+	}
165	+	}
166	+
167	+	#[cfg(test)]
168	+	mod tests {
169	+	use super::*;
170	+
171	+	#[test]
172	+	fn values_are_written_as_ruby_writes_them() {
173	+	// Each as `Marshal.dump` writes it in Ruby 3.3.
174	+	assert_eq!(dump(&Value::Nil), b"\x04\x080");
175	+	assert_eq!(dump(&Value::Bool(true)), b"\x04\x08T");
176	+	assert_eq!(dump(&Value::Int(0)), b"\x04\x08i\x00");
177	+	assert_eq!(dump(&Value::Int(4)), b"\x04\x08i\x09");
178	+	assert_eq!(dump(&Value::Int(-1)), b"\x04\x08i\xfa");
179	+	assert_eq!(dump(&Value::Int(123)), b"\x04\x08i\x01\x7b");
180	+	assert_eq!(dump(&Value::Int(256)), b"\x04\x08i\x02\x00\x01");
181	+	assert_eq!(dump(&Value::Int(-124)), b"\x04\x08i\xff\x84");
182	+	assert_eq!(dump(&Value::Int(-256)), b"\x04\x08i\xff\x00");
183	+	assert_eq!(dump(&Value::Int(-257)), b"\x04\x08i\xfe\xff\xfe");
184	+	assert_eq!(dump(&Value::str("hi")), b"\x04\x08I\"\x07hi\x06:\x06ET");
185	+	assert_eq!(dump(&Value::Bytes(b"hi".to_vec())), b"\x04\x08\"\x07hi");
186	+	// The second `:a` is a link to the first.
187	+	assert_eq!(
188	+	dump(&Value::Array(vec![Value::Symbol("a".into()), Value::Symbol("a".into())])),
189	+	b"\x04\x08[\x07:\x06a;\x00"
190	+	);
191	+	assert_eq!(dump(&Value::Hash(vec![(Value::Int(1), Value::Nil)])), b"\x04\x08{\x06i\x060");
192	+	// Gem::Version.new("1.0")
193	+	assert_eq!(
194	+	dump(&Value::UserMarshal { class: "Gem::Version".into(), data: Box::new(Value::Array(vec![Value::str("1.0")])) }),
195	+	b"\x04\x08U:\x11Gem::Version[\x06I\"\x081.0\x06:\x06ET"
196	+	);
197	+	assert_eq!(
198	+	dump(&Value::Object { class: "Point".into(), ivars: vec![("@x".into(), Value::Int(1))] }),
199	+	b"\x04\x08o:\x0aPoint\x06:\x07@xi\x06"
200	+	);
201	+	assert_eq!(dump(&Value::UserDef { class: "X".into(), data: b"ab".to_vec() }), b"\x04\x08u:\x06X\x07ab");
202	+	}
203	+	}

+305−2View BlameViewed

1	1		//! What the RubyGems registry needs that does not touch the network: gem
2	2		//! names and versions, the registry's paths, the `Gem::Specification` read
3		−	//! from a `.gem` (a tar holding `metadata.gz`), and the compact index
4		−	//! Bundler reads: `versions`, `info/<gem>` and `names`.
	3	+	//! from a `.gem` (a tar holding `metadata.gz`), the compact index
	4	+	//! Bundler reads (`versions`, `info/<gem>` and `names`), and the full
	5	+	//! index `gem install --source` reads: `specs.4.8.gz` and its latest and
	6	+	//! pre-release kin, and each version's `quick/Marshal.4.8` specification.
5	7		//!
6	8		//! A version is keyed by its number and platform as the compact index
7	9		//! writes it (`1.0.0`, `1.0.0-x86_64-linux`), and keeps what its index
⋯
9	11
10	12		use serde_json::{Value, json};
11	13
	14	+	use std::cmp::Ordering;
	15	+
12	16		use crate::archive;
	17	+	use crate::marshal::{self, Value as Ruby};
13	18		use crate::yaml;
14	19
15	20		/// The longest gem name taken.
⋯
61	66		/// `gems/<name>-<version>[-<platform>].gem`; the name and version are
62	67		/// told apart by the handler, as names may hold `-`.
63	68		Gem { stem: String },
	69	+	/// `specs.4.8.gz`, `latest_specs.4.8.gz` or `prerelease_specs.4.8.gz`.
	70	+	Specs(Specs),
	71	+	/// `quick/Marshal.4.8/<name>-<version>[-<platform>].gemspec.rz`: one
	72	+	/// version's specification.
	73	+	QuickSpec { stem: String },
64	74		/// `api/v1/gems`: `gem push`.
65	75		Push,
66	76		/// `api/v1/gems/yank`: `gem yank`.
⋯
79	89		"names" => GemRoute::Names,
80	90		"api/v1/gems" => GemRoute::Push,
81	91		"api/v1/gems/yank" => GemRoute::Yank,
	92	+	"specs.4.8.gz" => GemRoute::Specs(Specs::Released),
	93	+	"latest_specs.4.8.gz" => GemRoute::Specs(Specs::Latest),
	94	+	"prerelease_specs.4.8.gz" => GemRoute::Specs(Specs::Prerelease),
82	95		other => {
	96	+	if let Some(file) = other.strip_prefix("quick/Marshal.4.8/") {
	97	+	let stem = file.strip_suffix(".gemspec.rz")?;
	98	+	if stem.contains('/') \|\| candidates(stem).is_empty() {
	99	+	return None;
	100	+	}
	101	+	return Some((workspace, GemRoute::QuickSpec { stem: stem.to_owned() }));
	102	+	}
83	103		if let Some(name) = other.strip_prefix("info/") {
84	104		if !valid_name(name) {
85	105		return None;
⋯
276	296		out
277	297		}
278	298
	299	+	/// Which of the full index's files: every released version, the highest
	300	+	/// released version of each gem and platform, or every pre-release.
	301	+	#[derive(Clone, Copy, Debug, PartialEq, Eq)]
	302	+	pub enum Specs {
	303	+	Released,
	304	+	Latest,
	305	+	Prerelease,
	306	+	}
	307	+
	308	+	/// A version's parts as `Gem::Version` compares them: `1.0.0.rc1` is
	309	+	/// `1 0 0 rc 1`, and `1.0.0-beta` is `1.0.0.pre.beta`.
	310	+	#[derive(Clone, Debug, PartialEq, Eq)]
	311	+	enum Part {
	312	+	Number(u64),
	313	+	Word(String),
	314	+	}
	315	+
	316	+	fn parts(version: &str) -> Vec<Part> {
	317	+	let version = version.trim().replace('-', ".pre.");
	318	+	let mut out = Vec::new();
	319	+	for piece in version.split('.') {
	320	+	let mut rest = piece;
	321	+	while !rest.is_empty() {
	322	+	let digits = rest.bytes().next().is_some_and(\|b\| b.is_ascii_digit());
	323	+	let len = rest.bytes().take_while(\|b\| b.is_ascii_digit() == digits).count();
	324	+	let (run, after) = rest.split_at(len);
	325	+	out.push(if digits { Part::Number(run.parse().unwrap_or(u64::MAX)) } else { Part::Word(run.to_owned()) });
	326	+	rest = after;
	327	+	}
	328	+	}
	329	+	out
	330	+	}
	331	+
	332	+	/// `Gem::Version`'s order: by part, a missing part is 0, and a word (a
	333	+	/// pre-release) is lower than any number.
	334	+	pub fn compare(a: &str, b: &str) -> Ordering {
	335	+	let (a, b) = (parts(a), parts(b));
	336	+	for i in 0..a.len().max(b.len()) {
	337	+	let zero = Part::Number(0);
	338	+	let (x, y) = (a.get(i).unwrap_or(&zero), b.get(i).unwrap_or(&zero));
	339	+	let order = match (x, y) {
	340	+	(Part::Number(x), Part::Number(y)) => x.cmp(y),
	341	+	(Part::Word(x), Part::Word(y)) => x.cmp(y),
	342	+	(Part::Word(_), Part::Number(_)) => Ordering::Less,
	343	+	(Part::Number(_), Part::Word(_)) => Ordering::Greater,
	344	+	};
	345	+	if order != Ordering::Equal {
	346	+	return order;
	347	+	}
	348	+	}
	349	+	Ordering::Equal
	350	+	}
	351	+
	352	+	/// One version in the full index: its gem, number and platform.
	353	+	#[derive(Clone, Debug, PartialEq, Eq)]
	354	+	pub struct Tuple {
	355	+	pub name: String,
	356	+	pub number: String,
	357	+	pub platform: String,
	358	+	}
	359	+
	360	+	impl Tuple {
	361	+	/// What a version keeps says its number and platform.
	362	+	pub fn of(name: &str, key: &str, stored: &Value) -> Tuple {
	363	+	let platform = stored["platform"].as_str().filter(\|p\| !p.is_empty()).unwrap_or("ruby").to_owned();
	364	+	let number = stored["number"].as_str().map(str::to_owned).unwrap_or_else(\|\| {
	365	+	key.strip_suffix(&format!("-{platform}")).unwrap_or(key).to_owned()
	366	+	});
	367	+	Tuple { name: name.to_owned(), number, platform }
	368	+	}
	369	+	}
	370	+
	371	+	/// A `Gem::Version`, marshalled as its `marshal_dump`: `[version]`.
	372	+	fn gem_version(number: &str) -> Ruby {
	373	+	Ruby::UserMarshal { class: "Gem::Version".into(), data: Box::new(Ruby::Array(vec![Ruby::str(number)])) }
	374	+	}
	375	+
	376	+	/// A full index file: each version of `tuples` that `which` lists, as
	377	+	/// `[name, Gem::Version, platform]`, marshalled and gzipped.
	378	+	pub fn specs_file(which: Specs, tuples: &[Tuple]) -> Vec<u8> {
	379	+	let mut chosen: Vec<&Tuple> = match which {
	380	+	Specs::Released => tuples.iter().filter(\|t\| !is_prerelease(&t.number)).collect(),
	381	+	Specs::Prerelease => tuples.iter().filter(\|t\| is_prerelease(&t.number)).collect(),
	382	+	Specs::Latest => {
	383	+	let mut highest: Vec<&Tuple> = Vec::new();
	384	+	for tuple in tuples.iter().filter(\|t\| !is_prerelease(&t.number)) {
	385	+	match highest.iter_mut().find(\|h\| h.name == tuple.name && h.platform == tuple.platform) {
	386	+	Some(kept) if compare(&tuple.number, &kept.number) == Ordering::Greater => *kept = tuple,
	387	+	Some(_) => {}
	388	+	None => highest.push(tuple),
	389	+	}
	390	+	}
	391	+	highest
	392	+	}
	393	+	};
	394	+	chosen.sort_by(\|a, b\| a.name.cmp(&b.name).then_with(\|\| compare(&a.number, &b.number)).then_with(\|\| a.platform.cmp(&b.platform)));
	395	+	let list = chosen
	396	+	.into_iter()
	397	+	.map(\|t\| Ruby::Array(vec![Ruby::str(&t.name), gem_version(&t.number), Ruby::str(&t.platform)]))
	398	+	.collect();
	399	+	archive::gzip(&marshal::dump(&Ruby::Array(list)))
	400	+	}
	401	+
	402	+	/// A `Gem::Requirement` from the index's form (`< 4&>= 2.0`), marshalled
	403	+	/// as its `marshal_dump`: `[[[op, Gem::Version], ...]]`.
	404	+	fn gem_requirement(text: Option<&str>) -> Ruby {
	405	+	let mut pairs: Vec<Ruby> = text
	406	+	.unwrap_or("")
	407	+	.split('&')
	408	+	.map(str::trim)
	409	+	.filter(\|r\| !r.is_empty())
	410	+	.map(\|r\| {
	411	+	let (op, number) = match r.split_once(' ') {
	412	+	Some((op, number)) => (op.trim(), number.trim()),
	413	+	None => ("=", r),
	414	+	};
	415	+	Ruby::Array(vec![Ruby::str(op), gem_version(number)])
	416	+	})
	417	+	.collect();
	418	+	if pairs.is_empty() {
	419	+	pairs.push(Ruby::Array(vec![Ruby::str(">="), gem_version("0")]));
	420	+	}
	421	+	Ruby::UserMarshal { class: "Gem::Requirement".into(), data: Box::new(Ruby::Array(vec![Ruby::Array(pairs)])) }
	422	+	}
	423	+
	424	+	/// A `Gem::Platform` (`x86_64-linux` is cpu `x86_64`, os `linux`), or the
	425	+	/// string `ruby` for a pure-Ruby gem, as RubyGems marshals it.
	426	+	fn gem_platform(platform: &str) -> Ruby {
	427	+	if platform == "ruby" {
	428	+	return Ruby::str("ruby");
	429	+	}
	430	+	let parts: Vec<&str> = platform.splitn(3, '-').collect();
	431	+	let (cpu, os, version) = match parts.as_slice() {
	432	+	[os] => (None, *os, None),
	433	+	[cpu, os] => (Some(cpu), os, None),
	434	+	[cpu, os, version, ..] => (Some(cpu), os, Some(*version)),
	435	+	[] => (None, platform, None),
	436	+	};
	437	+	Ruby::Object {
	438	+	class: "Gem::Platform".into(),
	439	+	ivars: vec![("@cpu".into(), Ruby::opt(cpu)), ("@os".into(), Ruby::str(os)), ("@version".into(), Ruby::opt(version))],
	440	+	}
	441	+	}
	442	+
	443	+	/// A version's `Gem::Specification`, from what it keeps, marshalled as
	444	+	/// RubyGems' `_dump` writes it and deflated: the `.gemspec.rz` that
	445	+	/// `gem install` reads before it downloads the gem.
	446	+	pub fn quick_spec(name: &str, key: &str, stored: &Value, published_at: &str) -> Vec<u8> {
	447	+	let tuple = Tuple::of(name, key, stored);
	448	+	let text = \|key: &str\| stored[key].as_str().map(str::trim).filter(\|t\| !t.is_empty());
	449	+	let strings = \|key: &str\| Ruby::Array(texts(&stored[key]).into_iter().map(Ruby::Str).collect());
	450	+	let dependencies = stored["dependencies"]
	451	+	.as_array()
	452	+	.map(\|deps\| {
	453	+	deps.iter()
	454	+	.filter_map(\|d\| {
	455	+	let name = d["name"].as_str()?;
	456	+	let requirement = gem_requirement(d["requirement"].as_str());
	457	+	Some(Ruby::Object {
	458	+	class: "Gem::Dependency".into(),
	459	+	ivars: vec![
	460	+	("@name".into(), Ruby::str(name)),
	461	+	("@requirement".into(), requirement.clone()),
	462	+	("@type".into(), Ruby::Symbol("runtime".into())),
	463	+	("@prerelease".into(), Ruby::Bool(false)),
	464	+	("@version_requirements".into(), requirement),
	465	+	],
	466	+	})
	467	+	})
	468	+	.collect()
	469	+	})
	470	+	.unwrap_or_default();
	471	+	let mut metadata = Vec::new();
	472	+	if let Some(uri) = text("source_code_uri") {
	473	+	metadata.push((Ruby::str("source_code_uri"), Ruby::str(uri)));
	474	+	}
	475	+	let date = published_at.get(..10).filter(\|d\| d.len() == 10).unwrap_or("1980-01-02");
	476	+	// The fields of `Gem::Specification#_dump`, in its order.
	477	+	let fields = Ruby::Array(vec![
	478	+	Ruby::str(text("rubygems_version").unwrap_or("3.5.0")),
	479	+	Ruby::Int(4),
	480	+	Ruby::str(&tuple.name),
	481	+	gem_version(&tuple.number),
	482	+	Ruby::str(date),
	483	+	Ruby::str(text("summary").unwrap_or("")),
	484	+	gem_requirement(text("ruby")),
	485	+	gem_requirement(text("rubygems")),
	486	+	Ruby::str(&tuple.platform),
	487	+	Ruby::Array(dependencies),
	488	+	Ruby::str(""),
	489	+	Ruby::Nil,
	490	+	strings("authors"),
	491	+	Ruby::opt(text("description")),
	492	+	Ruby::opt(text("homepage")),
	493	+	Ruby::Bool(true),
	494	+	gem_platform(&tuple.platform),
	495	+	strings("licenses"),
	496	+	Ruby::Hash(metadata),
	497	+	]);
	498	+	let spec = Ruby::UserDef { class: "Gem::Specification".into(), data: marshal::dump(&fields) };
	499	+	miniz_oxide::deflate::compress_to_vec_zlib(&marshal::dump(&spec), 6)
	500	+	}
	501	+
279	502		/// A value from a form body or query string (`gem_name=hello&version=1.0`).
280	503		pub fn form_value(form: &str, key: &str) -> Option<String> {
281	504		let url = worker::Url::parse(&format!("http://form.invalid/?{form}")).ok()?;
⋯
301	524		assert!(!valid_version(bad), "{bad}");
302	525		}
303	526		assert!(is_prerelease("2.0.0.rc1") && !is_prerelease("2.0.0"));
	527	+	let mut sorted = vec!["1.10.0", "1.0.0", "1.0.0.rc1", "1.0", "1.2.0-beta.1", "1.2.0", "0.9"];
	528	+	sorted.sort_by(\|a, b\| compare(a, b));
	529	+	assert_eq!(sorted, ["0.9", "1.0.0.rc1", "1.0.0", "1.0", "1.2.0-beta.1", "1.2.0", "1.10.0"]);
304	530		assert_eq!(key("1.0.0", "ruby"), "1.0.0");
305	531		assert_eq!(key("1.0.0", "x86_64-linux"), "1.0.0-x86_64-linux");
306	532		}
⋯
315	541		assert_eq!(route("/-/rubygems/acme/api/v1/gems"), at(GemRoute::Push));
316	542		assert_eq!(route("/-/rubygems/acme/api/v1/gems/yank"), at(GemRoute::Yank));
317	543		assert_eq!(route("/-/rubygems/acme/gems/hello.gem"), None, "no version");
	544	+	assert_eq!(route("/-/rubygems/acme/specs.4.8.gz"), at(GemRoute::Specs(Specs::Released)));
	545	+	assert_eq!(route("/-/rubygems/acme/latest_specs.4.8.gz"), at(GemRoute::Specs(Specs::Latest)));
	546	+	assert_eq!(route("/-/rubygems/acme/prerelease_specs.4.8.gz"), at(GemRoute::Specs(Specs::Prerelease)));
	547	+	assert_eq!(
	548	+	route("/-/rubygems/acme/quick/Marshal.4.8/hello-world-0.1.0.gemspec.rz"),
	549	+	at(GemRoute::QuickSpec { stem: "hello-world-0.1.0".into() })
	550	+	);
	551	+	assert_eq!(route("/-/rubygems/acme/quick/Marshal.4.8/hello.gemspec.rz"), None, "no version");
318	552		assert_eq!(route("/-/rubygems/acme/info/a b"), None);
319	553		assert_eq!(route("/-/rubygems/acme/other"), None);
320	554		assert_eq!(route("/-/rubygems/acme"), None);
⋯
436	670		assert_eq!(form_value("gem_name=a%2Bb", "gem_name").as_deref(), Some("a+b"));
437	671		assert_eq!(form_value("version=1", "platform"), None);
438	672		}
	673	+
	674	+	fn tuples() -> Vec<Tuple> {
	675	+	let tuple = \|name: &str, number: &str, platform: &str\| Tuple { name: name.into(), number: number.into(), platform: platform.into() };
	676	+	vec![
	677	+	tuple("hello", "0.2.0", "ruby"),
	678	+	tuple("hello", "0.10.0", "ruby"),
	679	+	tuple("hello", "1.0.0.rc1", "ruby"),
	680	+	tuple("hello", "0.10.0", "java"),
	681	+	tuple("abc", "1.0.0", "ruby"),
	682	+	]
	683	+	}
	684	+
	685	+	#[test]
	686	+	fn the_full_index_lists_versions_as_tuples() {
	687	+	let file = specs_file(Specs::Latest, &tuples());
	688	+	let bytes = archive::gunzip(&file, 1 << 20).unwrap();
	689	+	// By name, then version and platform: hello's highest of each.
	690	+	let dumped = marshal::dump(&Ruby::Array(vec![
	691	+	Ruby::Array(vec![Ruby::str("abc"), gem_version("1.0.0"), Ruby::str("ruby")]),
	692	+	Ruby::Array(vec![Ruby::str("hello"), gem_version("0.10.0"), Ruby::str("java")]),
	693	+	Ruby::Array(vec![Ruby::str("hello"), gem_version("0.10.0"), Ruby::str("ruby")]),
	694	+	]));
	695	+	assert_eq!(bytes, dumped);
	696	+	let released = archive::gunzip(&specs_file(Specs::Released, &tuples()), 1 << 20).unwrap();
	697	+	assert_eq!(released.windows(5).filter(\|w\| *w == b"hello").count(), 3, "every released version");
	698	+	let pre = archive::gunzip(&specs_file(Specs::Prerelease, &tuples()), 1 << 20).unwrap();
	699	+	assert!(pre.windows(9).any(\|w\| w == b"1.0.0.rc1"));
	700	+	assert!(!pre.windows(6).any(\|w\| w == b"0.10.0"));
	701	+	}
	702	+
	703	+	#[test]
	704	+	fn a_quick_spec_is_a_deflated_specification() {
	705	+	let stored = json!({
	706	+	"name": "hello-world", "number": "0.2.0", "platform": "ruby", "summary": "Says hello",
	707	+	"authors": ["Ada"], "licenses": ["MIT"], "homepage": "https://g1t.sh/acme/hello-world",
	708	+	"dependencies": [{ "name": "rack", "requirement": "< 4&>= 2.0" }], "ruby": ">= 3.0.0", "rubygems": null,
	709	+	});
	710	+	let rz = quick_spec("hello-world", "0.2.0", &stored, "2026-10-07T01:02:03.000Z");
	711	+	let bytes = miniz_oxide::inflate::decompress_to_vec_zlib(&rz).unwrap();
	712	+	assert_eq!(&bytes[..3], b"\x04\x08u");
	713	+	assert!(bytes.windows(18).any(\|w\| w == b"Gem::Specification"));
	714	+	assert!(bytes.windows(10).any(\|w\| w == b"2026-10-07"));
	715	+	assert!(bytes.windows(15).any(\|w\| w == b"Gem::Dependency"));
	716	+	// A gem built for a platform names it as a Gem::Platform too.
	717	+	let native = quick_spec("native", "1.0.0-x86_64-linux", &json!({ "number": "1.0.0", "platform": "x86_64-linux" }), "");
	718	+	let bytes = miniz_oxide::inflate::decompress_to_vec_zlib(&native).unwrap();
	719	+	assert!(bytes.windows(13).any(\|w\| w == b"Gem::Platform"));
	720	+	assert_eq!(Tuple::of("native", "1.0.0-x86_64-linux", &json!({})).number, "1.0.0-x86_64-linux", "no platform kept: the key");
	721	+	assert_eq!(Tuple::of("native", "1.0.0-java", &json!({ "platform": "java" })).number, "1.0.0");
	722	+	}
	723	+
	724	+	/// Writes the full index for `tuples()` and a quick spec where a real
	725	+	/// Ruby can read them: `G1T_MARSHAL_OUT=<dir> cargo test marshal_files`,
	726	+	/// then `ruby -e` over the files (see the RubyGems guide's notes).
	727	+	#[test]
	728	+	fn marshal_files_for_ruby() {
	729	+	let Ok(dir) = std::env::var("G1T_MARSHAL_OUT") else { return };
	730	+	let dir = std::path::Path::new(&dir);
	731	+	std::fs::write(dir.join("specs.4.8.gz"), specs_file(Specs::Released, &tuples())).unwrap();
	732	+	std::fs::write(dir.join("latest_specs.4.8.gz"), specs_file(Specs::Latest, &tuples())).unwrap();
	733	+	let stored = json!({
	734	+	"name": "hello-world", "number": "0.2.0", "platform": "ruby", "summary": "Says hello", "description": "Says hello.",
	735	+	"authors": ["Ada"], "licenses": ["MIT"], "homepage": "https://g1t.sh/acme/hello-world", "source_code_uri": "https://g1t.sh/acme/hello-world",
	736	+	"dependencies": [{ "name": "rack", "requirement": "< 4&>= 2.0" }, { "name": "json", "requirement": ">= 0" }], "ruby": ">= 3.0.0",
	737	+	});
	738	+	std::fs::write(dir.join("hello-world-0.2.0.gemspec.rz"), quick_spec("hello-world", "0.2.0", &stored, "2026-10-07T00:00:00.000Z")).unwrap();
	739	+	let native = json!({ "name": "native", "number": "1.0.0", "platform": "x86_64-linux", "dependencies": [] });
	740	+	std::fs::write(dir.join("native-1.0.0-x86_64-linux.gemspec.rz"), quick_spec("native", "1.0.0-x86_64-linux", &native, "2026-10-07T00:00:00.000Z")).unwrap();
	741	+	}
439	742		}

+47−1View BlameViewed

6	6		//!
7	7		//! Bundler installs from the compact index (`versions`, `info/<gem>`,
8	8		//! `names`), made from the versions on each read, with each file's MD5 as
9		−	//! its `ETag` as Bundler checks it. A `.gem` is stored once, by its
	9	+	//! its `ETag` as Bundler checks it. `gem install --source` and `gem search`
	10	+	//! read the full index: `specs.4.8.gz` (and `latest_` and `prerelease_`),
	11	+	//! and a version's `quick/Marshal.4.8/<gem>.gemspec.rz`, made from what
	12	+	//! each version keeps, in Ruby's Marshal format. A `.gem` is stored once, by its
10	13		//! SHA-256, which is also its index `checksum`. `gem yank` takes a version
11	14		//! out of the index; its file stays for lockfiles that name it.
12	15
⋯
66	69		Ok(Response::from_body(body)?.with_headers(headers))
67	70		}
68	71
	72	+	/// A full index file or a specification, which `gem` reads as bytes.
	73	+	fn binary(bytes: Vec<u8>, head: bool, cache: &str) -> Result<Response> {
	74	+	let headers = Headers::new();
	75	+	headers.set("content-type", "application/octet-stream")?;
	76	+	headers.set("content-length", &bytes.len().to_string())?;
	77	+	headers.set("cache-control", cache)?;
	78	+	let body = if head { ResponseBody::Empty } else { ResponseBody::Body(bytes) };
	79	+	Ok(Response::from_body(body)?.with_headers(headers))
	80	+	}
	81	+
69	82		/// A version's line in the index.
70	83		fn line(row: &VersionRow) -> String {
71	84		let checksum = Digest::parse(&row.digest).map(\|d\| d.hex().to_owned()).unwrap_or_default();
⋯
128	141		GemRoute::Names if read => self.gem_names(&request, workspace, viewer, head).await,
129	142		GemRoute::Info { name } if read => self.gem_info(&request, workspace, &name, viewer, head).await,
130	143		GemRoute::Gem { stem } if read => self.gem_download(workspace, &stem, viewer, head, ctx).await,
	144	+	GemRoute::Specs(which) if read => self.gem_specs(workspace, which, viewer, head).await,
	145	+	GemRoute::QuickSpec { stem } if read => self.gem_quick_spec(workspace, &stem, viewer, head).await,
131	146		GemRoute::Push if method == Method::Post => self.gem_push(&mut request, workspace, viewer).await,
132	147		GemRoute::Yank if method == Method::Delete => self.gem_yank(&mut request, url, workspace, viewer).await,
133	148		_ => error(405, "Not a method this address takes."),
⋯
224	239		index_file(request, rubygems::info(&rows.iter().map(line).collect::<Vec<_>>()), head)
225	240		}
226	241
	242	+	/// A full index file: the versions in the index of every gem the
	243	+	/// viewer may see, as `[name, Gem::Version, platform]`.
	244	+	async fn gem_specs(&self, workspace: &str, which: rubygems::Specs, viewer: Option<&User>, head: bool) -> Result<Response> {
	245	+	let gems = match self.gem_index(workspace, viewer).await? {
	246	+	Ok(gems) => gems,
	247	+	Err(refused) => return Ok(refused),
	248	+	};
	249	+	let tuples: Vec<rubygems::Tuple> =
	250	+	gems.iter().flat_map(\|(package, rows)\| rows.iter().map(\|row\| rubygems::Tuple::of(&package.name, &row.version, &row.meta()))).collect();
	251	+	binary(rubygems::specs_file(which, &tuples), head, "no-cache")
	252	+	}
	253	+
	254	+	/// A version's specification, marshalled and deflated, which `gem
	255	+	/// install` reads before the gem. Yanked versions' too, as their files.
	256	+	async fn gem_quick_spec(&self, workspace: &str, stem: &str, viewer: Option<&User>, head: bool) -> Result<Response> {
	257	+	for (name, key) in rubygems::candidates(stem).into_iter().rev() {
	258	+	let Some(package) = self.db.package(workspace, RUBYGEMS, &name).await?.filter(\|p\| !p.hidden()) else {
	259	+	continue;
	260	+	};
	261	+	if let Some(refusal) = self.gem_check(viewer, &package, Action::Pull).await? {
	262	+	return Ok(refusal);
	263	+	}
	264	+	let Some(row) = self.db.version_named(&package.id, &key).await? else {
	265	+	continue;
	266	+	};
	267	+	let spec = rubygems::quick_spec(&package.name, &row.version, &row.meta(), &row.published_at);
	268	+	return binary(spec, head, "max-age=300");
	269	+	}
	270	+	self.gem_absent(workspace, viewer).await
	271	+	}
	272	+
227	273		/// A `.gem`, yanked ones too: a lockfile may still name them.
228	274		async fn gem_download(&self, workspace: &str, stem: &str, viewer: Option<&User>, head: bool, ctx: &Context) -> Result<Response> {
229	275		for (name, key) in rubygems::candidates(stem).into_iter().rev() {