Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/security/src/registries.rs

364 lines16,622 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Teams and CODEOWNERS, labels and milestones, dependency updates, the security suite, and a clearer top bar1//! What versions a package has, from its registry: npm's, crates.io (or a
2//! Cargo sparse index), the Go module proxy and PyPI (or a simple index),
3//! or a private registry the dependency update file names in their place.
4//! Reading each registry's answer is kept apart from fetching it, so it is
5//! tested on its own.
6
7use serde_json::Value;
8
9use g1t_contracts::time::parse_rfc3339;
10
11/// One published version.
12#[derive(Clone, Debug, PartialEq, Eq)]
13pub struct Release {
14 pub version: String,
15 /// When it was published, in milliseconds; unknown for some registries.
16 pub published_ms: Option<u64>,
17 /// Yanked or deprecated: never updated to.
18 pub withdrawn: bool,
19}
20
21/// What a registry says about a package.
22#[derive(Clone, Debug, Default, PartialEq, Eq)]
23pub struct Package {
24 pub releases: Vec<Release>,
25 /// Its source repository, for links in the pull request.
26 pub source: Option<String>,
27 /// Its changelog or release notes, when the registry names one.
28 pub changelog: Option<String>,
29 /// Its page on the registry.
30 pub page: Option<String>,
31}
32
33/// A registry to ask, with what it needs to answer.
34#[derive(Clone, Debug, PartialEq, Eq)]
35pub struct Source {
36 /// Without a trailing slash.
37 pub url: String,
38 /// The `authorization` header's value.
39 pub authorization: Option<String>,
40}
41
42/// A repository URL as registries write it (`git+https://…/x.git`,
43/// `github:owner/repo`), as a page a person can open.
44pub fn web_url(raw: &str) -> Option<String> {
45 let raw = raw.trim();
46 let raw = raw.strip_prefix("git+").unwrap_or(raw);
47 let raw = raw.strip_suffix(".git").unwrap_or(raw);
48 if let Some(path) = raw.strip_prefix("github:") {
49 return Some(format!("https://github.com/{path}"));
50 }
51 let raw = raw.replace("git://", "https://").replace("ssh://git@", "https://");
52 let raw = raw.strip_prefix("git@").map(|rest| format!("https://{}", rest.replacen(':', "/", 1))).unwrap_or(raw);
53 (raw.starts_with("https://") || raw.starts_with("http://")).then_some(raw)
54}
55
56/// npm's package document: `versions`, `time` and `repository`.
57pub fn npm(document: &Value, name: &str) -> Package {
58 let times = document.get("time");
59 let releases = document
60 .get("versions")
61 .and_then(Value::as_object)
62 .into_iter()
63 .flatten()
64 .map(|(version, manifest)| Release {
65 version: version.clone(),
66 published_ms: times.and_then(|times| times.get(version)).and_then(Value::as_str).and_then(parse_rfc3339),
67 withdrawn: manifest.get("deprecated").is_some_and(|deprecated| deprecated.as_str().is_some_and(|text| !text.is_empty())),
68 })
69 .collect();
70 let repository = document.get("repository").and_then(|repository| repository.as_str().or_else(|| repository.get("url")?.as_str()));
71 Package {
72 releases,
73 source: repository.and_then(web_url),
74 changelog: None,
75 page: Some(format!("https://www.npmjs.com/package/{name}")),
76 }
77}
78
79/// crates.io's `/api/v1/crates/<name>`: the crate and its versions.
80pub fn crates_io(document: &Value, name: &str) -> Package {
81 let releases = document
82 .get("versions")
83 .and_then(Value::as_array)
84 .into_iter()
85 .flatten()
86 .filter_map(|version| {
87 Some(Release {
88 version: version.get("num")?.as_str()?.to_owned(),
89 published_ms: version.get("created_at").and_then(Value::as_str).and_then(normalized_time),
90 withdrawn: version.get("yanked").and_then(Value::as_bool).unwrap_or(false),
91 })
92 })
93 .collect();
94 let krate = document.get("crate");
95 Package {
96 releases,
97 source: krate.and_then(|krate| krate.get("repository")).and_then(Value::as_str).and_then(web_url),
98 changelog: None,
99 page: Some(format!("https://crates.io/crates/{name}")),
100 }
101}
102
103/// A Cargo sparse index file: one JSON object per line.
104pub fn sparse_index(text: &str) -> Package {
105 let releases = text
106 .lines()
107 .filter_map(|line| serde_json::from_str::<Value>(line).ok())
108 .filter_map(|entry| {
109 Some(Release {
110 version: entry.get("vers")?.as_str()?.to_owned(),
111 published_ms: entry.get("pubtime").and_then(Value::as_str).and_then(normalized_time),
112 withdrawn: entry.get("yanked").and_then(Value::as_bool).unwrap_or(false),
113 })
114 })
115 .collect();
116 Package { releases, ..Package::default() }
117}
118
119/// Where a crate's sparse index file is: `se/rd/serde`, `3/a/abc`, `1/a`.
120pub fn sparse_path(name: &str) -> String {
121 let name = name.to_lowercase();
122 match name.len() {
123 1 => format!("1/{name}"),
124 2 => format!("2/{name}"),
125 3 => format!("3/{}/{name}", &name[..1]),
126 _ => format!("{}/{}/{name}", &name[..2], &name[2..4]),
127 }
128}
129
130/// A Go module path as the proxy takes it: capitals as `!` and the letter.
131pub fn go_escape(module: &str) -> String {
132 let mut out = String::with_capacity(module.len());
133 for c in module.chars() {
134 if c.is_ascii_uppercase() {
135 out.push('!');
136 out.push(c.to_ascii_lowercase());
137 } else {
138 out.push(c);
139 }
140 }
141 out
142}
143
144/// The proxy's `@v/list`: one version per line. When each was published
145/// is asked of `@v/<version>.info` only for the versions that matter.
146pub fn go_list(text: &str, module: &str) -> Package {
147 let releases = text
148 .lines()
149 .map(str::trim)
150 .filter(|line| !line.is_empty() && !line.contains("+incompatible"))
151 .map(|version| Release { version: version.to_owned(), published_ms: None, withdrawn: false })
152 .collect();
153 let source = ["github.com/", "gitlab.com/", "bitbucket.org/", "g1t.sh/"]
154 .iter()
155 .any(|host| module.starts_with(host))
156 .then(|| format!("https://{}", module.split('/').take(3).collect::<Vec<_>>().join("/")));
157 Package { releases, source, changelog: None, page: Some(format!("https://pkg.go.dev/{module}")) }
158}
159
160/// `@v/<version>.info`'s `Time`.
161pub fn go_info_time(document: &Value) -> Option<u64> {
162 document.get("Time").and_then(Value::as_str).and_then(normalized_time)
163}
164
165/// PyPI's `/pypi/<name>/json`: every release's files, and the project's links.
166pub fn pypi(document: &Value, name: &str) -> Package {
167 let releases = document
168 .get("releases")
169 .and_then(Value::as_object)
170 .into_iter()
171 .flatten()
172 .filter_map(|(version, files)| {
173 let files = files.as_array()?;
174 if files.is_empty() {
175 return None;
176 }
177 let published_ms = files
178 .iter()
179 .filter_map(|file| file.get("upload_time_iso_8601").and_then(Value::as_str).and_then(normalized_time))
180 .min();
181 let withdrawn = files.iter().all(|file| file.get("yanked").and_then(Value::as_bool).unwrap_or(false));
182 Some(Release { version: version.clone(), published_ms, withdrawn })
183 })
184 .collect();
185 let info = document.get("info");
186 let urls = info.and_then(|info| info.get("project_urls")).and_then(Value::as_object);
187 let link = |keys: &[&str]| -> Option<String> {
188 urls?.iter().find(|(key, _)| keys.iter().any(|wanted| key.eq_ignore_ascii_case(wanted))).and_then(|(_, url)| url.as_str()).map(str::to_owned)
189 };
190 Package {
191 releases,
192 source: link(&["Source", "Source Code", "Repository", "Code", "GitHub"]).and_then(|url| web_url(&url)),
193 changelog: link(&["Changelog", "Changes", "Release Notes", "Release notes", "History"]),
194 page: Some(format!("https://pypi.org/project/{name}/")),
195 }
196}
197
198/// A simple index's JSON page (PEP 691 and 700): its `versions`, and when
199/// each file was uploaded.
200pub fn simple_index(document: &Value) -> Package {
201 let versions: Vec<String> =
202 document.get("versions").and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).map(str::to_owned).collect();
203 let files: Vec<&Value> = document.get("files").and_then(Value::as_array).into_iter().flatten().collect();
204 let releases = versions
205 .into_iter()
206 .map(|version| {
207 let mine: Vec<&&Value> = files
208 .iter()
209 .filter(|file| {
210 file.get("filename").and_then(Value::as_str).is_some_and(|name| name.contains(&format!("-{version}-")) || name.contains(&format!("-{version}.")))
211 })
212 .collect();
213 Release {
214 published_ms: mine.iter().filter_map(|file| file.get("upload-time").and_then(Value::as_str).and_then(normalized_time)).min(),
215 withdrawn: !mine.is_empty() && mine.iter().all(|file| file.get("yanked").is_some_and(|yanked| yanked.as_bool() != Some(false))),
216 version,
217 }
218 })
219 .collect();
220 Package { releases, ..Package::default() }
221}
222
223/// An ISO 8601 time with or without fractions or a zone offset, as
224/// registries write them, as milliseconds.
225pub fn normalized_time(text: &str) -> Option<u64> {
226 let text = text.trim();
227 let (main, offset_minutes) = if let Some(stripped) = text.strip_suffix('Z') {
228 (stripped, 0i64)
229 } else if let Some(at) = text.rfind(['+', '-']).filter(|at| *at > 10) {
230 let (main, zone) = text.split_at(at);
231 let sign = if zone.starts_with('-') { -1 } else { 1 };
232 let digits: String = zone[1..].chars().filter(char::is_ascii_digit).collect();
233 let hours: i64 = digits.get(..2)?.parse().ok()?;
234 let minutes: i64 = digits.get(2..4).unwrap_or("00").parse().ok()?;
235 (main, sign * (hours * 60 + minutes))
236 } else {
237 (text, 0)
238 };
239 let (date, time) = main.split_once('T').or_else(|| main.split_once(' '))?;
240 let (clock, fraction) = time.split_once('.').unwrap_or((time, ""));
241 let millis: String = fraction.chars().take_while(char::is_ascii_digit).chain("000".chars()).take(3).collect();
242 let ms = parse_rfc3339(&format!("{date}T{clock}.{millis}Z"))? as i64 - offset_minutes * 60_000;
243 u64::try_from(ms).ok()
244}
245
246/// The URL to ask about `name` in `ecosystem`, given the registry to use
247/// (`None` for the public one).
248pub fn package_url(ecosystem: &str, name: &str, private: Option<&Source>) -> Option<String> {
249 Some(match (ecosystem, private) {
250 ("npm", None) => format!("https://registry.npmjs.org/{}", name.replace('/', "%2f")),
251 ("npm", Some(source)) => format!("{}/{}", source.url, name.replace('/', "%2f")),
252 ("cargo", None) => format!("https://crates.io/api/v1/crates/{name}"),
253 ("cargo", Some(source)) => format!("{}/{}", source.url, sparse_path(name)),
254 ("gomod", None) => format!("https://proxy.golang.org/{}/@v/list", go_escape(name)),
255 ("gomod", Some(source)) => format!("{}/{}/@v/list", source.url, go_escape(name)),
256 ("pip", None) => format!("https://pypi.org/pypi/{name}/json"),
257 ("pip", Some(source)) => format!("{}/{name}/", source.url),
258 _ => return None,
259 })
260}
261
262/// Reads a registry's answer for `name` in `ecosystem`.
263pub fn read(ecosystem: &str, name: &str, private: bool, body: &str) -> Package {
264 let json = || serde_json::from_str::<Value>(body).unwrap_or(Value::Null);
265 match (ecosystem, private) {
266 ("npm", _) => npm(&json(), name),
267 ("cargo", false) => crates_io(&json(), name),
268 ("cargo", true) => sparse_index(body),
269 ("gomod", _) => go_list(body, name),
270 ("pip", false) => pypi(&json(), name),
271 ("pip", true) => simple_index(&json()),
272 _ => Package::default(),
273 }
274}
275
276#[cfg(test)]
277mod tests {
278 use super::*;
279 use serde_json::json;
280
281 #[test]
282 fn npm_documents() {
283 let document = json!({
284 "versions": {"4.17.20": {}, "4.17.21": {}, "5.0.0-beta": {"deprecated": "do not use"}},
285 "time": {"4.17.20": "2020-08-13T16:53:54.152Z", "4.17.21": "2021-02-20T15:42:16.891Z"},
286 "repository": {"type": "git", "url": "git+https://github.com/lodash/lodash.git"},
287 });
288 let package = npm(&document, "lodash");
289 assert_eq!(package.releases.len(), 3);
290 let latest = package.releases.iter().find(|r| r.version == "4.17.21").unwrap();
291 assert_eq!(latest.published_ms, Some(parse_rfc3339("2021-02-20T15:42:16.891Z").unwrap()));
292 assert!(package.releases.iter().find(|r| r.version == "5.0.0-beta").unwrap().withdrawn);
293 assert_eq!(package.source.as_deref(), Some("https://github.com/lodash/lodash"));
294 assert_eq!(package.page.as_deref(), Some("https://www.npmjs.com/package/lodash"));
295 assert_eq!(package_url("npm", "@babel/core", None).unwrap(), "https://registry.npmjs.org/@babel%2fcore");
296 }
297
298 #[test]
299 fn crates_documents() {
300 let document = json!({
301 "crate": {"repository": "https://github.com/serde-rs/serde"},
302 "versions": [
303 {"num": "1.0.200", "created_at": "2024-05-01T10:00:00.123456+00:00", "yanked": false},
304 {"num": "1.0.199", "created_at": "2024-04-01T10:00:00+00:00", "yanked": true}
305 ]
306 });
307 let package = crates_io(&document, "serde");
308 assert_eq!(package.releases[0].published_ms, Some(parse_rfc3339("2024-05-01T10:00:00.123Z").unwrap()));
309 assert!(package.releases[1].withdrawn);
310 assert_eq!(package.source.as_deref(), Some("https://github.com/serde-rs/serde"));
311 let sparse = sparse_index("{\"name\":\"x\",\"vers\":\"0.1.0\",\"yanked\":false}\n{\"name\":\"x\",\"vers\":\"0.2.0\",\"yanked\":true,\"pubtime\":\"2025-01-01T00:00:00Z\"}\n");
312 assert_eq!(sparse.releases.len(), 2);
313 assert!(sparse.releases[1].withdrawn && sparse.releases[1].published_ms.is_some());
314 assert_eq!(sparse_path("serde"), "se/rd/serde");
315 assert_eq!(sparse_path("abc"), "3/a/abc");
316 assert_eq!(sparse_path("ab"), "2/ab");
317 assert_eq!(sparse_path("A"), "1/a");
318 }
319
320 #[test]
321 fn go_proxy() {
322 assert_eq!(go_escape("github.com/BurntSushi/toml"), "github.com/!burnt!sushi/toml");
323 let package = go_list("v0.7.0\nv0.23.0\nv2.0.0+incompatible\n\n", "golang.org/x/net");
324 assert_eq!(package.releases.iter().map(|r| r.version.as_str()).collect::<Vec<_>>(), ["v0.7.0", "v0.23.0"]);
325 assert_eq!(go_list("", "github.com/gin-gonic/gin/v2").source.as_deref(), Some("https://github.com/gin-gonic/gin"));
326 assert_eq!(go_info_time(&json!({"Version": "v0.23.0", "Time": "2024-04-04T17:13:08Z"})), parse_rfc3339("2024-04-04T17:13:08Z"));
327 assert_eq!(package_url("gomod", "github.com/BurntSushi/toml", None).unwrap(), "https://proxy.golang.org/github.com/!burnt!sushi/toml/@v/list");
328 }
329
330 #[test]
331 fn pypi_documents() {
332 let document = json!({
333 "info": {"project_urls": {"Changelog": "https://docs.example/changes", "Source": "https://github.com/psf/requests"}},
334 "releases": {
335 "2.31.0": [{"upload_time_iso_8601": "2023-05-22T15:12:44.175893Z", "yanked": false}],
336 "2.30.0": [{"upload_time_iso_8601": "2023-05-03T15:00:00Z", "yanked": true}],
337 "0.0.1": []
338 }
339 });
340 let package = pypi(&document, "requests");
341 assert_eq!(package.releases.len(), 2);
342 assert!(package.releases.iter().find(|r| r.version == "2.30.0").unwrap().withdrawn);
343 assert_eq!(package.changelog.as_deref(), Some("https://docs.example/changes"));
344 assert_eq!(package.source.as_deref(), Some("https://github.com/psf/requests"));
345 let simple = simple_index(&json!({
346 "versions": ["1.0", "1.1"],
347 "files": [
348 {"filename": "pkg-1.0-py3-none-any.whl", "upload-time": "2024-01-01T00:00:00Z"},
349 {"filename": "pkg-1.1.tar.gz", "upload-time": "2024-02-01T00:00:00Z", "yanked": "broken"}
350 ]
351 }));
352 assert_eq!(simple.releases.len(), 2);
353 assert!(!simple.releases[0].withdrawn && simple.releases[1].withdrawn);
354 }
355
356 #[test]
357 fn times_and_urls() {
358 assert_eq!(normalized_time("2024-01-01T02:00:00+02:00"), parse_rfc3339("2024-01-01T00:00:00Z"));
359 assert_eq!(normalized_time("2024-01-01 00:00:00"), parse_rfc3339("2024-01-01T00:00:00Z"));
360 assert_eq!(web_url("git@github.com:a/b.git").as_deref(), Some("https://github.com/a/b"));
361 assert_eq!(web_url("github:a/b").as_deref(), Some("https://github.com/a/b"));
362 assert_eq!(web_url("not a url"), None);
363 }
364}