Sign in

pulls/pr_01m47d15m3e54sn21z27rpy5n9

public

pr_01m47d15m3e54sn21z27rpy5n9/crates/scan/src/lockfiles.rs

484 lines20,551 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API1//! What a project depends on, read from the lockfiles its package managers
2//! write: the exact versions that get installed, which is what an advisory
3//! is about.
4
5use serde_json::Value;
6use std::collections::BTreeSet;
7
8/// A package registry, named as OSV names it.
9#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
10pub enum Ecosystem {
11 Npm,
12 Cargo,
13 Go,
14 PyPI,
15}
16
17impl Ecosystem {
18 pub const ALL: [Ecosystem; 4] = [Ecosystem::Npm, Ecosystem::Cargo, Ecosystem::Go, Ecosystem::PyPI];
19
20 /// OSV's name, which is also what is stored.
21 pub fn osv(self) -> &'static str {
22 match self {
23 Ecosystem::Npm => "npm",
24 Ecosystem::Cargo => "crates.io",
25 Ecosystem::Go => "Go",
26 Ecosystem::PyPI => "PyPI",
27 }
28 }
29
30 pub fn parse(name: &str) -> Option<Ecosystem> {
31 Ecosystem::ALL.into_iter().find(|ecosystem| ecosystem.osv() == name)
32 }
33
34 /// Package names compared as the registry compares them.
35 pub fn normalize(self, name: &str) -> String {
36 match self {
37 Ecosystem::PyPI => name.to_lowercase().replace(['_', '.'], "-"),
38 _ => name.to_owned(),
39 }
40 }
41}
42
43/// The lockfiles g1t reads, by file name.
44#[derive(Clone, Copy, Debug, PartialEq, Eq)]
45pub enum Lockfile {
46 PackageLock,
47 PnpmLock,
48 YarnLock,
49 CargoLock,
50 GoMod,
51 GoSum,
52 Requirements,
53 PoetryLock,
54}
55
56impl Lockfile {
57 pub const NAMES: [&'static str; 8] = [
58 "package-lock.json",
59 "pnpm-lock.yaml",
60 "yarn.lock",
61 "Cargo.lock",
62 "go.mod",
63 "go.sum",
64 "requirements.txt",
65 "poetry.lock",
66 ];
67
68 pub fn for_path(path: &str) -> Option<Lockfile> {
69 Some(match path.rsplit('/').next().unwrap_or(path) {
70 "package-lock.json" => Lockfile::PackageLock,
71 "pnpm-lock.yaml" => Lockfile::PnpmLock,
72 "yarn.lock" => Lockfile::YarnLock,
73 "Cargo.lock" => Lockfile::CargoLock,
74 "go.mod" => Lockfile::GoMod,
75 "go.sum" => Lockfile::GoSum,
76 "requirements.txt" => Lockfile::Requirements,
77 "poetry.lock" => Lockfile::PoetryLock,
78 _ => return None,
79 })
80 }
81
82 pub fn ecosystem(self) -> Ecosystem {
83 match self {
84 Lockfile::PackageLock | Lockfile::PnpmLock | Lockfile::YarnLock => Ecosystem::Npm,
85 Lockfile::CargoLock => Ecosystem::Cargo,
86 Lockfile::GoMod | Lockfile::GoSum => Ecosystem::Go,
87 Lockfile::Requirements | Lockfile::PoetryLock => Ecosystem::PyPI,
88 }
89 }
90
91 pub fn parse(self, text: &str) -> Vec<Package> {
92 let found = match self {
93 Lockfile::PackageLock => package_lock(text),
94 Lockfile::PnpmLock => pnpm_lock(text),
95 Lockfile::YarnLock => yarn_lock(text),
96 Lockfile::CargoLock => toml_packages(text, true),
97 Lockfile::GoMod => go_mod(text),
98 Lockfile::GoSum => go_sum(text),
99 Lockfile::Requirements => requirements(text),
100 Lockfile::PoetryLock => toml_packages(text, false),
101 };
102 let ecosystem = self.ecosystem();
103 let unique: BTreeSet<(String, String)> = found
104 .into_iter()
105 .filter(|(name, version)| !name.is_empty() && version.starts_with(|c: char| c.is_ascii_digit() || c == 'v'))
106 .map(|(name, version)| (ecosystem.normalize(&name), version))
107 .collect();
108 unique
109 .into_iter()
110 .map(|(name, version)| Package { ecosystem, name, version })
111 .collect()
112 }
113}
114
115/// One package at one version.
116#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
117pub struct Package {
118 pub ecosystem: Ecosystem,
119 pub name: String,
120 pub version: String,
121}
122
123/// `package-lock.json` (and `npm-shrinkwrap.json`): lockfile v2 and v3 list
124/// every installed path under `packages`; v1 nests `dependencies`.
125fn package_lock(text: &str) -> Vec<(String, String)> {
126 let Ok(lock) = serde_json::from_str::<Value>(text) else {
127 return Vec::new();
128 };
129 let mut found = Vec::new();
130 if let Some(packages) = lock.get("packages").and_then(Value::as_object) {
131 for (path, entry) in packages {
132 let Some(at) = path.rfind("node_modules/") else {
133 continue; // the project itself, or one of its workspaces
134 };
135 if entry.get("link").and_then(Value::as_bool) == Some(true) {
136 continue;
137 }
138 let name = entry
139 .get("name")
140 .and_then(Value::as_str)
141 .unwrap_or(&path[at + "node_modules/".len()..]);
142 if let Some(version) = entry.get("version").and_then(Value::as_str) {
143 found.push((name.to_owned(), version.to_owned()));
144 }
145 }
146 return found;
147 }
148 fn walk(dependencies: &Value, found: &mut Vec<(String, String)>) {
149 let Some(dependencies) = dependencies.as_object() else {
150 return;
151 };
152 for (name, entry) in dependencies {
153 if let Some(version) = entry.get("version").and_then(Value::as_str) {
154 found.push((name.clone(), version.to_owned()));
155 }
156 if let Some(nested) = entry.get("dependencies") {
157 walk(nested, found);
158 }
159 }
160 }
161 if let Some(dependencies) = lock.get("dependencies") {
162 walk(dependencies, &mut found);
163 }
164 found
165}
166
167/// `name@version` or `name/version`, as pnpm writes a package's key, with
168/// the peer-dependency suffix that follows removed.
169fn pnpm_key(key: &str) -> Option<(String, String)> {
170 let key = key.trim().trim_end_matches(':').trim_matches(['\'', '"']);
171 let key = key.strip_prefix('/').unwrap_or(key);
172 let key = key.split('(').next().unwrap_or(key);
173 // A scoped name has one slash in it, any other none.
174 let named = |name: &str| !name.is_empty() && name.matches('/').count() == usize::from(name.starts_with('@'));
175 // `@scope/name@1.0.0`: the version follows the last `@` after the first character.
176 if let Some(at) = key.get(1..).and_then(|rest| rest.rfind('@')).map(|at| at + 1) {
177 let (name, version) = (&key[..at], &key[at + 1..]);
178 if named(name) {
179 return Some((name.to_owned(), version.split('_').next().unwrap_or_default().to_owned()));
180 }
181 }
182 // Lockfile v5: `/name/1.0.0_peer@2.0.0` or `/@scope/name/1.0.0`.
183 let key = key.split('_').next().unwrap_or(key);
184 let (name, version) = key.rsplit_once('/')?;
185 named(name).then(|| (name.to_owned(), version.to_owned()))
186}
187
188/// `pnpm-lock.yaml`: each package is a two-space-indented key under the
189/// top-level `packages:`.
190fn pnpm_lock(text: &str) -> Vec<(String, String)> {
191 let mut found = Vec::new();
192 let mut inside = false;
193 for line in text.lines() {
194 if !line.starts_with(' ') && !line.trim().is_empty() {
195 inside = line.trim_end() == "packages:";
196 continue;
197 }
198 if !inside || !line.starts_with(" ") || line.starts_with(" ") || !line.trim_end().ends_with(':') {
199 continue;
200 }
201 if let Some(package) = pnpm_key(line) {
202 found.push(package);
203 }
204 }
205 found
206}
207
208/// `yarn.lock`, classic and Berry: a header naming the package and the
209/// ranges it satisfies, then an indented `version`.
210fn yarn_lock(text: &str) -> Vec<(String, String)> {
211 let mut found = Vec::new();
212 let mut name: Option<String> = None;
213 for line in text.lines() {
214 if line.starts_with('#') || line.trim().is_empty() {
215 continue;
216 }
217 if !line.starts_with(' ') {
218 name = None;
219 let first = line.trim_end_matches(':').split(',').next().unwrap_or_default();
220 let spec = first.trim().trim_matches('"');
221 if spec.contains("@workspace:") || spec.contains("@patch:") || spec.contains("@link:") {
222 continue;
223 }
224 if let Some(at) = spec[1.min(spec.len())..].find('@').map(|at| at + 1) {
225 name = Some(spec[..at].to_owned());
226 }
227 continue;
228 }
229 let trimmed = line.trim();
230 if let (Some(current), Some(rest)) = (&name, trimmed.strip_prefix("version")) {
231 let version = rest.trim_start_matches(':').trim().trim_matches('"');
232 found.push((current.clone(), version.to_owned()));
233 name = None;
234 }
235 }
236 found
237}
238
239/// `Cargo.lock` and `poetry.lock`: `[[package]]` tables with `name` and
240/// `version`. For Cargo only crates from a registry count; the project's
241/// own crates have no `source`.
242fn toml_packages(text: &str, registry_only: bool) -> Vec<(String, String)> {
243 let mut found = Vec::new();
244 let mut current: Option<(Option<String>, Option<String>, bool)> = None;
245 let mut finish = |current: &mut Option<(Option<String>, Option<String>, bool)>| {
246 if let Some((Some(name), Some(version), from_registry)) = current.take()
247 && (!registry_only || from_registry)
248 {
249 found.push((name, version));
250 }
251 };
252 for line in text.lines() {
253 let line = line.trim();
254 if line.starts_with('[') {
255 finish(&mut current);
256 if line == "[[package]]" {
257 current = Some((None, None, false));
258 }
259 continue;
260 }
261 let Some(entry) = current.as_mut() else {
262 continue;
263 };
264 let Some((key, value)) = line.split_once('=') else {
265 continue;
266 };
267 let value = value.trim().trim_matches('"').to_owned();
268 match key.trim() {
269 "name" => entry.0 = Some(value),
270 "version" => entry.1 = Some(value),
271 "source" => entry.2 = value.starts_with("registry+") || value.starts_with("sparse+"),
272 _ => {}
273 }
274 }
275 finish(&mut current);
276 found
277}
278
279/// `go.mod`'s `require` lines, in a block or one at a time: the versions
280/// the build actually uses.
281fn go_mod(text: &str) -> Vec<(String, String)> {
282 let mut found = Vec::new();
283 let mut block = false;
284 for line in text.lines() {
285 let line = line.split("//").next().unwrap_or_default().trim();
286 if block {
287 if line == ")" {
288 block = false;
289 continue;
290 }
291 } else if line.starts_with("require (") || line == "require(" {
292 block = true;
293 continue;
294 }
295 let line = if block { line } else if let Some(rest) = line.strip_prefix("require ") { rest.trim() } else { continue };
296 let mut words = line.split_whitespace();
297 if let (Some(module), Some(version)) = (words.next(), words.next()) {
298 found.push((module.to_owned(), version.to_owned()));
299 }
300 }
301 found
302}
303
304/// `go.sum` lists every version a build ever considered; the highest of
305/// each module is the one in use.
306fn go_sum(text: &str) -> Vec<(String, String)> {
307 let mut highest: std::collections::BTreeMap<String, String> = Default::default();
308 for line in text.lines() {
309 let mut words = line.split_whitespace();
310 let (Some(module), Some(version)) = (words.next(), words.next()) else {
311 continue;
312 };
313 let version = version.trim_end_matches("/go.mod");
314 let keep = highest
315 .get(module)
316 .is_none_or(|current| crate::version::compare(version, current).is_gt());
317 if keep {
318 highest.insert(module.to_owned(), version.to_owned());
319 }
320 }
321 highest.into_iter().collect()
322}
323
324/// `requirements.txt`: only pinned lines, `name==1.2.3`, say what is
325/// installed.
326fn requirements(text: &str) -> Vec<(String, String)> {
327 let mut found = Vec::new();
328 for line in text.lines() {
329 let line = line.split('#').next().unwrap_or_default();
330 let line = line.split(';').next().unwrap_or_default().trim();
331 if line.starts_with('-') || line.contains("://") {
332 continue;
333 }
334 let Some((name, version)) = line.split_once("===").or_else(|| line.split_once("==")) else {
335 continue;
336 };
337 let name = name.split('[').next().unwrap_or_default().trim();
338 let version = version.split([',', ' ']).next().unwrap_or_default().trim();
339 if !name.is_empty() && !version.contains('*') {
340 found.push((name.to_owned(), version.to_owned()));
341 }
342 }
343 found
344}
345
346/// A shell command that fails while `lockfile` still resolves `name` at
347/// `version`: an acceptance check for an upgrade, which passes only once
348/// the vulnerable version is gone from the lockfile.
349pub fn still_locked_check(lockfile: Lockfile, path: &str, name: &str, version: &str) -> String {
350 let quote = |text: &str| format!("'{}'", text.replace('\'', "'\\''"));
351 let escape = |text: &str| {
352 text.chars()
353 .flat_map(|c| if ".[]^$*+?(){}|\\".contains(c) { vec!['\\', c] } else { vec![c] })
354 .collect::<String>()
355 };
356 let file = quote(path);
357 match lockfile {
358 Lockfile::PackageLock => format!(
359 "node -e {} {file}",
360 quote(&format!(
361 "const l=require(require('path').resolve(process.argv[1]));const hit=Object.entries(l.packages||{{}}).some(([k,p])=>k.endsWith('node_modules/{name}')&&p.version==='{version}');process.exit(hit?1:0)"
362 ))
363 ),
364 Lockfile::PnpmLock => format!(
365 "! grep -Eq {} {file}",
366 quote(&format!("^ +'?/?{}[@/]{}[:(_']", escape(name), escape(version)))
367 ),
368 Lockfile::YarnLock => format!(
369 "! grep -A3 -E {} {file} | grep -Eq {}",
370 quote(&format!("^\"?{}@", escape(name))),
371 quote(&format!("^ +version:? \"?{}\"?$", escape(version)))
372 ),
373 Lockfile::CargoLock | Lockfile::PoetryLock => format!(
374 "! grep -A1 -x {} {file} | grep -qx {}",
375 quote(&format!("name = \"{name}\"")),
376 quote(&format!("version = \"{version}\""))
377 ),
378 Lockfile::GoMod | Lockfile::GoSum => format!(
379 "! grep -Eq {} {file}",
380 quote(&format!("(^|[[:space:]]){} {}([[:space:]]|/|$)", escape(name), escape(version)))
381 ),
382 Lockfile::Requirements => format!(
383 "! grep -Eiq {} {file}",
384 quote(&format!("^{}(\\[.*\\])? *===? *{}([^0-9.]|$)", escape(name).replace('-', "[-_.]"), escape(version)))
385 ),
386 }
387}
388
389/// The command that runs a project's tests, by what its lockfile says it is.
390pub fn test_command(lockfile: Lockfile, directory: &str) -> Option<String> {
391 let cd = if directory.is_empty() { String::new() } else { format!("cd '{directory}' && ") };
392 Some(match lockfile {
393 Lockfile::PackageLock => format!("{cd}npm ci && npm test --if-present"),
394 Lockfile::PnpmLock => format!("{cd}pnpm install --frozen-lockfile && pnpm test --if-present"),
395 Lockfile::YarnLock => format!("{cd}yarn install --immutable || yarn install --frozen-lockfile; yarn test"),
396 Lockfile::CargoLock => format!("{cd}cargo test --locked"),
397 Lockfile::GoMod | Lockfile::GoSum => format!("{cd}go test ./..."),
398 Lockfile::Requirements | Lockfile::PoetryLock => return None,
399 })
400}
401
402#[cfg(test)]
403mod tests {
404 use super::*;
405
406 fn names(lockfile: Lockfile, text: &str) -> Vec<String> {
407 lockfile
408 .parse(text)
409 .into_iter()
410 .map(|package| format!("{}@{}", package.name, package.version))
411 .collect()
412 }
413
414 #[test]
415 fn package_lock_v3_and_v1() {
416 let v3 = r#"{"lockfileVersion":3,"packages":{
417 "":{"name":"app","version":"1.0.0"},
418 "node_modules/lodash":{"version":"4.17.20"},
419 "node_modules/@babel/core":{"version":"7.0.0"},
420 "node_modules/a/node_modules/lodash":{"version":"4.17.4"},
421 "packages/web":{"version":"0.1.0"},
422 "node_modules/web":{"resolved":"packages/web","link":true}
423 }}"#;
424 assert_eq!(names(Lockfile::PackageLock, v3), ["@babel/core@7.0.0", "lodash@4.17.20", "lodash@4.17.4"]);
425 let v1 = r#"{"lockfileVersion":1,"dependencies":{"minimist":{"version":"0.0.8","dependencies":{"x":{"version":"1.0.0"}}}}}"#;
426 assert_eq!(names(Lockfile::PackageLock, v1), ["minimist@0.0.8", "x@1.0.0"]);
427 }
428
429 #[test]
430 fn pnpm_lock_v5_v6_and_v9() {
431 let v5 = "lockfileVersion: 5.4\npackages:\n /lodash/4.17.20:\n resolution: {integrity: x}\n /@babel/core/7.0.0_react@18.0.0:\n dev: true\n";
432 assert_eq!(names(Lockfile::PnpmLock, v5), ["@babel/core@7.0.0", "lodash@4.17.20"]);
433 let v6 = "lockfileVersion: '6.0'\npackages:\n /lodash@4.17.20:\n resolution: {}\n /@types/node@20.1.0(typescript@5.0.0):\n dev: true\n";
434 assert_eq!(names(Lockfile::PnpmLock, v6), ["@types/node@20.1.0", "lodash@4.17.20"]);
435 let v9 = "lockfileVersion: '9.0'\nimporters:\n .:\n dependencies: {}\npackages:\n lodash@4.17.20:\n resolution: {}\n '@babel/core@7.24.0':\n resolution: {}\nsnapshots:\n lodash@4.17.20: {}\n";
436 assert_eq!(names(Lockfile::PnpmLock, v9), ["@babel/core@7.24.0", "lodash@4.17.20"]);
437 }
438
439 #[test]
440 fn yarn_classic_and_berry() {
441 let classic = "# yarn lockfile v1\n\nlodash@^4.17.0, lodash@^4.17.15:\n version \"4.17.20\"\n resolved \"x\"\n\n\"@babel/core@^7.0.0\":\n version \"7.1.0\"\n";
442 assert_eq!(names(Lockfile::YarnLock, classic), ["@babel/core@7.1.0", "lodash@4.17.20"]);
443 let berry = "__metadata:\n version: 6\n\n\"lodash@npm:^4.17.0\":\n version: 4.17.20\n resolution: \"lodash@npm:4.17.20\"\n\n\"app@workspace:.\":\n version: 0.0.0-use.local\n";
444 assert_eq!(names(Lockfile::YarnLock, berry), ["lodash@4.17.20"]);
445 }
446
447 #[test]
448 fn cargo_lock_counts_registry_crates_only() {
449 let lock = "version = 3\n\n[[package]]\nname = \"app\"\nversion = \"0.1.0\"\n\n[[package]]\nname = \"time\"\nversion = \"0.1.43\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"x\"\n\n[[package]]\nname = \"smallvec\"\nversion = \"1.6.0\"\nsource = \"sparse+https://index.crates.io/\"\n";
450 assert_eq!(names(Lockfile::CargoLock, lock), ["smallvec@1.6.0", "time@0.1.43"]);
451 }
452
453 #[test]
454 fn go_mod_and_go_sum() {
455 let module = "module example.com/app\n\ngo 1.22\n\nrequire golang.org/x/text v0.3.0\n\nrequire (\n\tgithub.com/gin-gonic/gin v1.6.0 // indirect\n\tgolang.org/x/net v0.7.0\n)\n";
456 assert_eq!(
457 names(Lockfile::GoMod, module),
458 ["github.com/gin-gonic/gin@v1.6.0", "golang.org/x/net@v0.7.0", "golang.org/x/text@v0.3.0"]
459 );
460 let sum = "golang.org/x/text v0.3.0 h1:x=\ngolang.org/x/text v0.3.0/go.mod h1:y=\ngolang.org/x/text v0.3.8 h1:z=\n";
461 assert_eq!(names(Lockfile::GoSum, sum), ["golang.org/x/text@v0.3.8"]);
462 }
463
464 #[test]
465 fn requirements_and_poetry() {
466 let requirements = "# web\nDjango==3.2.0\nrequests[security]==2.19.1 ; python_version >= '3'\nflask>=2.0\n-r other.txt\nPyYAML===5.3\n";
467 assert_eq!(names(Lockfile::Requirements, requirements), ["django@3.2.0", "pyyaml@5.3", "requests@2.19.1"]);
468 let poetry = "[[package]]\nname = \"Jinja2\"\nversion = \"2.10\"\ndescription = \"x\"\n\n[package.dependencies]\nMarkupSafe = \">=0.23\"\n\n[[package]]\nname = \"urllib3\"\nversion = \"1.24.1\"\n\n[metadata]\nlock-version = \"2.0\"\n";
469 assert_eq!(names(Lockfile::PoetryLock, poetry), ["jinja2@2.10", "urllib3@1.24.1"]);
470 }
471
472 #[test]
473 fn the_check_names_the_file_and_the_version() {
474 let check = still_locked_check(Lockfile::CargoLock, "Cargo.lock", "time", "0.1.43");
475 assert_eq!(check, "! grep -A1 -x 'name = \"time\"' 'Cargo.lock' | grep -qx 'version = \"0.1.43\"'");
476 let npm = still_locked_check(Lockfile::PackageLock, "web/package-lock.json", "lodash", "4.17.20");
477 assert!(npm.starts_with("node -e '") && npm.ends_with(" 'web/package-lock.json'"));
478 assert!(npm.contains("node_modules/lodash") && npm.contains("4.17.20"));
479 let go = still_locked_check(Lockfile::GoMod, "go.mod", "golang.org/x/net", "v0.7.0");
480 assert!(go.contains("golang\\.org/x/net v0\\.7\\.0"));
481 assert_eq!(test_command(Lockfile::CargoLock, ""), Some("cargo test --locked".to_owned()));
482 assert_eq!(test_command(Lockfile::PackageLock, "web").as_deref(), Some("cd 'web' && npm ci && npm test --if-present"));
483 }
484}