g1t/services/security/src/deps.rs

439 lines19,930 bytesCodeBlame
1//! Dependencies: reading a repository's lockfiles, asking OSV about every
2//! package in them, and recording what is vulnerable. The security updates
3//! that fix them are `security_updates`'s.
4
5use std::collections::{BTreeSet, HashMap};
6
7use g1t_contracts::repos::RepoPath;
8use g1t_contracts::repos::{BlobArgs, BlobView};
9use g1t_contracts::security::{AlertState, FindLockfilesArgs, Lockfiles, VersionUpdatesState, VulnStatus, Vulnerability};
10use g1t_contracts::time::rfc3339;
11use g1t_contracts::work::{AddCommentArgs, Issue, IssueDetail, ViewArgs};
12use g1t_contracts::{Outcome, User};
13use g1t_kit::now_ms;
14use g1t_scan::lockfiles::{Lockfile, Package, still_locked_check, test_command};
15use g1t_scan::osv::{self, Advisory};
16use serde_json::{Value, json};
17use worker::{Fetch, Headers, Method, Request, RequestInit, Result};
18
19use crate::Security;
20use crate::store::{RepoRow, VulnRow};
21use crate::updates;
22
23/// OSV's records are fetched again after this long.
24const ADVISORY_MAX_AGE_MS: u64 = 7 * 24 * 60 * 60 * 1000;
25/// A record that names no fixed version is fetched again after a day, so
26/// a fix is noticed the day it is published.
27const UNFIXED_ADVISORY_MAX_AGE_MS: u64 = 24 * 60 * 60 * 1000;
28/// Records fetched per scan, at most; the rest wait for the next one.
29const MAX_ADVISORY_FETCHES: usize = 150;
30/// CPU one call to OSV takes, sending it and reading its answer, in
31/// milliseconds (an estimate, rounded up). OSV itself is free, and a
32/// Worker's outgoing requests are not charged.
33const CPU_MS_PER_OSV_CALL: f64 = 2.0;
34
35/// What a dependency check cost g1t, in millionths of a dollar, rounded
36/// up, at the prices in `history`: the CPU of its OSV calls, and the rows
37/// it writes (an advisory kept for each call at most, each vulnerability
38/// found, where the check stands and the month's usage).
39pub fn dependency_check_cost(calls: u32, found: usize) -> i64 {
40 use crate::history::{MICROS_PER_CPU_MS, MICROS_PER_ROW_WRITTEN};
41 let cpu = f64::from(calls) * CPU_MS_PER_OSV_CALL * MICROS_PER_CPU_MS;
42 let rows = (calls as usize + found + 2) as f64 * MICROS_PER_ROW_WRITTEN;
43 (cpu + rows).ceil() as i64
44}
45
46async fn osv_call(method: Method, url: &str, body: Option<&Value>) -> Result<Option<Value>> {
47 let headers = Headers::new();
48 headers.set("user-agent", "g1t (+https://g1t.sh)")?;
49 headers.set("accept", "application/json")?;
50 if body.is_some() {
51 headers.set("content-type", "application/json")?;
52 }
53 let mut init = RequestInit::new();
54 init.with_method(method).with_headers(headers);
55 if let Some(body) = body {
56 init.with_body(Some(body.to_string().into()));
57 }
58 let mut response = Fetch::Request(Request::new_with_init(url, &init)?).send().await?;
59 if response.status_code() == 404 {
60 return Ok(None);
61 }
62 if !(200..300).contains(&response.status_code()) {
63 return Err(worker::Error::RustError(format!("OSV answered {}", response.status_code())));
64 }
65 Ok(Some(response.json().await?))
66}
67
68/// Whether an OSV record names a version that fixes it, for any package.
69fn names_a_fix(record: &Value) -> bool {
70 record["affected"].as_array().into_iter().flatten().any(|affected| {
71 affected["ranges"]
72 .as_array()
73 .into_iter()
74 .flatten()
75 .any(|range| range["events"].as_array().into_iter().flatten().any(|event| event.get("fixed").is_some()))
76 })
77}
78
79/// One package in one lockfile.
80pub(crate) struct Located {
81 pub(crate) package: Package,
82 pub(crate) lockfile: Lockfile,
83 pub(crate) path: String,
84}
85
86/// Every package the lockfiles resolve. A directory with a `go.mod` is read
87/// from it rather than from its `go.sum`, which lists versions not built.
88fn packages(files: &Lockfiles) -> Vec<Located> {
89 let go_mods: BTreeSet<&str> = files
90 .files
91 .iter()
92 .filter(|file| file.path.ends_with("go.mod"))
93 .map(|file| file.path.trim_end_matches("go.mod"))
94 .collect();
95 let mut located = Vec::new();
96 for file in &files.files {
97 let Some(lockfile) = Lockfile::for_path(&file.path) else { continue };
98 if lockfile == Lockfile::GoSum && go_mods.contains(file.path.trim_end_matches("go.sum")) {
99 continue;
100 }
101 for package in lockfile.parse(&file.text) {
102 located.push(Located { package, lockfile, path: file.path.clone() });
103 }
104 }
105 located
106}
107
108fn directory(path: &str) -> &str {
109 path.rsplit_once('/').map_or("", |(directory, _)| directory)
110}
111
112impl Security {
113 /// The ids of the vulnerabilities affecting each package, from OSV.
114 async fn query_osv(&self, packages: &[Package]) -> Result<(Vec<Vec<String>>, u32)> {
115 let mut ids = Vec::with_capacity(packages.len());
116 let mut calls = 0;
117 for (body, chunk) in osv::batch_bodies(packages).iter().zip(packages.chunks(osv::MAX_BATCH)) {
118 calls += 1;
119 let answer = osv_call(Method::Post, osv::QUERY_BATCH_URL, Some(body)).await?.unwrap_or(Value::Null);
120 let (mut found, more) = osv::read_batch(&answer, chunk.len());
121 // A package with many advisories is paged; fetch the rest.
122 for (index, token) in more.into_iter().take(20) {
123 let package = &chunk[index];
124 let query = json!({
125 "package": {"name": package.name, "ecosystem": package.ecosystem.osv()},
126 "version": package.version,
127 "page_token": token,
128 });
129 calls += 1;
130 if let Some(page) = osv_call(Method::Post, "https://api.osv.dev/v1/query", Some(&query)).await? {
131 found[index].extend(
132 page["vulns"].as_array().into_iter().flatten().filter_map(|v| v["id"].as_str().map(str::to_owned)),
133 );
134 }
135 }
136 ids.extend(found);
137 }
138 Ok((ids, calls))
139 }
140
141 /// OSV's record of each id, from the cache when it is fresh.
142 async fn advisories(&self, ids: &BTreeSet<String>) -> Result<(HashMap<String, Value>, u32)> {
143 let fresh_after = rfc3339(now_ms().saturating_sub(ADVISORY_MAX_AGE_MS));
144 let unfixed_fresh_after = rfc3339(now_ms().saturating_sub(UNFIXED_ADVISORY_MAX_AGE_MS));
145 let mut records = HashMap::new();
146 let mut fetched = 0u32;
147 for id in ids {
148 if let Some(record) = self.store.advisory(id, &fresh_after).await? {
149 // One without a fix is asked about again daily; until then,
150 // or if asking fails below, the kept record stands.
151 let fixed = names_a_fix(&record);
152 let recent = !fixed && self.store.advisory(id, &unfixed_fresh_after).await?.is_some();
153 if fixed || recent || fetched as usize >= MAX_ADVISORY_FETCHES {
154 records.insert(id.clone(), record);
155 continue;
156 }
157 fetched += 1;
158 let refreshed = osv_call(Method::Get, &osv::vuln_url(id), None).await.ok().flatten();
159 if let Some(refreshed) = &refreshed {
160 self.store.keep_advisory(id, refreshed).await?;
161 }
162 records.insert(id.clone(), refreshed.unwrap_or(record));
163 continue;
164 }
165 if fetched as usize >= MAX_ADVISORY_FETCHES {
166 continue;
167 }
168 fetched += 1;
169 if let Some(record) = osv_call(Method::Get, &osv::vuln_url(id), None).await? {
170 self.store.keep_advisory(id, &record).await?;
171 records.insert(id.clone(), record);
172 }
173 }
174 Ok((records, fetched))
175 }
176
177 /// Reads a repository's dependencies, records which are vulnerable,
178 /// and starts security updates for those with a fix. Also reads
179 /// `.g1t/dependencies.yml`. Returns what went wrong, for the Security
180 /// page, if anything did.
181 pub async fn scan_dependencies(&self, repo: &RepoRow) -> Result<Option<String>> {
182 let files: Lockfiles = g1t_kit::call(&self.repos, "find_lockfiles", &FindLockfilesArgs { repo_id: repo.repo_id.clone() }).await?;
183 let paths: Vec<String> = files.files.iter().map(|file| file.path.clone()).collect();
184 let located = packages(&files);
185 let unique: Vec<Package> = located.iter().map(|l| l.package.clone()).collect::<BTreeSet<_>>().into_iter().collect();
186 let outcome = async {
187 let (ids, calls) = self.query_osv(&unique).await?;
188 let by_package: HashMap<&Package, &Vec<String>> = unique.iter().zip(ids.iter()).collect();
189 let wanted: BTreeSet<String> = ids.iter().flatten().cloned().collect();
190 let (records, fetched) = self.advisories(&wanted).await?;
191 let mut found = Vec::new();
192 for item in &located {
193 for id in by_package.get(&item.package).into_iter().flat_map(|ids| ids.iter()) {
194 let Some(advisory) = records.get(id).and_then(|record| osv::read_vuln(record, &item.package)) else {
195 continue;
196 };
197 found.push(vulnerability(&repo.repo_id, item, &advisory));
198 }
199 }
200 Ok::<_, worker::Error>((found, calls + fetched))
201 }
202 .await;
203 let (found, calls) = match outcome {
204 Ok(result) => result,
205 Err(error) => {
206 let problem = format!("The dependencies could not be checked: {error}");
207 self.store
208 .set_dependencies_scanned(&repo.repo_id, files.commit.as_deref(), &paths, Some(&problem))
209 .await?;
210 return Ok(Some(problem));
211 }
212 };
213 self.store.replace_vulnerabilities(&repo.repo_id, &found).await?;
214 self.store.set_dependencies_scanned(&repo.repo_id, files.commit.as_deref(), &paths, None).await?;
215 self.meter(&repo.namespace, 0, 0, calls, dependency_check_cost(calls, found.len())).await?;
216 if let Some(commit) = files.commit.as_deref() {
217 self.read_version_updates(repo, commit).await?;
218 }
219 // No security updates on an archived (read-only) or deleted
220 // repository; ones in flight for packages no longer vulnerable are
221 // closed either way.
222 let active = self.active(&repo.repo_id).await?;
223 self.security_updates(repo, repo.upkeep != 0 && active).await?;
224 Ok(None)
225 }
226
227 /// Reads `.g1t/dependencies.yml` at `commit` and keeps what it says.
228 async fn read_version_updates(&self, repo: &RepoRow, commit: &str) -> Result<()> {
229 let found: Outcome<BlobView> = g1t_kit::call(
230 &self.repos,
231 "blob",
232 &BlobArgs {
233 path: RepoPath { namespace: repo.namespace.clone(), name: repo.name.clone() },
234 viewer: Some(User::system(&repo.namespace)),
235 git_ref: commit.to_owned(),
236 file_path: updates::PATH.to_owned(),
237 },
238 )
239 .await?;
240 let mut state = VersionUpdatesState { read_at: Some(crate::store::now()), ..VersionUpdatesState::default() };
241 if let Outcome::Ok(blob) = found {
242 state.found = true;
243 match blob.text.as_deref().map(updates::parse) {
244 Some(Ok(entries)) => state.updates = entries,
245 Some(Err(error)) => state.error = Some(error),
246 None => state.error = Some(format!("{} is too large or not text.", updates::PATH)),
247 }
248 }
249 self.store.set_version_updates(&repo.repo_id, &state).await
250 }
251
252 pub(crate) async fn issue(&self, actor: &User, repo: &RepoPath, number: u32) -> Result<Option<Issue>> {
253 let found: Outcome<IssueDetail> = g1t_kit::call(
254 &self.work,
255 "get_issue",
256 &ViewArgs { repo: repo.clone(), number, viewer: Some(actor.clone()), after_seq: 0 },
257 )
258 .await?;
259 Ok(found.into_result().ok().map(|detail| detail.issue))
260 }
261
262 pub(crate) async fn comment(&self, actor: &User, repo: &RepoPath, number: u32, body: String) -> Result<()> {
263 let _: Outcome<Value> = g1t_kit::call(
264 &self.work,
265 "add_comment",
266 &AddCommentArgs {
267 actor: actor.clone(),
268 repo: repo.clone(),
269 number,
270 body,
271 path: None,
272 line: None,
273 verdict: None,
274 },
275 )
276 .await?;
277 Ok(())
278 }
279}
280
281fn vulnerability(repo_id: &str, item: &Located, advisory: &Advisory) -> Vulnerability {
282 Vulnerability {
283 id: String::new(),
284 repo_id: repo_id.to_owned(),
285 ecosystem: item.package.ecosystem.osv().to_owned(),
286 package: item.package.name.clone(),
287 version: item.package.version.clone(),
288 manifest: item.path.clone(),
289 advisory: advisory.display_id.clone(),
290 osv_id: advisory.id.clone(),
291 summary: advisory.summary.clone(),
292 severity: advisory.severity.as_str().to_owned(),
293 fixed_version: advisory.fixed.clone(),
294 status: VulnStatus::Open,
295 issue: None,
296 found_at: String::new(),
297 fixed_at: None,
298 state: AlertState::Open,
299 dismissed_by: None,
300 dismissed_reason: None,
301 dismissed_comment: None,
302 dismissed_at: None,
303 update: None,
304 }
305}
306
307/// The issue's body, for when raising the version is not enough, written
308/// for the agent that takes it as much as for a person, ending with what
309/// done means: commands that show no lockfile still resolves a vulnerable
310/// version, and the project's tests. The pull request merges on the
311/// repository's required checks, like any other.
312pub(crate) fn issue_text(ecosystem: &str, package: &str, target: &str, vulns: &[&VulnRow], located: &[Located]) -> String {
313 let mut body = format!(
314 "`{package}` ({ecosystem}) has known vulnerabilities with a fix in **{target}**, and raising its version alone \
315 does not pass this project's checks. Upgrade it to {target} or later everywhere it is locked, and change the \
316 code that depends on it, keeping other changes to what the upgrade needs.\n\n"
317 );
318 body.push_str(&advisory_table(vulns));
319 let mut checks = Vec::new();
320 let mut manifests = BTreeSet::new();
321 let mut tests = BTreeSet::new();
322 for vuln in vulns {
323 let Some(item) = located
324 .iter()
325 .find(|item| item.path == vuln.manifest && item.package.name == vuln.package && item.package.version == vuln.version)
326 else {
327 continue;
328 };
329 if manifests.insert((item.path.clone(), item.package.version.clone())) {
330 checks.push(still_locked_check(item.lockfile, &item.path, &item.package.name, &item.package.version));
331 }
332 if let Some(test) = test_command(item.lockfile, directory(&item.path)) {
333 tests.insert(test);
334 }
335 }
336 let locked: Vec<String> = manifests.iter().map(|(path, version)| format!("`{path}` ({version})")).collect();
337 body.push_str(&format!("\nLocked in: {}.\n", locked.join(", ")));
338 body.push_str(
339 "\nIf the fix needs a major upgrade that breaks the build, change the code that depends on it in the same pull request.",
340 );
341 checks.extend(tests);
342 let mut done = vec!["No lockfile resolves a vulnerable version, and the tests still pass.".to_owned()];
343 done.extend(g1t_contracts::work::commands_pass(&checks));
344 let mut body = g1t_contracts::work::with_definition_of_done(&body, &done);
345 body.push_str("\n\n---\n_Opened by g1t's security updates. Turn them off for this project on its Security page._");
346 body
347}
348
349/// The advisories a package's vulnerabilities name, as a table.
350pub(crate) fn advisory_table(vulns: &[&VulnRow]) -> String {
351 let mut table = "| Advisory | Severity | Affected | Fixed in | Summary |\n| --- | --- | --- | --- | --- |\n".to_owned();
352 let mut seen = BTreeSet::new();
353 for vuln in vulns {
354 if !seen.insert((vuln.advisory.clone(), vuln.version.clone())) {
355 continue;
356 }
357 table.push_str(&format!(
358 "| [{}]({}) | {} | {} | {} | {} |\n",
359 vuln.advisory,
360 osv::page_url(&vuln.osv_id),
361 vuln.severity,
362 vuln.version,
363 vuln.fixed_version.as_deref().unwrap_or("none yet"),
364 vuln.summary.replace('|', "\\|").replace('\n', " "),
365 ));
366 }
367 table
368}
369
370#[cfg(test)]
371mod tests {
372 use super::*;
373 use g1t_contracts::security::LockfileText;
374
375 #[test]
376 fn scans_cost_their_cpu_and_the_rows_they_write() {
377 // 10 OSV calls and 3 vulnerabilities: 0.4 of CPU, 15 rows.
378 assert_eq!(dependency_check_cost(10, 3), 16);
379 assert_eq!(dependency_check_cost(0, 0), 2);
380 // A page of 25 commits that read 100 objects and found nothing:
381 // 10 of CPU and 2 rows. The old placeholder charged 100.
382 assert_eq!(crate::history::history_page_cost(100, 0), 12);
383 assert_eq!(crate::history::history_page_cost(0, 1), 3);
384 }
385
386 #[test]
387 fn go_sum_is_skipped_beside_go_mod() {
388 let files = Lockfiles {
389 commit: None,
390 files: vec![
391 LockfileText { path: "go.mod".into(), text: "require golang.org/x/net v0.7.0\n".into() },
392 LockfileText { path: "go.sum".into(), text: "golang.org/x/net v0.1.0 h1:x=\n".into() },
393 LockfileText { path: "tools/go.sum".into(), text: "golang.org/x/text v0.3.0 h1:x=\n".into() },
394 ],
395 };
396 let found: Vec<String> = packages(&files).iter().map(|l| format!("{}:{}", l.path, l.package.version)).collect();
397 assert_eq!(found, ["go.mod:v0.7.0", "tools/go.sum:v0.3.0"]);
398 }
399
400 #[test]
401 fn the_issue_names_the_advisories_and_checks_the_lockfile() {
402 let row = VulnRow {
403 id: "vul_1".into(),
404 repo_id: "rep_1".into(),
405 ecosystem: "npm".into(),
406 package: "lodash".into(),
407 version: "4.17.20".into(),
408 manifest: "web/package-lock.json".into(),
409 osv_id: "GHSA-35jh-r3h4-6jhm".into(),
410 advisory: "GHSA-35jh-r3h4-6jhm".into(),
411 summary: "Command Injection in lodash".into(),
412 severity: "high".into(),
413 fixed_version: Some("4.17.21".into()),
414 status: "open".into(),
415 found_at: "2026-10-04T00:00:00Z".into(),
416 fixed_at: None,
417 number: None,
418 dismiss_reason: None,
419 dismiss_comment: None,
420 dismissed_by: None,
421 dismissed_at: None,
422 };
423 let located = vec![Located {
424 package: Package { ecosystem: g1t_scan::lockfiles::Ecosystem::Npm, name: "lodash".into(), version: "4.17.20".into() },
425 lockfile: Lockfile::PackageLock,
426 path: "web/package-lock.json".into(),
427 }];
428 let body = issue_text("npm", "lodash", "4.17.21", &[&row], &located);
429 assert!(body.contains("[GHSA-35jh-r3h4-6jhm](https://osv.dev/vulnerability/GHSA-35jh-r3h4-6jhm) | high | 4.17.20 | 4.17.21"));
430 assert!(body.contains("`web/package-lock.json` (4.17.20)"));
431 let (_, done) = body.split_once("## Definition of done\n\n").unwrap();
432 let items: Vec<&str> = done.lines().take_while(|line| line.starts_with("- ")).collect();
433 assert_eq!(items.len(), 3);
434 assert!(items[1].contains("node_modules/lodash") && items[1].contains("'web/package-lock.json'"));
435 assert_eq!(items[2], "- `cd 'web' && npm ci && npm test --if-present` passes.");
436 assert!(body.ends_with("on its Security page._"));
437 assert!(advisory_table(&[&row]).contains("| [GHSA-35jh-r3h4-6jhm](https://osv.dev/vulnerability/GHSA-35jh-r3h4-6jhm) | high |"));
438 }
439}