Skip to content
Sign up

flagon-io/g1t

public
0
456 linesCodeBlameRaw
1//! Dependencies: reading a repository's lockfiles, asking OSV about every
2//! package in them, and recording what is vulnerable. The security updates
3//! that fix them are `security_updates`'s.
4
5use std::collections::{BTreeSet, HashMap};
6
7use g1t_contracts::repos::RepoPath;
8use g1t_contracts::repos::{BlobArgs, BlobView};
9use g1t_contracts::security::{AlertState, FindLockfilesArgs, Lockfiles, VersionUpdatesState, VulnStatus, Vulnerability};
10use g1t_contracts::updates::DEPENDABOT_PATHS;
11use g1t_contracts::time::rfc3339;
12use g1t_contracts::work::{AddCommentArgs, Issue, IssueDetail, ViewArgs};
13use g1t_contracts::{Outcome, User};
14use g1t_kit::now_ms;
15use g1t_scan::lockfiles::{Lockfile, Package};
16use g1t_scan::osv::{self, Advisory};
17use serde_json::{Value, json};
18use worker::{Fetch, Headers, Method, Request, RequestInit, Result};
19
20use crate::Security;
21use crate::store::{RepoRow, VulnRow};
22use crate::updates;
23use crate::version_updates::Loaded;
24
25/// OSV's records are fetched again after this long.
26const ADVISORY_MAX_AGE_MS: u64 = 7 * 24 * 60 * 60 * 1000;
27/// A record that names no fixed version is fetched again after a day, so
28/// a fix is noticed the day it is published.
29const UNFIXED_ADVISORY_MAX_AGE_MS: u64 = 24 * 60 * 60 * 1000;
30/// Records fetched per scan, at most; the rest wait for the next one.
31const MAX_ADVISORY_FETCHES: usize = 150;
32/// CPU one call to OSV takes, sending it and reading its answer, in
33/// milliseconds (an estimate, rounded up). OSV itself is free, and a
34/// Worker's outgoing requests are not charged.
35const CPU_MS_PER_OSV_CALL: f64 = 2.0;
36
37/// What a dependency check cost g1t, in millionths of a dollar, rounded
38/// up, at the prices in `history`: the CPU of its OSV calls, and the rows
39/// it writes (an advisory kept for each call at most, each vulnerability
40/// found, where the check stands and the month's usage).
41pub fn dependency_check_cost(calls: u32, found: usize) -> i64 {
42 use crate::history::{MICROS_PER_CPU_MS, MICROS_PER_ROW_WRITTEN};
43 let cpu = f64::from(calls) * CPU_MS_PER_OSV_CALL * MICROS_PER_CPU_MS;
44 let rows = (calls as usize + found + 2) as f64 * MICROS_PER_ROW_WRITTEN;
45 (cpu + rows).ceil() as i64
46}
47
48async fn osv_call(method: Method, url: &str, body: Option<&Value>) -> Result<Option<Value>> {
49 let headers = Headers::new();
50 headers.set("user-agent", "g1t (+https://g1t.sh)")?;
51 headers.set("accept", "application/json")?;
52 if body.is_some() {
53 headers.set("content-type", "application/json")?;
54 }
55 let mut init = RequestInit::new();
56 init.with_method(method).with_headers(headers);
57 if let Some(body) = body {
58 init.with_body(Some(body.to_string().into()));
59 }
60 let mut response = Fetch::Request(Request::new_with_init(url, &init)?).send().await?;
61 if response.status_code() == 404 {
62 return Ok(None);
63 }
64 if !(200..300).contains(&response.status_code()) {
65 return Err(worker::Error::RustError(format!("OSV answered {}", response.status_code())));
66 }
67 Ok(Some(response.json().await?))
68}
69
70/// Whether an OSV record names a version that fixes it, for any package.
71fn names_a_fix(record: &Value) -> bool {
72 record["affected"].as_array().into_iter().flatten().any(|affected| {
73 affected["ranges"]
74 .as_array()
75 .into_iter()
76 .flatten()
77 .any(|range| range["events"].as_array().into_iter().flatten().any(|event| event.get("fixed").is_some()))
78 })
79}
80
81/// One package in one lockfile.
82pub(crate) struct Located {
83 pub(crate) package: Package,
84 pub(crate) path: String,
85}
86
87/// Every package the lockfiles resolve. A directory with a `go.mod` is read
88/// from it rather than from its `go.sum`, which lists versions not built.
89fn packages(files: &Lockfiles) -> Vec<Located> {
90 let go_mods: BTreeSet<&str> = files
91 .files
92 .iter()
93 .filter(|file| file.path.ends_with("go.mod"))
94 .map(|file| file.path.trim_end_matches("go.mod"))
95 .collect();
96 let mut located = Vec::new();
97 for file in &files.files {
98 let Some(lockfile) = Lockfile::for_path(&file.path) else { continue };
99 if lockfile == Lockfile::GoSum && go_mods.contains(file.path.trim_end_matches("go.sum")) {
100 continue;
101 }
102 for package in lockfile.parse(&file.text) {
103 located.push(Located { package, path: file.path.clone() });
104 }
105 }
106 located
107}
108
109impl Security {
110 /// The ids of the vulnerabilities affecting each package, from OSV.
111 pub(crate) async fn query_osv(&self, packages: &[Package]) -> Result<(Vec<Vec<String>>, u32)> {
112 let mut ids = Vec::with_capacity(packages.len());
113 let mut calls = 0;
114 for (body, chunk) in osv::batch_bodies(packages).iter().zip(packages.chunks(osv::MAX_BATCH)) {
115 calls += 1;
116 let answer = osv_call(Method::Post, osv::QUERY_BATCH_URL, Some(body)).await?.unwrap_or(Value::Null);
117 let (mut found, more) = osv::read_batch(&answer, chunk.len());
118 // A package with many advisories is paged; fetch the rest.
119 for (index, token) in more.into_iter().take(20) {
120 let package = &chunk[index];
121 let query = json!({
122 "package": {"name": package.name, "ecosystem": package.ecosystem.osv()},
123 "version": package.version,
124 "page_token": token,
125 });
126 calls += 1;
127 if let Some(page) = osv_call(Method::Post, "https://api.osv.dev/v1/query", Some(&query)).await? {
128 found[index].extend(
129 page["vulns"].as_array().into_iter().flatten().filter_map(|v| v["id"].as_str().map(str::to_owned)),
130 );
131 }
132 }
133 ids.extend(found);
134 }
135 Ok((ids, calls))
136 }
137
138 /// OSV's record of each id, from the cache when it is fresh.
139 pub(crate) async fn advisories(&self, ids: &BTreeSet<String>) -> Result<(HashMap<String, Value>, u32)> {
140 let fresh_after = rfc3339(now_ms().saturating_sub(ADVISORY_MAX_AGE_MS));
141 let unfixed_fresh_after = rfc3339(now_ms().saturating_sub(UNFIXED_ADVISORY_MAX_AGE_MS));
142 let mut records = HashMap::new();
143 let mut fetched = 0u32;
144 for id in ids {
145 if let Some(record) = self.store.advisory(id, &fresh_after).await? {
146 // One without a fix is asked about again daily; until then,
147 // or if asking fails below, the kept record stands.
148 let fixed = names_a_fix(&record);
149 let recent = !fixed && self.store.advisory(id, &unfixed_fresh_after).await?.is_some();
150 if fixed || recent || fetched as usize >= MAX_ADVISORY_FETCHES {
151 records.insert(id.clone(), record);
152 continue;
153 }
154 fetched += 1;
155 let refreshed = osv_call(Method::Get, &osv::vuln_url(id), None).await.ok().flatten();
156 if let Some(refreshed) = &refreshed {
157 self.store.keep_advisory(id, refreshed).await?;
158 }
159 records.insert(id.clone(), refreshed.unwrap_or(record));
160 continue;
161 }
162 if fetched as usize >= MAX_ADVISORY_FETCHES {
163 continue;
164 }
165 fetched += 1;
166 if let Some(record) = osv_call(Method::Get, &osv::vuln_url(id), None).await? {
167 self.store.keep_advisory(id, &record).await?;
168 records.insert(id.clone(), record);
169 }
170 }
171 Ok((records, fetched))
172 }
173
174 /// Reads a repository's dependencies, records which are vulnerable,
175 /// and starts security updates for those with a fix. Also reads
176 /// `.g1t/dependencies.yml`. Returns what went wrong, for the Security
177 /// page, if anything did.
178 pub async fn scan_dependencies(&self, repo: &RepoRow) -> Result<Option<String>> {
179 let files: Lockfiles = g1t_kit::call(&self.repos, "find_lockfiles", &FindLockfilesArgs { repo_id: repo.repo_id.clone(), git_ref: None }).await?;
180 let paths: Vec<String> = files.files.iter().map(|file| file.path.clone()).collect();
181 let located = packages(&files);
182 let unique: Vec<Package> = located.iter().map(|l| l.package.clone()).collect::<BTreeSet<_>>().into_iter().collect();
183 let outcome = async {
184 let (ids, calls) = self.query_osv(&unique).await?;
185 let by_package: HashMap<&Package, &Vec<String>> = unique.iter().zip(ids.iter()).collect();
186 let wanted: BTreeSet<String> = ids.iter().flatten().cloned().collect();
187 let (records, fetched) = self.advisories(&wanted).await?;
188 let mut found = Vec::new();
189 for item in &located {
190 for id in by_package.get(&item.package).into_iter().flat_map(|ids| ids.iter()) {
191 let Some(advisory) = records.get(id).and_then(|record| osv::read_vuln(record, &item.package)) else {
192 continue;
193 };
194 found.push(vulnerability(&repo.repo_id, item, &advisory));
195 }
196 }
197 Ok::<_, worker::Error>((found, calls + fetched))
198 }
199 .await;
200 let (found, calls) = match outcome {
201 Ok(result) => result,
202 Err(error) => {
203 let problem = format!("The dependencies could not be checked: {error}");
204 self.store
205 .set_dependencies_scanned(&repo.repo_id, files.commit.as_deref(), &paths, Some(&problem))
206 .await?;
207 return Ok(Some(problem));
208 }
209 };
210 let before: BTreeSet<String> = self.store.open_vulnerabilities(&repo.repo_id).await?.into_iter().map(|row| row.id).collect();
211 self.store.replace_vulnerabilities(&repo.repo_id, &found).await?;
212 self.store.set_dependencies_scanned(&repo.repo_id, files.commit.as_deref(), &paths, None).await?;
213 // g1t's update pull requests whose alerts are now fixed or dismissed,
214 // or whose versions the lockfiles already have, close first, so that
215 // nothing later in the scan can keep them open.
216 if let Err(error) = self.resolve_updates(repo, Some((&located, &paths))).await {
217 worker::console_error!("security: updates of {} not resolved: {error}", repo.repo_id);
218 }
219 if let Err(error) = self.record_graph(repo, &files).await {
220 worker::console_error!("security: dependency graph of {} not kept: {error}", repo.repo_id);
221 }
222 self.vulnerabilities_changed(repo, &before).await?;
223 self.meter(&repo.namespace, 0, 0, calls, dependency_check_cost(calls, found.len())).await?;
224 // The dependency update file says how security updates are made too.
225 let rules = match files.commit.as_deref() {
226 Some(commit) => self.read_version_updates(repo, commit).await?,
227 None => None,
228 };
229 // No security updates on an archived (read-only) or deleted
230 // repository, nor on a mirror until it is taken over or moved to
231 // g1t (`g1t_contracts::mirrors`: a standby mirror is an exact copy
232 // that opens nothing of its own); ones in flight for packages no
233 // longer vulnerable are closed either way.
234 let takes_updates = self.takes_updates(repo).await?.is_ok();
235 self.security_updates(repo, repo.upkeep != 0 && takes_updates, rules.as_ref()).await?;
236 Ok(None)
237 }
238
239 /// Reads the dependency update file at `commit` and keeps what it
240 /// says; see `updates`. Its entries' schedules start or move. Returns
241 /// it, when it has no problems.
242 pub(crate) async fn read_version_updates(&self, repo: &RepoRow, commit: &str) -> Result<Option<Loaded>> {
243 let path = RepoPath { namespace: repo.namespace.clone(), name: repo.name.clone() };
244 let asks: Vec<BlobArgs> = DEPENDABOT_PATHS
245 .iter()
246 .map(|file| BlobArgs {
247 path: path.clone(),
248 viewer: Some(User::system(&repo.namespace)),
249 git_ref: commit.to_owned(),
250 file_path: (*file).to_owned(),
251 })
252 .collect();
253 let reads = asks.iter().map(|ask| g1t_kit::call::<_, Outcome<BlobView>>(&self.repos, "blob", ask));
254 let mut found = Vec::new();
255 let mut default_branch = None;
256 for (file, read) in DEPENDABOT_PATHS.iter().zip(futures_util::future::join_all(reads).await) {
257 if let Outcome::Ok(blob) = read? {
258 default_branch = Some(blob.repo.default_branch.clone());
259 found.push(((*file).to_owned(), blob.text));
260 }
261 }
262 let (mut state, config) = match updates::choose(found) {
263 Some(((file, text), ignored)) => {
264 let (state, config) = updates::state(&file, text.as_deref(), ignored, &repo.repo_id, default_branch.as_deref());
265 (state, config.map(|config| (config, file)))
266 }
267 None => (VersionUpdatesState::default(), None),
268 };
269 state.read_at = Some(crate::store::now());
270 state.commit = Some(commit.to_owned());
271 self.store.set_version_updates(&repo.repo_id, &state).await?;
272 self.schedule_entries(repo, config.as_ref().map(|(config, _)| config), default_branch.as_deref()).await?;
273 Ok(config.map(|(config, file)| Loaded { config, file, default_branch: default_branch.unwrap_or_default() }))
274 }
275
276 pub(crate) async fn issue(&self, actor: &User, repo: &RepoPath, number: u32) -> Result<Option<Issue>> {
277 let found: Outcome<IssueDetail> = g1t_kit::call(
278 &self.work,
279 "get_issue",
280 &ViewArgs { repo: repo.clone(), number, viewer: Some(actor.clone()), after_seq: 0 },
281 )
282 .await?;
283 Ok(found.into_result().ok().map(|detail| detail.issue))
284 }
285
286 pub(crate) async fn comment(&self, actor: &User, repo: &RepoPath, number: u32, body: String) -> Result<()> {
287 let _: Outcome<Value> = g1t_kit::call(
288 &self.work,
289 "add_comment",
290 &AddCommentArgs {
291 actor: actor.clone(),
292 repo: repo.clone(),
293 number,
294 body,
295 path: None,
296 line: None,
297 verdict: None,
298 },
299 )
300 .await?;
301 Ok(())
302 }
303}
304
305/// The event that tells of a vulnerability alert.
306pub(crate) fn vulnerability_event(repo: &RepoRow, vuln: &Vulnerability) -> g1t_contracts::security_suite::SecurityEvent {
307 g1t_contracts::security_suite::SecurityEvent {
308 repo_id: repo.repo_id.clone(),
309 alert_id: vuln.id.clone(),
310 alert_type: g1t_contracts::security_suite::AlertType::Vulnerability.as_str().to_owned(),
311 severity: vuln.severity.clone(),
312 title: format!("{} {} in {}: {}", vuln.package, vuln.version, vuln.manifest, vuln.advisory),
313 link: crate::suite::link(repo, &format!("vulnerabilities?finding={}", vuln.id)),
314 path: Some(vuln.manifest.clone()),
315 state: vuln.state.as_str().to_owned(),
316 ..Default::default()
317 }
318}
319
320impl Security {
321 /// Tells of vulnerability alerts a dependency read opened or fixed,
322 /// given the ids that were open before it: an event each for webhooks
323 /// (the first `MAX_SUMMARY_IDS` of each), and one summary of the read
324 /// for the inbox, which carries the counts whatever the number.
325 async fn vulnerabilities_changed(&self, repo: &RepoRow, before: &BTreeSet<String>) -> Result<()> {
326 use g1t_contracts::security_suite::{AlertType, MAX_SUMMARY_IDS};
327 let after: Vec<VulnRow> = self.store.open_vulnerabilities(&repo.repo_id).await?;
328 let now: BTreeSet<&str> = after.iter().map(|row| row.id.as_str()).collect();
329 // Worst first, as the rows come: the summary's headline is the worst new one.
330 let opened: Vec<String> = after.iter().map(|row| row.id.clone()).filter(|id| !before.contains(id)).collect();
331 let mut fixed: Vec<String> = Vec::new();
332 for id in before.iter().filter(|id| !now.contains(id.as_str())) {
333 if let Some(vuln) = self.store.vulnerability(&repo.repo_id, id).await?
334 && vuln.status == VulnStatus::Fixed
335 {
336 fixed.push(id.clone());
337 }
338 }
339 for id in opened.iter().take(MAX_SUMMARY_IDS) {
340 if let Some(vuln) = self.store.vulnerability(&repo.repo_id, id).await? {
341 self.alert_event(AlertType::Vulnerability, "created", repo, vulnerability_event(repo, &vuln), None).await;
342 }
343 }
344 for id in fixed.iter().take(MAX_SUMMARY_IDS) {
345 if let Some(vuln) = self.store.vulnerability(&repo.repo_id, id).await? {
346 self.alert_event(AlertType::Vulnerability, "fixed", repo, vulnerability_event(repo, &vuln), None).await;
347 }
348 }
349 let changed = crate::summaries::Changed { scan: "dependencies", opened, fixed, ..crate::summaries::Changed::default() };
350 self.summarize(repo, AlertType::Vulnerability, changed, None).await
351 }
352}
353
354fn vulnerability(repo_id: &str, item: &Located, advisory: &Advisory) -> Vulnerability {
355 Vulnerability {
356 id: String::new(),
357 repo_id: repo_id.to_owned(),
358 ecosystem: item.package.ecosystem.osv().to_owned(),
359 package: item.package.name.clone(),
360 version: item.package.version.clone(),
361 manifest: item.path.clone(),
362 advisory: advisory.display_id.clone(),
363 osv_id: advisory.id.clone(),
364 summary: advisory.summary.clone(),
365 severity: advisory.severity.as_str().to_owned(),
366 fixed_version: advisory.fixed.clone(),
367 status: VulnStatus::Open,
368 issue: None,
369 found_at: String::new(),
370 fixed_at: None,
371 state: AlertState::Open,
372 dismissed_by: None,
373 dismissed_reason: None,
374 dismissed_comment: None,
375 dismissed_at: None,
376 update: None,
377 }
378}
379
380/// The advisories a package's vulnerabilities name, as a table.
381pub(crate) fn advisory_table(vulns: &[&VulnRow]) -> String {
382 let mut table = "| Advisory | Severity | Affected | Fixed in | Summary |\n| --- | --- | --- | --- | --- |\n".to_owned();
383 let mut seen = BTreeSet::new();
384 for vuln in vulns {
385 if !seen.insert((vuln.advisory.clone(), vuln.version.clone())) {
386 continue;
387 }
388 table.push_str(&format!(
389 "| [{}]({}) | {} | {} | {} | {} |\n",
390 vuln.advisory,
391 osv::page_url(&vuln.osv_id),
392 vuln.severity,
393 vuln.version,
394 vuln.fixed_version.as_deref().unwrap_or("none yet"),
395 vuln.summary.replace('|', "\\|").replace('\n', " "),
396 ));
397 }
398 table
399}
400
401#[cfg(test)]
402mod tests {
403 use super::*;
404 use g1t_contracts::security::LockfileText;
405
406 #[test]
407 fn scans_cost_their_cpu_and_the_rows_they_write() {
408 // 10 OSV calls and 3 vulnerabilities: 0.4 of CPU, 15 rows.
409 assert_eq!(dependency_check_cost(10, 3), 16);
410 assert_eq!(dependency_check_cost(0, 0), 2);
411 // A page of 25 commits that read 100 objects and found nothing:
412 // 10 of CPU and 2 rows. The old placeholder charged 100.
413 assert_eq!(crate::history::history_page_cost(100, 0), 12);
414 assert_eq!(crate::history::history_page_cost(0, 1), 3);
415 }
416
417 #[test]
418 fn go_sum_is_skipped_beside_go_mod() {
419 let files = Lockfiles {
420 commit: None,
421 files: vec![
422 LockfileText { path: "go.mod".into(), text: "require golang.org/x/net v0.7.0\n".into() },
423 LockfileText { path: "go.sum".into(), text: "golang.org/x/net v0.1.0 h1:x=\n".into() },
424 LockfileText { path: "tools/go.sum".into(), text: "golang.org/x/text v0.3.0 h1:x=\n".into() },
425 ],
426 };
427 let found: Vec<String> = packages(&files).iter().map(|l| format!("{}:{}", l.path, l.package.version)).collect();
428 assert_eq!(found, ["go.mod:v0.7.0", "tools/go.sum:v0.3.0"]);
429 }
430
431 #[test]
432 fn the_table_links_each_advisory() {
433 let row = VulnRow {
434 id: "vul_1".into(),
435 repo_id: "rep_1".into(),
436 ecosystem: "npm".into(),
437 package: "lodash".into(),
438 version: "4.17.20".into(),
439 manifest: "web/package-lock.json".into(),
440 osv_id: "GHSA-35jh-r3h4-6jhm".into(),
441 advisory: "GHSA-35jh-r3h4-6jhm".into(),
442 summary: "Command Injection in lodash".into(),
443 severity: "high".into(),
444 fixed_version: Some("4.17.21".into()),
445 status: "open".into(),
446 found_at: "2026-10-04T00:00:00Z".into(),
447 fixed_at: None,
448 number: None,
449 dismiss_reason: None,
450 dismiss_comment: None,
451 dismissed_by: None,
452 dismissed_at: None,
453 };
454 assert!(advisory_table(&[&row]).contains("| [GHSA-35jh-r3h4-6jhm](https://osv.dev/vulnerability/GHSA-35jh-r3h4-6jhm) | high |"));
455 }
456}