| 1 | //! The supply chain: a repository's dependency graph, read from the same |
| 2 | //! lockfiles as its vulnerability alerts; the graph as an SPDX SBOM; and |
| 3 | //! dependency review, which compares a pull request's dependencies with |
| 4 | //! its base and fails the `Dependency review` check when it adds a package |
| 5 | //! with a known vulnerability at or above the repository's threshold, or a |
| 6 | //! license the repository does not allow. |
| 7 | |
| 8 | use std::collections::{BTreeMap, BTreeSet, HashMap}; |
| 9 | |
| 10 | use g1t_contracts::repos::RepoPath; |
| 11 | use g1t_contracts::security::{FindLockfilesArgs, Lockfiles}; |
| 12 | use g1t_contracts::security_suite::{ |
| 13 | DEPENDENCY_REVIEW_CHECK, DependencyGraph, DependencyGraphArgs, DependencyReview, DependencyReviewArgs, GraphDependency, |
| 14 | GraphManifest, PaidFeature, ReviewChange, ReviewVulnerability, SbomArgs, |
| 15 | }; |
| 16 | use g1t_contracts::work::AddCommentArgs; |
| 17 | use g1t_contracts::{Outcome, User}; |
| 18 | use g1t_scan::graph::{self, Dependency, Relationship}; |
| 19 | use g1t_scan::lockfiles::{Ecosystem, Lockfile, Package}; |
| 20 | use g1t_scan::osv::{self, Severity}; |
| 21 | use g1t_scan::review::{self, ChangeKind, Finding, Policy, Verdict}; |
| 22 | use g1t_scan::sbom; |
| 23 | use serde_json::Value; |
| 24 | use worker::Result; |
| 25 | |
| 26 | use crate::Security; |
| 27 | use crate::store::RepoRow; |
| 28 | use crate::suite_store::DependencyRow; |
| 29 | |
| 30 | /// Added packages a review asks OSV about, at most; the rest are listed |
| 31 | /// without. |
| 32 | const MAX_REVIEWED: usize = 500; |
| 33 | |
| 34 | /// Every package `files` resolve, with what each lockfile says of it. A |
| 35 | /// directory with a `go.mod` is read from it rather than its `go.sum`. |
| 36 | pub fn graph_of(files: &Lockfiles) -> Vec<Dependency> { |
| 37 | let go_mods: BTreeSet<&str> = files |
| 38 | .files |
| 39 | .iter() |
| 40 | .filter(|file| file.path.ends_with("go.mod")) |
| 41 | .map(|file| file.path.trim_end_matches("go.mod")) |
| 42 | .collect(); |
| 43 | let mut deps = Vec::new(); |
| 44 | for file in &files.files { |
| 45 | let Some(lockfile) = Lockfile::for_path(&file.path) else { continue }; |
| 46 | if lockfile == Lockfile::GoSum && go_mods.contains(file.path.trim_end_matches("go.sum")) { |
| 47 | continue; |
| 48 | } |
| 49 | deps.extend(graph::dependencies(lockfile, &file.path, &file.text)); |
| 50 | } |
| 51 | deps |
| 52 | } |
| 53 | |
| 54 | fn row_dependency(row: &DependencyRow) -> Option<Dependency> { |
| 55 | Some(Dependency { |
| 56 | package: Package { ecosystem: Ecosystem::parse(&row.ecosystem)?, name: row.name.clone(), version: row.version.clone() }, |
| 57 | manifest: row.manifest.clone(), |
| 58 | relationship: Relationship::parse(&row.relationship), |
| 59 | development: row.development != 0, |
| 60 | license: row.license.clone(), |
| 61 | }) |
| 62 | } |
| 63 | |
| 64 | /// The policy a repository's settings make. |
| 65 | pub fn policy_of(fail_on: &str, deny: &[String]) -> Policy { |
| 66 | Policy { |
| 67 | fail_on: match fail_on { |
| 68 | "none" => None, |
| 69 | other => Some(Severity::parse(other)).filter(|severity| *severity != Severity::Unknown), |
| 70 | }, |
| 71 | deny_licenses: deny.iter().map(|id| id.trim().to_owned()).filter(|id| !id.is_empty()).collect(), |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | /// The review as the API and the pull request page show it. |
| 76 | pub fn review_view(base: &str, head: &str, verdict: &Verdict, policy: &Policy) -> DependencyReview { |
| 77 | let change = |kind: &str, dep: &Dependency, findings: &[Finding], failing: &[Finding], denied: bool| ReviewChange { |
| 78 | change_type: kind.to_owned(), |
| 79 | manifest: dep.manifest.clone(), |
| 80 | ecosystem: dep.package.ecosystem.osv().to_owned(), |
| 81 | name: dep.package.name.clone(), |
| 82 | version: dep.package.version.clone(), |
| 83 | relationship: dep.relationship.as_str().to_owned(), |
| 84 | development: dep.development, |
| 85 | license: dep.license.clone(), |
| 86 | purl: dep.purl(), |
| 87 | vulnerabilities: findings |
| 88 | .iter() |
| 89 | .map(|finding| ReviewVulnerability { |
| 90 | advisory: finding.advisory.clone(), |
| 91 | osv_id: finding.osv_id.clone(), |
| 92 | summary: finding.summary.clone(), |
| 93 | severity: finding.severity.as_str().to_owned(), |
| 94 | fixed_version: finding.fixed.clone(), |
| 95 | url: osv::page_url(&finding.osv_id), |
| 96 | }) |
| 97 | .collect(), |
| 98 | denied_license: denied, |
| 99 | failing: !failing.is_empty() || denied, |
| 100 | }; |
| 101 | let mut changes: Vec<ReviewChange> = verdict |
| 102 | .added |
| 103 | .iter() |
| 104 | .map(|reviewed| change("added", &reviewed.dependency, &reviewed.findings, &reviewed.failing, reviewed.denied_license)) |
| 105 | .collect(); |
| 106 | changes.extend(verdict.removed.iter().map(|dep| change("removed", dep, &[], &[], false))); |
| 107 | DependencyReview { |
| 108 | base: base.to_owned(), |
| 109 | head: head.to_owned(), |
| 110 | changes, |
| 111 | passed: verdict.passed, |
| 112 | headline: verdict.headline(), |
| 113 | fail_on: policy.fail_on.map_or("none", |severity| severity.as_str()).to_owned(), |
| 114 | deny_licenses: policy.deny_licenses.clone(), |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | /// Whether a comparison touches any lockfile: if not, nothing in the |
| 119 | /// dependency graph can have changed. |
| 120 | pub fn touches_lockfiles(paths: &[String]) -> bool { |
| 121 | paths.iter().any(|path| Lockfile::for_path(path).is_some()) |
| 122 | } |
| 123 | |
| 124 | impl Security { |
| 125 | /// Keeps the dependency graph the lockfiles describe. Called with the |
| 126 | /// lockfiles each dependency read fetched. |
| 127 | pub(crate) async fn record_graph(&self, repo: &RepoRow, files: &Lockfiles) -> Result<()> { |
| 128 | self.store.replace_dependencies(&repo.repo_id, &graph_of(files)).await |
| 129 | } |
| 130 | |
| 131 | async fn lockfiles_at(&self, repo_id: &str, git_ref: Option<&str>) -> Result<Lockfiles> { |
| 132 | g1t_kit::call(&self.repos, "find_lockfiles", &FindLockfilesArgs { repo_id: repo_id.to_owned(), git_ref: git_ref.map(str::to_owned) }).await |
| 133 | } |
| 134 | |
| 135 | pub(crate) async fn dependency_graph(&self, a: DependencyGraphArgs) -> Result<Outcome<DependencyGraph>> { |
| 136 | let mut repo = match self.member_repo(&a.repo, &a.viewer, crate::SEE_FINDINGS).await? { |
| 137 | Outcome::Ok(repo) => repo, |
| 138 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 139 | }; |
| 140 | if repo.deps_scanned_at.is_none() { |
| 141 | self.scan_dependencies(&repo).await?; |
| 142 | repo = self.store.repo(&repo.repo_id).await?.unwrap_or(repo); |
| 143 | } |
| 144 | let rows = self.store.dependencies(&repo.repo_id).await?; |
| 145 | let mut vulnerable: HashMap<(String, String, String), u32> = HashMap::new(); |
| 146 | for vuln in self.store.open_vulnerabilities(&repo.repo_id).await? { |
| 147 | *vulnerable.entry((vuln.ecosystem, vuln.package, vuln.version)).or_default() += 1; |
| 148 | } |
| 149 | let mut manifests: BTreeMap<String, GraphManifest> = BTreeMap::new(); |
| 150 | let mut dependencies = Vec::with_capacity(rows.len()); |
| 151 | for row in &rows { |
| 152 | let Some(dep) = row_dependency(row) else { continue }; |
| 153 | let manifest = manifests.entry(row.manifest.clone()).or_insert_with(|| GraphManifest { |
| 154 | path: row.manifest.clone(), |
| 155 | ecosystem: row.ecosystem.clone(), |
| 156 | dependencies: 0, |
| 157 | direct: 0, |
| 158 | }); |
| 159 | manifest.dependencies += 1; |
| 160 | manifest.direct += u32::from(dep.relationship == Relationship::Direct); |
| 161 | dependencies.push(GraphDependency { |
| 162 | vulnerabilities: vulnerable.get(&(row.ecosystem.clone(), row.name.clone(), row.version.clone())).copied().unwrap_or(0), |
| 163 | purl: dep.purl(), |
| 164 | ecosystem: row.ecosystem.clone(), |
| 165 | name: row.name.clone(), |
| 166 | version: row.version.clone(), |
| 167 | manifest: row.manifest.clone(), |
| 168 | relationship: row.relationship.clone(), |
| 169 | development: row.development != 0, |
| 170 | license: row.license.clone(), |
| 171 | }); |
| 172 | } |
| 173 | // Lockfiles with nothing in them still show. |
| 174 | for path in repo.scan_state().lockfiles { |
| 175 | manifests.entry(path.clone()).or_insert_with(|| GraphManifest { |
| 176 | ecosystem: Lockfile::for_path(&path).map(|lockfile| lockfile.ecosystem().osv().to_owned()).unwrap_or_default(), |
| 177 | path, |
| 178 | dependencies: 0, |
| 179 | direct: 0, |
| 180 | }); |
| 181 | } |
| 182 | Ok(Outcome::Ok(DependencyGraph { commit: self.store.deps_commit(&repo.repo_id).await?, manifests: manifests.into_values().collect(), dependencies })) |
| 183 | } |
| 184 | |
| 185 | pub(crate) async fn sbom(&self, a: SbomArgs) -> Result<Outcome<Value>> { |
| 186 | let graph = match self.dependency_graph(DependencyGraphArgs { viewer: a.viewer, repo: a.repo.clone() }).await? { |
| 187 | Outcome::Ok(graph) => graph, |
| 188 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 189 | }; |
| 190 | let deps: Vec<Dependency> = graph |
| 191 | .dependencies |
| 192 | .iter() |
| 193 | .filter_map(|dep| { |
| 194 | Some(Dependency { |
| 195 | package: Package { ecosystem: Ecosystem::parse(&dep.ecosystem)?, name: dep.name.clone(), version: dep.version.clone() }, |
| 196 | manifest: dep.manifest.clone(), |
| 197 | relationship: Relationship::parse(&dep.relationship), |
| 198 | development: dep.development, |
| 199 | license: dep.license.clone(), |
| 200 | }) |
| 201 | }) |
| 202 | .collect(); |
| 203 | let full_name = format!("{}/{}", a.repo.namespace, a.repo.name); |
| 204 | let url = format!("https://g1t.sh/{full_name}"); |
| 205 | let unique = g1t_contracts::new_id("sbom", g1t_kit::now_ms()); |
| 206 | let created = format!("{}Z", &crate::store::now()[..19]); |
| 207 | Ok(Outcome::Ok(sbom::spdx( |
| 208 | &sbom::Subject { full_name: &full_name, url: &url, commit: graph.commit.as_deref(), unique: &unique, created: &created }, |
| 209 | &deps, |
| 210 | ))) |
| 211 | } |
| 212 | |
| 213 | /// Asks OSV about the packages a change adds. |
| 214 | async fn findings_for(&self, added: &[&Dependency]) -> Result<BTreeMap<String, Vec<Finding>>> { |
| 215 | let packages: Vec<Package> = added.iter().map(|dep| dep.package.clone()).collect::<BTreeSet<_>>().into_iter().take(MAX_REVIEWED).collect(); |
| 216 | if packages.is_empty() { |
| 217 | return Ok(BTreeMap::new()); |
| 218 | } |
| 219 | let (ids, _) = self.query_osv(&packages).await?; |
| 220 | let wanted: BTreeSet<String> = ids.iter().flatten().cloned().collect(); |
| 221 | let (records, _) = self.advisories(&wanted).await?; |
| 222 | let mut found = BTreeMap::new(); |
| 223 | for (package, ids) in packages.iter().zip(ids) { |
| 224 | let findings: Vec<Finding> = ids |
| 225 | .iter() |
| 226 | .filter_map(|id| osv::read_vuln(records.get(id)?, package)) |
| 227 | .map(|advisory| Finding { |
| 228 | advisory: advisory.display_id, |
| 229 | osv_id: advisory.id, |
| 230 | summary: advisory.summary, |
| 231 | severity: advisory.severity, |
| 232 | fixed: advisory.fixed, |
| 233 | }) |
| 234 | .collect(); |
| 235 | if !findings.is_empty() { |
| 236 | found.insert(graph::purl(package.ecosystem, &package.name, &package.version), findings); |
| 237 | } |
| 238 | } |
| 239 | Ok(found) |
| 240 | } |
| 241 | |
| 242 | /// Reviews the change from `base` (in `base_repo`) to `head` (in |
| 243 | /// `head_repo`, a fork's for a pull request from one). |
| 244 | async fn review(&self, base_repo: &str, base: &str, head_repo: &str, head: &str, policy: &Policy) -> Result<(Verdict, DependencyReview)> { |
| 245 | let (before, after) = (self.lockfiles_at(base_repo, Some(base)).await?, self.lockfiles_at(head_repo, Some(head)).await?); |
| 246 | let changes = review::diff(&graph_of(&before), &graph_of(&after)); |
| 247 | let added: Vec<&Dependency> = changes.iter().filter(|change| change.kind == ChangeKind::Added).map(|change| &change.dependency).collect(); |
| 248 | let findings = self.findings_for(&added).await?; |
| 249 | let verdict = review::judge(&changes, &findings, policy); |
| 250 | let view = review_view(base, head, &verdict, policy); |
| 251 | Ok((verdict, view)) |
| 252 | } |
| 253 | |
| 254 | pub(crate) async fn dependency_review(&self, a: DependencyReviewArgs) -> Result<Outcome<DependencyReview>> { |
| 255 | let repo = match self.member_repo(&a.repo, &a.viewer, crate::SEE_FINDINGS).await? { |
| 256 | Outcome::Ok(repo) => repo, |
| 257 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 258 | }; |
| 259 | if let Some(refusal) = self.gate(&repo, PaidFeature::DependencyReview).await? { |
| 260 | return Ok(refusal); |
| 261 | } |
| 262 | let (settings, _) = self.store.repo_settings(&repo.repo_id).await?; |
| 263 | let policy = policy_of(&settings.review_fail_on, &settings.review_deny_licenses); |
| 264 | let (_, view) = self.review(&repo.repo_id, &a.base, &repo.repo_id, &a.head, &policy).await?; |
| 265 | Ok(Outcome::Ok(view)) |
| 266 | } |
| 267 | |
| 268 | /// The `Dependency review` check on a pull request whose head moved. |
| 269 | pub(crate) async fn review_pull(&self, repo_id: &str, number: u32) -> Result<()> { |
| 270 | let Some(repo) = self.register_by_id(repo_id).await? else { return Ok(()) }; |
| 271 | let (settings, _) = self.store.repo_settings(&repo.repo_id).await?; |
| 272 | if !settings.dependency_review || !self.entitled(&repo).await? { |
| 273 | return Ok(()); |
| 274 | } |
| 275 | let Some(pull) = self.pull_by_number(&repo, number).await? else { return Ok(()) }; |
| 276 | let Some(head) = pull.head_commit.clone() else { return Ok(()) }; |
| 277 | if let Some(previous) = self.store.pull_check(&repo.repo_id, number, "review").await? |
| 278 | && previous.commit_sha == head |
| 279 | { |
| 280 | return Ok(()); |
| 281 | } |
| 282 | let Some(comparison) = self.pull_comparison(&repo, &pull).await? else { return Ok(()) }; |
| 283 | let policy = policy_of(&settings.review_fail_on, &settings.review_deny_licenses); |
| 284 | let paths: Vec<String> = comparison.files.iter().map(|file| file.path.clone()).collect(); |
| 285 | let (state, description, detail, summary) = if !touches_lockfiles(&paths) { |
| 286 | let view = DependencyReview { |
| 287 | head: head.clone(), |
| 288 | passed: true, |
| 289 | headline: "No dependency changes".to_owned(), |
| 290 | fail_on: policy.fail_on.map_or("none", |severity| severity.as_str()).to_owned(), |
| 291 | deny_licenses: policy.deny_licenses.clone(), |
| 292 | ..DependencyReview::default() |
| 293 | }; |
| 294 | ("success", view.headline.clone(), view, None) |
| 295 | } else { |
| 296 | let base = comparison.base.clone().unwrap_or_else(|| "HEAD".to_owned()); |
| 297 | let head_repo = pull.fork_repo_id.clone().unwrap_or_else(|| repo.repo_id.clone()); |
| 298 | let (verdict, view) = self.review(&repo.repo_id, &base, &head_repo, &head, &policy).await?; |
| 299 | let summary = verdict.summary(&policy); |
| 300 | (if verdict.passed { "success" } else { "failure" }, verdict.headline(), view, Some(summary)) |
| 301 | }; |
| 302 | self.store |
| 303 | .set_pull_check(&repo.repo_id, number, "review", &head, state, &description, &serde_json::to_value(&detail)?, &[]) |
| 304 | .await?; |
| 305 | self.set_status(&repo, &head, DEPENDENCY_REVIEW_CHECK, state, &description, number).await; |
| 306 | if let (Some(summary), true) = (summary, settings.review_comment) { |
| 307 | let _: Result<Outcome<Value>> = g1t_kit::call( |
| 308 | &self.work, |
| 309 | "add_comment", |
| 310 | &AddCommentArgs { |
| 311 | actor: User::system(&repo.namespace), |
| 312 | repo: RepoPath { namespace: repo.namespace.clone(), name: repo.name.clone() }, |
| 313 | number, |
| 314 | body: summary, |
| 315 | path: None, |
| 316 | line: None, |
| 317 | verdict: None, |
| 318 | }, |
| 319 | ) |
| 320 | .await; |
| 321 | } |
| 322 | Ok(()) |
| 323 | } |
| 324 | } |
| 325 | |
| 326 | #[cfg(test)] |
| 327 | mod tests { |
| 328 | use super::*; |
| 329 | use g1t_contracts::security::LockfileText; |
| 330 | |
| 331 | #[test] |
| 332 | fn the_graph_reads_every_lockfile_and_skips_go_sum_beside_go_mod() { |
| 333 | let files = Lockfiles { |
| 334 | commit: None, |
| 335 | files: vec![ |
| 336 | LockfileText { path: "go.mod".into(), text: "require golang.org/x/net v0.7.0 // indirect\n".into() }, |
| 337 | LockfileText { path: "go.sum".into(), text: "golang.org/x/net v0.1.0 h1:x=\n".into() }, |
| 338 | LockfileText { |
| 339 | path: "web/package-lock.json".into(), |
| 340 | text: r#"{"packages":{"":{"dependencies":{"lodash":"^4"}},"node_modules/lodash":{"version":"4.17.21","license":"MIT"}}}"#.into(), |
| 341 | }, |
| 342 | ], |
| 343 | }; |
| 344 | let deps: Vec<String> = graph_of(&files) |
| 345 | .iter() |
| 346 | .map(|dep| format!("{} {} {}", dep.manifest, dep.package.name, dep.relationship.as_str())) |
| 347 | .collect(); |
| 348 | assert_eq!(deps, ["go.mod golang.org/x/net transitive", "web/package-lock.json lodash direct"]); |
| 349 | } |
| 350 | |
| 351 | #[test] |
| 352 | fn the_policy_comes_from_the_settings() { |
| 353 | assert_eq!(policy_of("high", &[" GPL-3.0-only ".into(), "".into()]).fail_on, Some(Severity::High)); |
| 354 | assert_eq!(policy_of("high", &[" GPL-3.0-only ".into(), "".into()]).deny_licenses, ["GPL-3.0-only"]); |
| 355 | assert_eq!(policy_of("none", &[]).fail_on, None); |
| 356 | assert_eq!(policy_of("moderate", &[]).fail_on, Some(Severity::Medium)); |
| 357 | } |
| 358 | |
| 359 | #[test] |
| 360 | fn only_lockfile_changes_need_a_review() { |
| 361 | assert!(!touches_lockfiles(&["src/app.ts".into(), "README.md".into()])); |
| 362 | assert!(touches_lockfiles(&["src/app.ts".into(), "web/package-lock.json".into()])); |
| 363 | } |
| 364 | |
| 365 | #[test] |
| 366 | fn the_review_view_marks_what_fails() { |
| 367 | let dep = |version: &str| Dependency { |
| 368 | package: Package { ecosystem: Ecosystem::Npm, name: "lodash".into(), version: version.into() }, |
| 369 | manifest: "package-lock.json".into(), |
| 370 | relationship: Relationship::Direct, |
| 371 | development: false, |
| 372 | license: Some("MIT".into()), |
| 373 | }; |
| 374 | let changes = review::diff(&[dep("4.17.21")], &[dep("4.17.20")]); |
| 375 | let findings = BTreeMap::from([( |
| 376 | "pkg:npm/lodash@4.17.20".to_owned(), |
| 377 | vec![Finding { |
| 378 | advisory: "GHSA-35jh-r3h4-6jhm".into(), |
| 379 | osv_id: "GHSA-35jh-r3h4-6jhm".into(), |
| 380 | summary: "Command Injection in lodash".into(), |
| 381 | severity: Severity::High, |
| 382 | fixed: Some("4.17.21".into()), |
| 383 | }], |
| 384 | )]); |
| 385 | let policy = policy_of("high", &[]); |
| 386 | let verdict = review::judge(&changes, &findings, &policy); |
| 387 | let view = review_view("abc", "def", &verdict, &policy); |
| 388 | assert!(!view.passed); |
| 389 | assert_eq!(view.changes.len(), 2); |
| 390 | let added = view.changes.iter().find(|change| change.change_type == "added").unwrap(); |
| 391 | assert!(added.failing && added.vulnerabilities[0].url.ends_with("GHSA-35jh-r3h4-6jhm")); |
| 392 | assert_eq!(view.fail_on, "high"); |
| 393 | } |
| 394 | } |