Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/security/src/supply_chain.rs

394 lines18,633 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Teams and CODEOWNERS, labels and milestones, dependency updates, the security suite, and a clearer top bar1//! The supply chain: a repository's dependency graph, read from the same
2//! lockfiles as its vulnerability alerts; the graph as an SPDX SBOM; and
3//! dependency review, which compares a pull request's dependencies with
4//! its base and fails the `Dependency review` check when it adds a package
5//! with a known vulnerability at or above the repository's threshold, or a
6//! license the repository does not allow.
7
8use std::collections::{BTreeMap, BTreeSet, HashMap};
9
10use g1t_contracts::repos::RepoPath;
11use g1t_contracts::security::{FindLockfilesArgs, Lockfiles};
12use g1t_contracts::security_suite::{
13 DEPENDENCY_REVIEW_CHECK, DependencyGraph, DependencyGraphArgs, DependencyReview, DependencyReviewArgs, GraphDependency,
14 GraphManifest, PaidFeature, ReviewChange, ReviewVulnerability, SbomArgs,
15};
16use g1t_contracts::work::AddCommentArgs;
17use g1t_contracts::{Outcome, User};
18use g1t_scan::graph::{self, Dependency, Relationship};
19use g1t_scan::lockfiles::{Ecosystem, Lockfile, Package};
20use g1t_scan::osv::{self, Severity};
21use g1t_scan::review::{self, ChangeKind, Finding, Policy, Verdict};
22use g1t_scan::sbom;
23use serde_json::Value;
24use worker::Result;
25
26use crate::Security;
27use crate::store::RepoRow;
28use crate::suite_store::DependencyRow;
29
30/// Added packages a review asks OSV about, at most; the rest are listed
31/// without.
32const MAX_REVIEWED: usize = 500;
33
34/// Every package `files` resolve, with what each lockfile says of it. A
35/// directory with a `go.mod` is read from it rather than its `go.sum`.
36pub fn graph_of(files: &Lockfiles) -> Vec<Dependency> {
37 let go_mods: BTreeSet<&str> = files
38 .files
39 .iter()
40 .filter(|file| file.path.ends_with("go.mod"))
41 .map(|file| file.path.trim_end_matches("go.mod"))
42 .collect();
43 let mut deps = Vec::new();
44 for file in &files.files {
45 let Some(lockfile) = Lockfile::for_path(&file.path) else { continue };
46 if lockfile == Lockfile::GoSum && go_mods.contains(file.path.trim_end_matches("go.sum")) {
47 continue;
48 }
49 deps.extend(graph::dependencies(lockfile, &file.path, &file.text));
50 }
51 deps
52}
53
54fn row_dependency(row: &DependencyRow) -> Option<Dependency> {
55 Some(Dependency {
56 package: Package { ecosystem: Ecosystem::parse(&row.ecosystem)?, name: row.name.clone(), version: row.version.clone() },
57 manifest: row.manifest.clone(),
58 relationship: Relationship::parse(&row.relationship),
59 development: row.development != 0,
60 license: row.license.clone(),
61 })
62}
63
64/// The policy a repository's settings make.
65pub fn policy_of(fail_on: &str, deny: &[String]) -> Policy {
66 Policy {
67 fail_on: match fail_on {
68 "none" => None,
69 other => Some(Severity::parse(other)).filter(|severity| *severity != Severity::Unknown),
70 },
71 deny_licenses: deny.iter().map(|id| id.trim().to_owned()).filter(|id| !id.is_empty()).collect(),
72 }
73}
74
75/// The review as the API and the pull request page show it.
76pub fn review_view(base: &str, head: &str, verdict: &Verdict, policy: &Policy) -> DependencyReview {
77 let change = |kind: &str, dep: &Dependency, findings: &[Finding], failing: &[Finding], denied: bool| ReviewChange {
78 change_type: kind.to_owned(),
79 manifest: dep.manifest.clone(),
80 ecosystem: dep.package.ecosystem.osv().to_owned(),
81 name: dep.package.name.clone(),
82 version: dep.package.version.clone(),
83 relationship: dep.relationship.as_str().to_owned(),
84 development: dep.development,
85 license: dep.license.clone(),
86 purl: dep.purl(),
87 vulnerabilities: findings
88 .iter()
89 .map(|finding| ReviewVulnerability {
90 advisory: finding.advisory.clone(),
91 osv_id: finding.osv_id.clone(),
92 summary: finding.summary.clone(),
93 severity: finding.severity.as_str().to_owned(),
94 fixed_version: finding.fixed.clone(),
95 url: osv::page_url(&finding.osv_id),
96 })
97 .collect(),
98 denied_license: denied,
99 failing: !failing.is_empty() || denied,
100 };
101 let mut changes: Vec<ReviewChange> = verdict
102 .added
103 .iter()
104 .map(|reviewed| change("added", &reviewed.dependency, &reviewed.findings, &reviewed.failing, reviewed.denied_license))
105 .collect();
106 changes.extend(verdict.removed.iter().map(|dep| change("removed", dep, &[], &[], false)));
107 DependencyReview {
108 base: base.to_owned(),
109 head: head.to_owned(),
110 changes,
111 passed: verdict.passed,
112 headline: verdict.headline(),
113 fail_on: policy.fail_on.map_or("none", |severity| severity.as_str()).to_owned(),
114 deny_licenses: policy.deny_licenses.clone(),
115 }
116}
117
118/// Whether a comparison touches any lockfile: if not, nothing in the
119/// dependency graph can have changed.
120pub fn touches_lockfiles(paths: &[String]) -> bool {
121 paths.iter().any(|path| Lockfile::for_path(path).is_some())
122}
123
124impl Security {
125 /// Keeps the dependency graph the lockfiles describe. Called with the
126 /// lockfiles each dependency read fetched.
127 pub(crate) async fn record_graph(&self, repo: &RepoRow, files: &Lockfiles) -> Result<()> {
128 self.store.replace_dependencies(&repo.repo_id, &graph_of(files)).await
129 }
130
131 async fn lockfiles_at(&self, repo_id: &str, git_ref: Option<&str>) -> Result<Lockfiles> {
132 g1t_kit::call(&self.repos, "find_lockfiles", &FindLockfilesArgs { repo_id: repo_id.to_owned(), git_ref: git_ref.map(str::to_owned) }).await
133 }
134
135 pub(crate) async fn dependency_graph(&self, a: DependencyGraphArgs) -> Result<Outcome<DependencyGraph>> {
136 let mut repo = match self.member_repo(&a.repo, &a.viewer, crate::SEE_FINDINGS).await? {
137 Outcome::Ok(repo) => repo,
138 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
139 };
140 if repo.deps_scanned_at.is_none() {
141 self.scan_dependencies(&repo).await?;
142 repo = self.store.repo(&repo.repo_id).await?.unwrap_or(repo);
143 }
144 let rows = self.store.dependencies(&repo.repo_id).await?;
145 let mut vulnerable: HashMap<(String, String, String), u32> = HashMap::new();
146 for vuln in self.store.open_vulnerabilities(&repo.repo_id).await? {
147 *vulnerable.entry((vuln.ecosystem, vuln.package, vuln.version)).or_default() += 1;
148 }
149 let mut manifests: BTreeMap<String, GraphManifest> = BTreeMap::new();
150 let mut dependencies = Vec::with_capacity(rows.len());
151 for row in &rows {
152 let Some(dep) = row_dependency(row) else { continue };
153 let manifest = manifests.entry(row.manifest.clone()).or_insert_with(|| GraphManifest {
154 path: row.manifest.clone(),
155 ecosystem: row.ecosystem.clone(),
156 dependencies: 0,
157 direct: 0,
158 });
159 manifest.dependencies += 1;
160 manifest.direct += u32::from(dep.relationship == Relationship::Direct);
161 dependencies.push(GraphDependency {
162 vulnerabilities: vulnerable.get(&(row.ecosystem.clone(), row.name.clone(), row.version.clone())).copied().unwrap_or(0),
163 purl: dep.purl(),
164 ecosystem: row.ecosystem.clone(),
165 name: row.name.clone(),
166 version: row.version.clone(),
167 manifest: row.manifest.clone(),
168 relationship: row.relationship.clone(),
169 development: row.development != 0,
170 license: row.license.clone(),
171 });
172 }
173 // Lockfiles with nothing in them still show.
174 for path in repo.scan_state().lockfiles {
175 manifests.entry(path.clone()).or_insert_with(|| GraphManifest {
176 ecosystem: Lockfile::for_path(&path).map(|lockfile| lockfile.ecosystem().osv().to_owned()).unwrap_or_default(),
177 path,
178 dependencies: 0,
179 direct: 0,
180 });
181 }
182 Ok(Outcome::Ok(DependencyGraph { commit: self.store.deps_commit(&repo.repo_id).await?, manifests: manifests.into_values().collect(), dependencies }))
183 }
184
185 pub(crate) async fn sbom(&self, a: SbomArgs) -> Result<Outcome<Value>> {
186 let graph = match self.dependency_graph(DependencyGraphArgs { viewer: a.viewer, repo: a.repo.clone() }).await? {
187 Outcome::Ok(graph) => graph,
188 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
189 };
190 let deps: Vec<Dependency> = graph
191 .dependencies
192 .iter()
193 .filter_map(|dep| {
194 Some(Dependency {
195 package: Package { ecosystem: Ecosystem::parse(&dep.ecosystem)?, name: dep.name.clone(), version: dep.version.clone() },
196 manifest: dep.manifest.clone(),
197 relationship: Relationship::parse(&dep.relationship),
198 development: dep.development,
199 license: dep.license.clone(),
200 })
201 })
202 .collect();
203 let full_name = format!("{}/{}", a.repo.namespace, a.repo.name);
204 let url = format!("https://g1t.sh/{full_name}");
205 let unique = g1t_contracts::new_id("sbom", g1t_kit::now_ms());
206 let created = format!("{}Z", &crate::store::now()[..19]);
207 Ok(Outcome::Ok(sbom::spdx(
208 &sbom::Subject { full_name: &full_name, url: &url, commit: graph.commit.as_deref(), unique: &unique, created: &created },
209 &deps,
210 )))
211 }
212
213 /// Asks OSV about the packages a change adds.
214 async fn findings_for(&self, added: &[&Dependency]) -> Result<BTreeMap<String, Vec<Finding>>> {
215 let packages: Vec<Package> = added.iter().map(|dep| dep.package.clone()).collect::<BTreeSet<_>>().into_iter().take(MAX_REVIEWED).collect();
216 if packages.is_empty() {
217 return Ok(BTreeMap::new());
218 }
219 let (ids, _) = self.query_osv(&packages).await?;
220 let wanted: BTreeSet<String> = ids.iter().flatten().cloned().collect();
221 let (records, _) = self.advisories(&wanted).await?;
222 let mut found = BTreeMap::new();
223 for (package, ids) in packages.iter().zip(ids) {
224 let findings: Vec<Finding> = ids
225 .iter()
226 .filter_map(|id| osv::read_vuln(records.get(id)?, package))
227 .map(|advisory| Finding {
228 advisory: advisory.display_id,
229 osv_id: advisory.id,
230 summary: advisory.summary,
231 severity: advisory.severity,
232 fixed: advisory.fixed,
233 })
234 .collect();
235 if !findings.is_empty() {
236 found.insert(graph::purl(package.ecosystem, &package.name, &package.version), findings);
237 }
238 }
239 Ok(found)
240 }
241
242 /// Reviews the change from `base` (in `base_repo`) to `head` (in
243 /// `head_repo`, a fork's for a pull request from one).
244 async fn review(&self, base_repo: &str, base: &str, head_repo: &str, head: &str, policy: &Policy) -> Result<(Verdict, DependencyReview)> {
245 let (before, after) = (self.lockfiles_at(base_repo, Some(base)).await?, self.lockfiles_at(head_repo, Some(head)).await?);
246 let changes = review::diff(&graph_of(&before), &graph_of(&after));
247 let added: Vec<&Dependency> = changes.iter().filter(|change| change.kind == ChangeKind::Added).map(|change| &change.dependency).collect();
248 let findings = self.findings_for(&added).await?;
249 let verdict = review::judge(&changes, &findings, policy);
250 let view = review_view(base, head, &verdict, policy);
251 Ok((verdict, view))
252 }
253
254 pub(crate) async fn dependency_review(&self, a: DependencyReviewArgs) -> Result<Outcome<DependencyReview>> {
255 let repo = match self.member_repo(&a.repo, &a.viewer, crate::SEE_FINDINGS).await? {
256 Outcome::Ok(repo) => repo,
257 Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)),
258 };
259 if let Some(refusal) = self.gate(&repo, PaidFeature::DependencyReview).await? {
260 return Ok(refusal);
261 }
262 let (settings, _) = self.store.repo_settings(&repo.repo_id).await?;
263 let policy = policy_of(&settings.review_fail_on, &settings.review_deny_licenses);
264 let (_, view) = self.review(&repo.repo_id, &a.base, &repo.repo_id, &a.head, &policy).await?;
265 Ok(Outcome::Ok(view))
266 }
267
268 /// The `Dependency review` check on a pull request whose head moved.
269 pub(crate) async fn review_pull(&self, repo_id: &str, number: u32) -> Result<()> {
270 let Some(repo) = self.register_by_id(repo_id).await? else { return Ok(()) };
271 let (settings, _) = self.store.repo_settings(&repo.repo_id).await?;
272 if !settings.dependency_review || !self.entitled(&repo).await? {
273 return Ok(());
274 }
275 let Some(pull) = self.pull_by_number(&repo, number).await? else { return Ok(()) };
276 let Some(head) = pull.head_commit.clone() else { return Ok(()) };
277 if let Some(previous) = self.store.pull_check(&repo.repo_id, number, "review").await?
278 && previous.commit_sha == head
279 {
280 return Ok(());
281 }
282 let Some(comparison) = self.pull_comparison(&repo, &pull).await? else { return Ok(()) };
283 let policy = policy_of(&settings.review_fail_on, &settings.review_deny_licenses);
284 let paths: Vec<String> = comparison.files.iter().map(|file| file.path.clone()).collect();
285 let (state, description, detail, summary) = if !touches_lockfiles(&paths) {
286 let view = DependencyReview {
287 head: head.clone(),
288 passed: true,
289 headline: "No dependency changes".to_owned(),
290 fail_on: policy.fail_on.map_or("none", |severity| severity.as_str()).to_owned(),
291 deny_licenses: policy.deny_licenses.clone(),
292 ..DependencyReview::default()
293 };
294 ("success", view.headline.clone(), view, None)
295 } else {
296 let base = comparison.base.clone().unwrap_or_else(|| "HEAD".to_owned());
297 let head_repo = pull.fork_repo_id.clone().unwrap_or_else(|| repo.repo_id.clone());
298 let (verdict, view) = self.review(&repo.repo_id, &base, &head_repo, &head, &policy).await?;
299 let summary = verdict.summary(&policy);
300 (if verdict.passed { "success" } else { "failure" }, verdict.headline(), view, Some(summary))
301 };
302 self.store
303 .set_pull_check(&repo.repo_id, number, "review", &head, state, &description, &serde_json::to_value(&detail)?, &[])
304 .await?;
305 self.set_status(&repo, &head, DEPENDENCY_REVIEW_CHECK, state, &description, number).await;
306 if let (Some(summary), true) = (summary, settings.review_comment) {
307 let _: Result<Outcome<Value>> = g1t_kit::call(
308 &self.work,
309 "add_comment",
310 &AddCommentArgs {
311 actor: User::system(&repo.namespace),
312 repo: RepoPath { namespace: repo.namespace.clone(), name: repo.name.clone() },
313 number,
314 body: summary,
315 path: None,
316 line: None,
317 verdict: None,
318 },
319 )
320 .await;
321 }
322 Ok(())
323 }
324}
325
326#[cfg(test)]
327mod tests {
328 use super::*;
329 use g1t_contracts::security::LockfileText;
330
331 #[test]
332 fn the_graph_reads_every_lockfile_and_skips_go_sum_beside_go_mod() {
333 let files = Lockfiles {
334 commit: None,
335 files: vec![
336 LockfileText { path: "go.mod".into(), text: "require golang.org/x/net v0.7.0 // indirect\n".into() },
337 LockfileText { path: "go.sum".into(), text: "golang.org/x/net v0.1.0 h1:x=\n".into() },
338 LockfileText {
339 path: "web/package-lock.json".into(),
340 text: r#"{"packages":{"":{"dependencies":{"lodash":"^4"}},"node_modules/lodash":{"version":"4.17.21","license":"MIT"}}}"#.into(),
341 },
342 ],
343 };
344 let deps: Vec<String> = graph_of(&files)
345 .iter()
346 .map(|dep| format!("{} {} {}", dep.manifest, dep.package.name, dep.relationship.as_str()))
347 .collect();
348 assert_eq!(deps, ["go.mod golang.org/x/net transitive", "web/package-lock.json lodash direct"]);
349 }
350
351 #[test]
352 fn the_policy_comes_from_the_settings() {
353 assert_eq!(policy_of("high", &[" GPL-3.0-only ".into(), "".into()]).fail_on, Some(Severity::High));
354 assert_eq!(policy_of("high", &[" GPL-3.0-only ".into(), "".into()]).deny_licenses, ["GPL-3.0-only"]);
355 assert_eq!(policy_of("none", &[]).fail_on, None);
356 assert_eq!(policy_of("moderate", &[]).fail_on, Some(Severity::Medium));
357 }
358
359 #[test]
360 fn only_lockfile_changes_need_a_review() {
361 assert!(!touches_lockfiles(&["src/app.ts".into(), "README.md".into()]));
362 assert!(touches_lockfiles(&["src/app.ts".into(), "web/package-lock.json".into()]));
363 }
364
365 #[test]
366 fn the_review_view_marks_what_fails() {
367 let dep = |version: &str| Dependency {
368 package: Package { ecosystem: Ecosystem::Npm, name: "lodash".into(), version: version.into() },
369 manifest: "package-lock.json".into(),
370 relationship: Relationship::Direct,
371 development: false,
372 license: Some("MIT".into()),
373 };
374 let changes = review::diff(&[dep("4.17.21")], &[dep("4.17.20")]);
375 let findings = BTreeMap::from([(
376 "pkg:npm/lodash@4.17.20".to_owned(),
377 vec![Finding {
378 advisory: "GHSA-35jh-r3h4-6jhm".into(),
379 osv_id: "GHSA-35jh-r3h4-6jhm".into(),
380 summary: "Command Injection in lodash".into(),
381 severity: Severity::High,
382 fixed: Some("4.17.21".into()),
383 }],
384 )]);
385 let policy = policy_of("high", &[]);
386 let verdict = review::judge(&changes, &findings, &policy);
387 let view = review_view("abc", "def", &verdict, &policy);
388 assert!(!view.passed);
389 assert_eq!(view.changes.len(), 2);
390 let added = view.changes.iter().find(|change| change.change_type == "added").unwrap();
391 assert!(added.failing && added.vulnerabilities[0].url.ends_with("GHSA-35jh-r3h4-6jhm"));
392 assert_eq!(view.fail_on, "high");
393 }
394}