| 1 | //! The security service: what g1t finds wrong in a repository, and the |
| 2 | //! upkeep that fixes it without a person. |
| 3 | //! |
| 4 | //! - Secrets. The repos service refuses pushes that add one |
| 5 | //! (`push_blocked` says which were allowed and records the rest), and |
| 6 | //! each repository's history is scanned once, in the background. |
| 7 | //! A push too large to scan first is let through, and its new commits |
| 8 | //! are scanned after they land (`history::advance_push_scan`). |
| 9 | //! - Alerts are open, dismissed (with a reason, a comment and who) or |
| 10 | //! fixed, and each keeps an activity log. Likely test values are listed |
| 11 | //! apart and never block a push or count as critical. |
| 12 | //! - Dependencies. On every push to a default branch, and daily, the |
| 13 | //! lockfiles are read and every package checked against OSV. Each |
| 14 | //! vulnerable package with a fix gets a security update: g1t itself |
| 15 | //! (`User::system`) opens a pull request raising its version, made in a |
| 16 | //! sandbox (the runner's `bump`), which lands through the branch's |
| 17 | //! required checks. Only when code has to change is g1t put on an |
| 18 | //! issue for it. |
| 19 | //! |
| 20 | //! - The security suite (`g1t_contracts::security_suite`): custom secret |
| 21 | //! patterns (`patterns`), push protection bypasses, their review and |
| 22 | //! validity checks (`secret_alerts`), code scanning from SARIF uploads |
| 23 | //! and its pull request check (`code_scanning`), the dependency graph, |
| 24 | //! its SBOM and dependency review (`supply_chain`), "Fix with g1t" |
| 25 | //! (`fixes`), and settings with the workspace's overview (`overview`). |
| 26 | //! On private repositories its paid parts need the workspace's Security |
| 27 | //! and quality activation (`suite`); it tells people through events. |
| 28 | //! |
| 29 | //! Other services reach it over `POST /rpc/<method>`; see |
| 30 | //! `g1t_contracts::security`. |
| 31 | |
| 32 | mod code_scanning; |
| 33 | mod config; |
| 34 | mod deps; |
| 35 | mod fixes; |
| 36 | mod history; |
| 37 | mod manifests; |
| 38 | mod overview; |
| 39 | mod patterns; |
| 40 | mod planning; |
| 41 | mod pull_text; |
| 42 | mod resolved; |
| 43 | mod ranges; |
| 44 | mod registries; |
| 45 | mod schedule; |
| 46 | mod secret_alerts; |
| 47 | mod security_updates; |
| 48 | mod store; |
| 49 | mod suite; |
| 50 | mod suite_store; |
| 51 | mod supply_chain; |
| 52 | mod timezones; |
| 53 | mod update_store; |
| 54 | mod updates; |
| 55 | mod version_updates; |
| 56 | mod yaml; |
| 57 | |
| 58 | use g1t_contracts::access::{self, Capability}; |
| 59 | use g1t_contracts::events::{Event, WorkspaceRenamed}; |
| 60 | use g1t_contracts::security::UPDATE_BRANCH_PREFIX; |
| 61 | use g1t_contracts::repos::{GetArgs, PathByIdArgs, Repo, RepoPath, RepoStatus, StatusByIdArgs}; |
| 62 | use g1t_contracts::security::*; |
| 63 | use g1t_contracts::time::rfc3339; |
| 64 | use g1t_contracts::{FailureCode, Outcome, User}; |
| 65 | use g1t_kit::{args, now_ms, reply, rpc_method}; |
| 66 | use serde::Deserialize; |
| 67 | use worker::{Context, Env, Fetcher, MessageBatch, Request, Response, Result, ScheduleContext, ScheduledEvent, event}; |
| 68 | |
| 69 | use store::{Activity, RepoRow, Store}; |
| 70 | |
| 71 | /// Repositories whose history is continued per sweep, and pages each. |
| 72 | const HISTORIES_PER_SWEEP: u32 = 5; |
| 73 | const PAGES_PER_SWEEP: u32 = 4; |
| 74 | /// Repositories whose dependencies are read again per sweep. |
| 75 | const DEPENDENCIES_PER_SWEEP: u32 = 10; |
| 76 | const DAY_MS: u64 = 24 * 60 * 60 * 1000; |
| 77 | const MAX_REASON_CHARS: usize = 500; |
| 78 | /// What seeing and dismissing a repository's findings takes: the Write |
| 79 | /// role, as on GitHub, or a security manager of its workspace. Changing its |
| 80 | /// security settings takes Admin (`ManageSecurity`). |
| 81 | const SEE_FINDINGS: Capability = Capability::SecurityAlerts; |
| 82 | |
| 83 | pub struct Security { |
| 84 | store: Store, |
| 85 | identity: Fetcher, |
| 86 | repos: Fetcher, |
| 87 | work: Fetcher, |
| 88 | runner: Fetcher, |
| 89 | billing: Fetcher, |
| 90 | actions: Fetcher, |
| 91 | /// The events service: security events for webhooks and the inbox, |
| 92 | /// and audit entries. Optional so a deployment without the binding |
| 93 | /// still scans. |
| 94 | events: Option<Fetcher>, |
| 95 | } |
| 96 | |
| 97 | fn fail<T>(code: FailureCode, message: impl Into<String>) -> Outcome<T> { |
| 98 | Outcome::fail(code, message) |
| 99 | } |
| 100 | |
| 101 | /// `git.push`, as far as this service reads it. |
| 102 | #[derive(Deserialize)] |
| 103 | #[serde(rename_all = "camelCase")] |
| 104 | struct Pushed { |
| 105 | repo_id: String, |
| 106 | #[serde(default)] |
| 107 | default_branch: bool, |
| 108 | #[serde(default, rename = "ref")] |
| 109 | git_ref: String, |
| 110 | #[serde(default)] |
| 111 | before: Option<String>, |
| 112 | #[serde(default)] |
| 113 | after: String, |
| 114 | /// Too large to scan before it was stored. |
| 115 | #[serde(default)] |
| 116 | unscanned: bool, |
| 117 | } |
| 118 | |
| 119 | /// `pull.merged`, `pull.closed` and `checks.completed`, as far as this |
| 120 | /// service reads them. |
| 121 | #[derive(Deserialize)] |
| 122 | #[serde(rename_all = "camelCase")] |
| 123 | struct PullHappened { |
| 124 | repo_id: String, |
| 125 | number: u32, |
| 126 | #[serde(default)] |
| 127 | status: Option<String>, |
| 128 | } |
| 129 | |
| 130 | /// The longest comment a dismissal keeps. |
| 131 | const MAX_COMMENT_CHARS: usize = MAX_REASON_CHARS; |
| 132 | /// Activity rows the Security page reads, newest first. |
| 133 | const ACTIVITY_SHOWN: u32 = 500; |
| 134 | |
| 135 | #[derive(Deserialize)] |
| 136 | #[serde(rename_all = "camelCase")] |
| 137 | struct Created { |
| 138 | repo_id: String, |
| 139 | /// Absent from events older than the field. |
| 140 | #[serde(default)] |
| 141 | is_private: Option<bool>, |
| 142 | } |
| 143 | |
| 144 | impl Security { |
| 145 | fn new(env: &Env) -> Result<Self> { |
| 146 | Ok(Security { |
| 147 | store: Store { db: env.d1("DB")? }, |
| 148 | identity: env.service("IDENTITY")?, |
| 149 | repos: env.service("REPOS")?, |
| 150 | work: env.service("WORK")?, |
| 151 | runner: env.service("RUNNER")?, |
| 152 | billing: env.service("BILLING")?, |
| 153 | actions: env.service("ACTIONS")?, |
| 154 | events: env.service("EVENTS").ok(), |
| 155 | }) |
| 156 | } |
| 157 | |
| 158 | /// The repository at `path`, recorded here, if `viewer` may see its |
| 159 | /// findings (the Write role) and do `capability`. Findings are for those |
| 160 | /// who can change the code: to anyone else the page does not exist, |
| 161 | /// public repository or not. |
| 162 | async fn member_repo( |
| 163 | &self, |
| 164 | path: &RepoPath, |
| 165 | viewer: &Option<User>, |
| 166 | capability: Capability, |
| 167 | ) -> Result<Outcome<RepoRow>> { |
| 168 | let hidden = || fail(FailureCode::NotFound, "Repository not found."); |
| 169 | if viewer.is_none() { |
| 170 | return Ok(hidden()); |
| 171 | } |
| 172 | let repo: Outcome<Repo> = |
| 173 | g1t_kit::call(&self.repos, "get", &GetArgs { path: path.clone(), viewer: viewer.clone() }).await?; |
| 174 | let repo = match repo { |
| 175 | Outcome::Ok(repo) if repo.fork_of.is_none() => repo, |
| 176 | _ => return Ok(hidden()), |
| 177 | }; |
| 178 | if !access::can(viewer.as_ref(), &repo, SEE_FINDINGS) { |
| 179 | return Ok(hidden()); |
| 180 | } |
| 181 | if !access::can(viewer.as_ref(), &repo, capability) { |
| 182 | return Ok(fail( |
| 183 | FailureCode::Forbidden, |
| 184 | access::needs(capability, &format!("{}/{}", repo.namespace, repo.name)), |
| 185 | )); |
| 186 | } |
| 187 | let row = self.store.register(&repo.id, &repo.namespace, &repo.name).await?; |
| 188 | self.store.set_private(&repo.id, repo.is_private).await?; |
| 189 | Ok(Outcome::Ok(row)) |
| 190 | } |
| 191 | |
| 192 | /// Where the repository stands: archived, deleted, a mirror. When repos |
| 193 | /// cannot say, it is taken as an ordinary active repository. |
| 194 | async fn status(&self, repo_id: &str) -> Result<RepoStatus> { |
| 195 | let status: Result<RepoStatus> = |
| 196 | g1t_kit::call(&self.repos, "status_by_id", &StatusByIdArgs { id: repo_id.to_owned() }).await; |
| 197 | Ok(match status { |
| 198 | Ok(status) => status, |
| 199 | Err(error) => { |
| 200 | worker::console_error!("security: status_by_id {repo_id}: {error}"); |
| 201 | RepoStatus::default() |
| 202 | } |
| 203 | }) |
| 204 | } |
| 205 | |
| 206 | /// Whether the repository is neither archived nor deleted. Scanning |
| 207 | /// goes on in a mirror: it reads. |
| 208 | async fn active(&self, repo_id: &str) -> Result<bool> { |
| 209 | Ok(self.status(repo_id).await?.active()) |
| 210 | } |
| 211 | |
| 212 | /// Whether g1t may open branches and pull requests here on its own |
| 213 | /// (security and dependency updates): active, and not a mirror short |
| 214 | /// of a takeover (`g1t_contracts::mirrors`). `Err` says why not. |
| 215 | async fn takes_updates(&self, repo: &RepoRow) -> Result<std::result::Result<(), String>> { |
| 216 | let status = self.status(&repo.repo_id).await?; |
| 217 | if status.takes_updates() { |
| 218 | return Ok(Ok(())); |
| 219 | } |
| 220 | Ok(Err(match &status.mirror { |
| 221 | Some(mirror) if status.active() => g1t_contracts::mirrors::updates_wait_message(&repo.namespace, &repo.name, mirror), |
| 222 | _ => "Updates wait while the repository is archived or deleted.".to_owned(), |
| 223 | })) |
| 224 | } |
| 225 | |
| 226 | /// Records a repository named in an event, by id. Forks are not |
| 227 | /// recorded: a pull request's findings belong to its repository. |
| 228 | async fn register_by_id(&self, repo_id: &str) -> Result<Option<RepoRow>> { |
| 229 | if let Some(row) = self.store.repo(repo_id).await? { |
| 230 | return Ok(Some(row)); |
| 231 | } |
| 232 | let path: Option<RepoPath> = g1t_kit::call(&self.repos, "path_by_id", &PathByIdArgs { id: repo_id.to_owned() }).await?; |
| 233 | match path { |
| 234 | Some(path) => Ok(Some(self.store.register(repo_id, &path.namespace, &path.name).await?)), |
| 235 | None => Ok(None), |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | async fn overview(&self, a: OverviewArgs) -> Result<Outcome<SecurityOverview>> { |
| 240 | let mut repo = match self.member_repo(&a.repo, &a.viewer, SEE_FINDINGS).await? { |
| 241 | Outcome::Ok(repo) => repo, |
| 242 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 243 | }; |
| 244 | // The first look at a repository reads its dependencies at once; |
| 245 | // its history is scanned in the background. |
| 246 | if repo.deps_scanned_at.is_none() { |
| 247 | self.scan_dependencies(&repo).await?; |
| 248 | repo = self.store.repo(&repo.repo_id).await?.unwrap_or(repo); |
| 249 | } |
| 250 | let (counts, _, _) = self.store.counts(&repo.repo_id).await?; |
| 251 | Ok(Outcome::Ok(SecurityOverview { |
| 252 | repo_id: repo.repo_id.clone(), |
| 253 | counts, |
| 254 | secret_counts: self.store.secret_counts(&repo.repo_id).await?, |
| 255 | secrets: self.store.secrets(&repo.repo_id).await?, |
| 256 | vulnerabilities: self.store.vulnerabilities(&repo.repo_id).await?, |
| 257 | activity: self.store.activity(&repo.repo_id, ACTIVITY_SHOWN).await?, |
| 258 | scan: repo.scan_state(), |
| 259 | upkeep: repo.upkeep != 0, |
| 260 | version_updates: self.version_updates_view(&repo).await?, |
| 261 | })) |
| 262 | } |
| 263 | |
| 264 | /// What dismissing or reopening alert `id` takes: Write, for a secret |
| 265 | /// as for a vulnerable dependency, as on GitHub; a security manager may |
| 266 | /// too. |
| 267 | fn capability_for(_id: &str) -> Capability { |
| 268 | SEE_FINDINGS |
| 269 | } |
| 270 | |
| 271 | async fn dismiss(&self, a: DismissArgs) -> Result<Outcome<AlertChange>> { |
| 272 | let repo = match self.member_repo(&a.repo, &Some(a.actor.clone()), Self::capability_for(&a.id)).await? { |
| 273 | Outcome::Ok(repo) => repo, |
| 274 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 275 | }; |
| 276 | if !a.actor.verified { |
| 277 | return Ok(fail(FailureCode::Forbidden, "Confirm your email address first.")); |
| 278 | } |
| 279 | let comment: String = a.comment.trim().chars().take(MAX_COMMENT_CHARS).collect(); |
| 280 | let comment = (!comment.is_empty()).then_some(comment); |
| 281 | if let Some(finding) = self.store.secret(&repo.repo_id, &a.id).await? { |
| 282 | if !a.reason.for_secrets() { |
| 283 | return Ok(fail( |
| 284 | FailureCode::Invalid, |
| 285 | "A secret is dismissed as false_positive, used_in_tests, revoked or wont_fix.", |
| 286 | )); |
| 287 | } |
| 288 | if finding.state != AlertState::Open { |
| 289 | return Ok(fail(FailureCode::Conflict, "This alert is not open. Reopen it first to dismiss it again.")); |
| 290 | } |
| 291 | self.store |
| 292 | .dismiss_secret(&repo.repo_id, &a.id, a.reason, &a.actor.username, comment.as_deref()) |
| 293 | .await?; |
| 294 | self.store |
| 295 | .record(&repo.repo_id, &[Activity { |
| 296 | alert_id: &a.id, |
| 297 | action: "dismissed", |
| 298 | actor: Some(&a.actor.username), |
| 299 | reason: Some(a.reason), |
| 300 | comment: comment.as_deref(), |
| 301 | number: None, |
| 302 | }]) |
| 303 | .await?; |
| 304 | let secret = self.store.secret(&repo.repo_id, &a.id).await?; |
| 305 | if let Some(secret) = &secret { |
| 306 | // Revoked is fixed; any other reason, dismissed. |
| 307 | let action = if a.reason == DismissReason::Revoked { "fixed" } else { "dismissed" }; |
| 308 | let event = g1t_contracts::security_suite::SecurityEvent { |
| 309 | reason: Some(a.reason.as_str().to_owned()), |
| 310 | ..secret_alerts::secret_event(&repo, secret) |
| 311 | }; |
| 312 | self.alert_event(g1t_contracts::security_suite::AlertType::SecretScanning, action, &repo, event, Some(a.actor.id.clone())) |
| 313 | .await; |
| 314 | } |
| 315 | return Ok(Outcome::Ok(AlertChange { secret, vulnerability: None })); |
| 316 | } |
| 317 | let Some(vuln) = self.store.vulnerability(&repo.repo_id, &a.id).await? else { |
| 318 | return Ok(fail(FailureCode::NotFound, "No such alert.")); |
| 319 | }; |
| 320 | if a.reason.for_secrets() { |
| 321 | return Ok(fail( |
| 322 | FailureCode::Invalid, |
| 323 | "A dependency is dismissed as fix_started, no_bandwidth, tolerable_risk, inaccurate or not_used.", |
| 324 | )); |
| 325 | } |
| 326 | if vuln.state != AlertState::Open { |
| 327 | return Ok(fail(FailureCode::Conflict, "This alert is not open. Reopen it first to dismiss it again.")); |
| 328 | } |
| 329 | self.store |
| 330 | .dismiss_vulnerability(&repo.repo_id, &a.id, a.reason, &a.actor.username, comment.as_deref()) |
| 331 | .await?; |
| 332 | self.store |
| 333 | .record(&repo.repo_id, &[Activity { |
| 334 | alert_id: &a.id, |
| 335 | action: "dismissed", |
| 336 | actor: Some(&a.actor.username), |
| 337 | reason: Some(a.reason), |
| 338 | comment: comment.as_deref(), |
| 339 | number: None, |
| 340 | }]) |
| 341 | .await?; |
| 342 | let vulnerability = self.store.vulnerability(&repo.repo_id, &a.id).await?; |
| 343 | if let Some(vuln) = &vulnerability { |
| 344 | let event = g1t_contracts::security_suite::SecurityEvent { reason: Some(a.reason.as_str().to_owned()), ..deps::vulnerability_event(&repo, vuln) }; |
| 345 | self.alert_event(g1t_contracts::security_suite::AlertType::Vulnerability, "dismissed", &repo, event, Some(a.actor.id.clone())).await; |
| 346 | } |
| 347 | // A security update whose alerts are now all fixed or dismissed closes. |
| 348 | if let Err(error) = self.resolve_updates(&repo, None).await { |
| 349 | worker::console_error!("security: updates of {} not resolved after a dismissal: {error}", repo.repo_id); |
| 350 | } |
| 351 | Ok(Outcome::Ok(AlertChange { secret: None, vulnerability })) |
| 352 | } |
| 353 | |
| 354 | async fn reopen(&self, a: ReopenArgs) -> Result<Outcome<AlertChange>> { |
| 355 | let repo = match self.member_repo(&a.repo, &Some(a.actor.clone()), Self::capability_for(&a.id)).await? { |
| 356 | Outcome::Ok(repo) => repo, |
| 357 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 358 | }; |
| 359 | if !a.actor.verified { |
| 360 | return Ok(fail(FailureCode::Forbidden, "Confirm your email address first.")); |
| 361 | } |
| 362 | let reopened = Activity { alert_id: &a.id, action: "reopened", actor: Some(&a.actor.username), reason: None, comment: None, number: None }; |
| 363 | if let Some(finding) = self.store.secret(&repo.repo_id, &a.id).await? { |
| 364 | if finding.state == AlertState::Open { |
| 365 | return Ok(fail(FailureCode::Conflict, "This alert is already open.")); |
| 366 | } |
| 367 | // A secret that never landed has nothing to reopen to but blocked. |
| 368 | let status = if finding.source == "push" && finding.test_value.is_none() { SecretStatus::Blocked } else { SecretStatus::Open }; |
| 369 | self.store.reopen_secret(&repo.repo_id, &a.id, status).await?; |
| 370 | self.store.record(&repo.repo_id, &[reopened]).await?; |
| 371 | let secret = self.store.secret(&repo.repo_id, &a.id).await?; |
| 372 | if let Some(secret) = &secret { |
| 373 | self.alert_event( |
| 374 | g1t_contracts::security_suite::AlertType::SecretScanning, |
| 375 | "reopened", |
| 376 | &repo, |
| 377 | secret_alerts::secret_event(&repo, secret), |
| 378 | Some(a.actor.id.clone()), |
| 379 | ) |
| 380 | .await; |
| 381 | } |
| 382 | return Ok(Outcome::Ok(AlertChange { secret, vulnerability: None })); |
| 383 | } |
| 384 | let Some(vuln) = self.store.vulnerability(&repo.repo_id, &a.id).await? else { |
| 385 | return Ok(fail(FailureCode::NotFound, "No such alert.")); |
| 386 | }; |
| 387 | if vuln.state != AlertState::Dismissed { |
| 388 | return Ok(fail(FailureCode::Conflict, "Only a dismissed alert can be reopened; a fixed one reopens when it is found again.")); |
| 389 | } |
| 390 | self.store.reopen_vulnerability(&repo.repo_id, &a.id).await?; |
| 391 | self.store.record(&repo.repo_id, &[reopened]).await?; |
| 392 | let vulnerability = self.store.vulnerability(&repo.repo_id, &a.id).await?; |
| 393 | if let Some(vuln) = &vulnerability { |
| 394 | self.alert_event( |
| 395 | g1t_contracts::security_suite::AlertType::Vulnerability, |
| 396 | "reopened", |
| 397 | &repo, |
| 398 | deps::vulnerability_event(&repo, vuln), |
| 399 | Some(a.actor.id.clone()), |
| 400 | ) |
| 401 | .await; |
| 402 | } |
| 403 | Ok(Outcome::Ok(AlertChange { secret: None, vulnerability })) |
| 404 | } |
| 405 | |
| 406 | async fn rescan(&self, a: RescanArgs) -> Result<Outcome<ScanState>> { |
| 407 | let repo = match self.member_repo(&a.repo, &Some(a.actor.clone()), SEE_FINDINGS).await? { |
| 408 | Outcome::Ok(repo) => repo, |
| 409 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 410 | }; |
| 411 | self.store.restart_history(&repo.repo_id).await?; |
| 412 | self.scan_dependencies(&repo).await?; |
| 413 | if let Some(fresh) = self.store.repo(&repo.repo_id).await? { |
| 414 | self.advance_history(&fresh, 1).await?; |
| 415 | } |
| 416 | let state = self.store.repo(&repo.repo_id).await?.map(|row| row.scan_state()).unwrap_or_default(); |
| 417 | Ok(Outcome::Ok(state)) |
| 418 | } |
| 419 | |
| 420 | async fn set_upkeep(&self, a: SetUpkeepArgs) -> Result<Outcome<bool>> { |
| 421 | // Whether agents keep its dependencies up to date is one of its |
| 422 | // settings. |
| 423 | let repo = match self.member_repo(&a.repo, &Some(a.actor.clone()), Capability::ManageSettings).await? { |
| 424 | Outcome::Ok(repo) => repo, |
| 425 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 426 | }; |
| 427 | if !a.actor.verified { |
| 428 | return Ok(fail(FailureCode::Forbidden, "Confirm your email address first.")); |
| 429 | } |
| 430 | self.store.set_upkeep(&repo.repo_id, a.enabled, &a.actor.username).await?; |
| 431 | Ok(Outcome::Ok(a.enabled)) |
| 432 | } |
| 433 | |
| 434 | async fn workspace(&self, a: WorkspaceArgs) -> Result<Outcome<Vec<RepoSecurity>>> { |
| 435 | let workspace = a.workspace.to_lowercase(); |
| 436 | if !a.viewer.as_ref().is_some_and(|user| user.is_member(&workspace)) { |
| 437 | return Ok(fail(FailureCode::NotFound, "Workspace not found.")); |
| 438 | } |
| 439 | // Only the repositories whose findings the viewer may see. The |
| 440 | // Write role is never had through being public, so treating each |
| 441 | // as private changes nothing. |
| 442 | let repos: Vec<RepoRow> = self |
| 443 | .store |
| 444 | .in_namespace(&workspace) |
| 445 | .await? |
| 446 | .into_iter() |
| 447 | .filter(|repo| { |
| 448 | let target = access::RepoRef { id: &repo.repo_id, namespace: &workspace, private: true }; |
| 449 | access::can(a.viewer.as_ref(), target, SEE_FINDINGS) |
| 450 | }) |
| 451 | .collect(); |
| 452 | // Every repository's counts at once, not one after another. |
| 453 | let counts = futures_util::future::try_join_all(repos.iter().map(|repo| self.store.counts(&repo.repo_id))).await?; |
| 454 | let list = repos |
| 455 | .into_iter() |
| 456 | .zip(counts) |
| 457 | .map(|(repo, (counts, secrets, vulnerabilities))| RepoSecurity { |
| 458 | repo_id: repo.repo_id, |
| 459 | name: repo.name, |
| 460 | counts, |
| 461 | secrets, |
| 462 | vulnerabilities, |
| 463 | upkeep: repo.upkeep != 0, |
| 464 | dependencies_scanned_at: repo.deps_scanned_at, |
| 465 | }) |
| 466 | .collect(); |
| 467 | Ok(Outcome::Ok(list)) |
| 468 | } |
| 469 | |
| 470 | /// Push protection's question: which of these secrets were allowed? |
| 471 | /// The others are recorded as blocked, so someone can allow them. |
| 472 | async fn push_blocked(&self, a: PushBlockedArgs) -> Result<PushVerdict> { |
| 473 | let repo = self.store.register(&a.repo_id, &a.path.namespace, &a.path.name).await?; |
| 474 | if let Some(private) = a.private { |
| 475 | self.store.set_private(&a.repo_id, private).await?; |
| 476 | } |
| 477 | let fingerprints: Vec<String> = a.secrets.iter().map(|secret| secret.fingerprint.clone()).collect(); |
| 478 | let known = self.store.known(&a.repo_id, &fingerprints).await?; |
| 479 | let mut allowed = let_through(&known, &a.secrets); |
| 480 | // Bypassed with a reason: let through, whatever the alert says now. |
| 481 | allowed.extend(self.store.bypassed(&a.repo_id, &fingerprints).await?); |
| 482 | allowed.sort(); |
| 483 | allowed.dedup(); |
| 484 | let all = a.secrets.clone(); |
| 485 | let fresh: Vec<NewSecret> = a |
| 486 | .secrets |
| 487 | .into_iter() |
| 488 | .filter(|secret| !known.iter().any(|(fingerprint, _, _)| *fingerprint == secret.fingerprint)) |
| 489 | .collect(); |
| 490 | // A likely test value goes through, so it lands: open, not blocked. |
| 491 | let (tests, real): (Vec<NewSecret>, Vec<NewSecret>) = fresh.into_iter().partition(|secret| secret.test_value.is_some()); |
| 492 | self.store |
| 493 | .add_secrets(&a.repo_id, &real, SecretStatus::Blocked, "push", a.pusher.as_deref()) |
| 494 | .await?; |
| 495 | self.store |
| 496 | .add_secrets(&a.repo_id, &tests, SecretStatus::Open, "push", a.pusher.as_deref()) |
| 497 | .await?; |
| 498 | let fresh: Vec<String> = real.iter().map(|secret| secret.fingerprint.clone()).collect(); |
| 499 | self.secrets_found(&repo, &all, "push", &fresh, a.pusher.as_deref()).await?; |
| 500 | let ids = self |
| 501 | .store |
| 502 | .known(&a.repo_id, &fingerprints) |
| 503 | .await? |
| 504 | .into_iter() |
| 505 | .filter(|(fingerprint, _, _)| !allowed.contains(fingerprint)) |
| 506 | .map(|(fingerprint, id, _)| (fingerprint, id)) |
| 507 | .collect(); |
| 508 | Ok(PushVerdict { allowed, ids }) |
| 509 | } |
| 510 | |
| 511 | /// What happens on the bus that concerns this service. |
| 512 | async fn on_event(&self, event: &Event) -> Result<()> { |
| 513 | match event.kind.as_str() { |
| 514 | "git.push" => { |
| 515 | let Ok(pushed) = serde_json::from_value::<Pushed>(event.data.clone()) else { |
| 516 | return Ok(()); |
| 517 | }; |
| 518 | // Too large to scan before it was stored: its new commits |
| 519 | // are scanned now, on whichever branch. |
| 520 | if pushed.unscanned |
| 521 | && !pushed.after.is_empty() |
| 522 | && let Some(repo) = self.register_by_id(&pushed.repo_id).await? |
| 523 | { |
| 524 | let id = self |
| 525 | .store |
| 526 | .add_push_scan(&repo.repo_id, &pushed.git_ref, &pushed.after, pushed.before.as_deref(), event.actor.as_deref()) |
| 527 | .await?; |
| 528 | self.advance_push_scan(&id, history::PUSH_PAGES_AT_ONCE).await?; |
| 529 | } |
| 530 | // A version update's branch, or a grouped security update's, |
| 531 | // pushed by its sandbox: time for its pull request. |
| 532 | if !pushed.default_branch |
| 533 | && let Some(branch) = pushed.git_ref.strip_prefix("refs/heads/") |
| 534 | && self.update_pull_pushed(&pushed.repo_id, branch, &pushed.after).await? |
| 535 | { |
| 536 | return Ok(()); |
| 537 | } |
| 538 | // A security update's branch, pushed by its sandbox: time for |
| 539 | // its pull request. |
| 540 | if let Some(branch) = pushed.git_ref.strip_prefix("refs/heads/") |
| 541 | && branch.starts_with(UPDATE_BRANCH_PREFIX) |
| 542 | { |
| 543 | self.update_pushed(&pushed.repo_id, branch, &pushed.after).await?; |
| 544 | return Ok(()); |
| 545 | } |
| 546 | if !pushed.default_branch { |
| 547 | return Ok(()); |
| 548 | } |
| 549 | if let Some(repo) = self.register_by_id(&pushed.repo_id).await? { |
| 550 | self.scan_dependencies(&repo).await?; |
| 551 | if repo.history != "done" { |
| 552 | self.advance_history(&repo, 1).await?; |
| 553 | } |
| 554 | } |
| 555 | } |
| 556 | "pull.merged" | "pull.closed" | "checks.completed" => { |
| 557 | if let Ok(happened) = serde_json::from_value::<PullHappened>(event.data.clone()) |
| 558 | && !(event.kind != "checks.completed" && self.update_pull_closed(&event.kind, &happened.repo_id, happened.number).await?) |
| 559 | { |
| 560 | self.update_pull_event( |
| 561 | &event.kind, |
| 562 | &happened.repo_id, |
| 563 | happened.number, |
| 564 | happened.status.as_deref(), |
| 565 | event.actor.as_deref(), |
| 566 | ) |
| 567 | .await?; |
| 568 | } |
| 569 | // One of g1t's update pull requests closed or merged, by g1t |
| 570 | // or by a person: its branch goes. |
| 571 | if event.kind != "checks.completed" |
| 572 | && let Ok(happened) = serde_json::from_value::<PullHappened>(event.data.clone()) |
| 573 | { |
| 574 | self.pull_finished(&happened.repo_id, happened.number).await?; |
| 575 | } |
| 576 | } |
| 577 | "comment.created" => self.update_comment(event).await?, |
| 578 | // A pull request opened or its head moved: a dependency update |
| 579 | // file it changes is checked (not on `pull.ready`, which moves |
| 580 | // nothing), and dependency review runs. |
| 581 | "pull.opened" | "pull.updated" | "pull.ready" => { |
| 582 | if event.kind != "pull.ready" { |
| 583 | self.check_dependabot_file(event).await?; |
| 584 | } |
| 585 | if let Ok(happened) = serde_json::from_value::<PullHappened>(event.data.clone()) { |
| 586 | self.review_pull(&happened.repo_id, happened.number).await?; |
| 587 | } |
| 588 | } |
| 589 | "repo.created" => { |
| 590 | if let Ok(created) = serde_json::from_value::<Created>(event.data.clone()) { |
| 591 | // Recorded with its visibility, so the overview never takes a |
| 592 | // public repository for a private one. |
| 593 | if self.register_by_id(&created.repo_id).await?.is_some() |
| 594 | && let Some(private) = created.is_private |
| 595 | { |
| 596 | self.store.set_private(&created.repo_id, private).await?; |
| 597 | } |
| 598 | } |
| 599 | } |
| 600 | "repo.visibility_changed" => { |
| 601 | if let Ok(changed) = serde_json::from_value::<g1t_contracts::events::RepoVisibilityChanged>(event.data.clone()) |
| 602 | && self.store.repo(&changed.repo_id).await?.is_some() |
| 603 | { |
| 604 | self.store.set_private(&changed.repo_id, changed.is_private).await?; |
| 605 | } |
| 606 | } |
| 607 | // A repository transferred or renamed: it is recorded under its new path. |
| 608 | "repo.transferred" | "repo.renamed" => { |
| 609 | if let Some(moved) = g1t_kit::transfer::read(event) { |
| 610 | // Where it is now, so moves heard out of order end in |
| 611 | // the same place. |
| 612 | let now: Option<g1t_contracts::repos::RepoPath> = g1t_kit::call( |
| 613 | &self.repos, |
| 614 | "path_by_id", |
| 615 | &g1t_contracts::repos::PathByIdArgs { id: moved.repo_id.clone() }, |
| 616 | ) |
| 617 | .await?; |
| 618 | let current = now.map_or_else( |
| 619 | || moved.destination().to_owned(), |
| 620 | |path| format!("{}/{}", path.namespace, path.name), |
| 621 | ); |
| 622 | if let Some((namespace, name)) = current.split_once('/') { |
| 623 | self.store.moved(&moved.repo_id, namespace, name).await?; |
| 624 | } |
| 625 | } |
| 626 | } |
| 627 | // A repository purged: everything found in it goes. |
| 628 | "repo.purged" => { |
| 629 | if let Some(g1t_kit::lifecycle::Lifecycle::Purged(purged)) = g1t_kit::lifecycle::read(event) { |
| 630 | self.store.purge_suite(&purged.repo_id).await?; |
| 631 | self.store.purge(&purged.repo_id).await?; |
| 632 | } |
| 633 | } |
| 634 | "workspace.renamed" => { |
| 635 | if let Ok(renamed) = serde_json::from_value::<WorkspaceRenamed>(event.data.clone()) { |
| 636 | self.store.rename_namespace(&renamed.stale_slugs(&renamed.to), &renamed.to).await?; |
| 637 | } |
| 638 | } |
| 639 | _ => {} |
| 640 | } |
| 641 | Ok(()) |
| 642 | } |
| 643 | |
| 644 | /// The sweep: continues history scans and scans of pushes that landed |
| 645 | /// unscanned, catches security updates whose sandbox never pushed, and |
| 646 | /// reads dependencies that have not been read for a day. |
| 647 | async fn sweep(&self) -> Result<()> { |
| 648 | for scan in self.store.pending_push_scans(HISTORIES_PER_SWEEP).await? { |
| 649 | if let Err(error) = self.advance_push_scan(&scan.id, PAGES_PER_SWEEP).await { |
| 650 | worker::console_error!("security: push scan {} not continued: {error}", scan.id); |
| 651 | } |
| 652 | } |
| 653 | if let Err(error) = self.stalled_updates().await { |
| 654 | worker::console_error!("security: stalled security updates not handled: {error}"); |
| 655 | } |
| 656 | if let Err(error) = self.sweep_suite().await { |
| 657 | worker::console_error!("security: daily snapshots and validity checks: {error}"); |
| 658 | } |
| 659 | // Archived and deleted repositories wait; a few more are looked at |
| 660 | // so that they do not hold up the rest. |
| 661 | let mut histories = 0; |
| 662 | for repo in self.store.unfinished_histories(HISTORIES_PER_SWEEP * 4).await? { |
| 663 | if histories == HISTORIES_PER_SWEEP { |
| 664 | break; |
| 665 | } |
| 666 | if !self.active(&repo.repo_id).await? { |
| 667 | continue; |
| 668 | } |
| 669 | histories += 1; |
| 670 | if let Err(error) = self.advance_history(&repo, PAGES_PER_SWEEP).await { |
| 671 | worker::console_error!("security: history of {} not scanned: {error}", repo.repo_id); |
| 672 | } |
| 673 | } |
| 674 | let day_ago = rfc3339(now_ms().saturating_sub(DAY_MS)); |
| 675 | for repo in self.store.stale_dependencies(&day_ago, DEPENDENCIES_PER_SWEEP).await? { |
| 676 | if !self.active(&repo.repo_id).await? { |
| 677 | self.store |
| 678 | .skip_dependencies(&repo.repo_id, "Dependencies are not checked while the repository is archived or deleted.") |
| 679 | .await?; |
| 680 | continue; |
| 681 | } |
| 682 | if let Err(error) = self.scan_dependencies(&repo).await { |
| 683 | worker::console_error!("security: dependencies of {} not read: {error}", repo.repo_id); |
| 684 | } |
| 685 | } |
| 686 | Ok(()) |
| 687 | } |
| 688 | } |
| 689 | |
| 690 | /// The fingerprints a push may carry: those someone dismissed (allowed), |
| 691 | /// and likely test values, which are recorded but never stop a push. |
| 692 | /// `known` is (fingerprint, id, status) of findings already recorded. |
| 693 | fn let_through(known: &[(String, String, String)], secrets: &[NewSecret]) -> Vec<String> { |
| 694 | let mut allowed: Vec<String> = known |
| 695 | .iter() |
| 696 | .filter(|(_, _, status)| status == "allowed") |
| 697 | .map(|(fingerprint, _, _)| fingerprint.clone()) |
| 698 | .chain(secrets.iter().filter(|secret| secret.test_value.is_some()).map(|secret| secret.fingerprint.clone())) |
| 699 | .collect(); |
| 700 | allowed.sort(); |
| 701 | allowed.dedup(); |
| 702 | allowed |
| 703 | } |
| 704 | |
| 705 | #[event(fetch)] |
| 706 | async fn fetch(mut request: Request, env: Env, _ctx: Context) -> Result<Response> { |
| 707 | let Some(method) = rpc_method(&request) else { |
| 708 | return Response::error("Not found", 404); |
| 709 | }; |
| 710 | let security = Security::new(&env)?; |
| 711 | let body: serde_json::Value = request.json().await?; |
| 712 | match method.as_str() { |
| 713 | "overview" => reply(&security.overview(args(body)?).await?), |
| 714 | "dismiss" => reply(&security.dismiss(args(body)?).await?), |
| 715 | "reopen" => reply(&security.reopen(args(body)?).await?), |
| 716 | "rescan" => reply(&security.rescan(args(body)?).await?), |
| 717 | "set_upkeep" => reply(&security.set_upkeep(args(body)?).await?), |
| 718 | "workspace" => reply(&security.workspace(args(body)?).await?), |
| 719 | "push_blocked" => reply(&security.push_blocked(args(body)?).await?), |
| 720 | "check_updates" => reply(&security.check_updates(args(body)?).await?), |
| 721 | // The security suite. |
| 722 | "patterns_for" => reply(&security.patterns_for(args(body)?).await?), |
| 723 | "custom_patterns" => reply(&security.custom_patterns(args(body)?).await?), |
| 724 | "save_custom_pattern" => reply(&security.save_custom_pattern(args(body)?).await?), |
| 725 | "delete_custom_pattern" => reply(&security.delete_custom_pattern(args(body)?).await?), |
| 726 | "dry_run_pattern" => reply(&security.dry_run_pattern(args(body)?).await?), |
| 727 | "secret_alert" => reply(&security.secret_alert(args(body)?).await?), |
| 728 | "bypass" => reply(&security.bypass(args(body)?).await?), |
| 729 | "bypass_requests" => reply(&security.bypass_requests(args(body)?).await?), |
| 730 | "review_bypass" => reply(&security.review_bypass(args(body)?).await?), |
| 731 | "check_validity" => reply(&security.check_validity(args(body)?).await?), |
| 732 | "upload_sarif" => reply(&security.upload_sarif(args(body)?).await?), |
| 733 | "sarif_status" => reply(&security.sarif_status(args(body)?).await?), |
| 734 | "code_scanning" => reply(&security.code_scanning(args(body)?).await?), |
| 735 | "code_alert" => reply(&security.code_alert(args(body)?).await?), |
| 736 | "set_code_alert_state" => reply(&security.set_code_alert_state(args(body)?).await?), |
| 737 | "pull_code_scanning" => reply(&security.pull_code_scanning(args(body)?).await?), |
| 738 | "fix_alert" => reply(&security.fix_alert(args(body)?).await?), |
| 739 | "dependency_graph" => reply(&security.dependency_graph(args(body)?).await?), |
| 740 | "sbom" => reply(&security.sbom(args(body)?).await?), |
| 741 | "dependency_review" => reply(&security.dependency_review(args(body)?).await?), |
| 742 | "security_settings" => reply(&security.security_settings(args(body)?).await?), |
| 743 | "set_security_settings" => reply(&security.set_security_settings(args(body)?).await?), |
| 744 | "workspace_security_settings" => reply(&security.workspace_security_settings(args(body)?).await?), |
| 745 | "set_workspace_security_settings" => reply(&security.set_workspace_security_settings(args(body)?).await?), |
| 746 | "security_overview" => reply(&security.security_overview(args(body)?).await?), |
| 747 | "workspace_alerts" => reply(&security.workspace_alerts(args(body)?).await?), |
| 748 | _ => Response::error("Unknown method", 404), |
| 749 | } |
| 750 | } |
| 751 | |
| 752 | #[event(queue)] |
| 753 | async fn queue(batch: MessageBatch<Event>, env: Env, _ctx: Context) -> Result<()> { |
| 754 | let security = Security::new(&env)?; |
| 755 | for message in batch.messages()? { |
| 756 | let event = message.body(); |
| 757 | if let Err(error) = security.on_event(event).await { |
| 758 | // Scans are idempotent and the sweep catches up, so one failed |
| 759 | // event is logged rather than retried. |
| 760 | worker::console_error!("security: {} {} failed: {error}", event.kind, event.id); |
| 761 | } |
| 762 | } |
| 763 | Ok(()) |
| 764 | } |
| 765 | |
| 766 | /// The cron (wrangler.jsonc) that runs version updates that are due. |
| 767 | const VERSION_UPDATES_CRON: &str = "*/5 * * * *"; |
| 768 | |
| 769 | #[event(scheduled)] |
| 770 | async fn scheduled(event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { |
| 771 | match Security::new(&env) { |
| 772 | // Version updates run every few minutes, so a schedule's time is kept. |
| 773 | Ok(security) if event.cron() == VERSION_UPDATES_CRON => { |
| 774 | if let Err(error) = security.version_update_sweep().await { |
| 775 | worker::console_error!("security: the version update sweep failed: {error}"); |
| 776 | } |
| 777 | } |
| 778 | Ok(security) => { |
| 779 | if let Err(error) = security.sweep().await { |
| 780 | worker::console_error!("security: the sweep failed: {error}"); |
| 781 | } |
| 782 | } |
| 783 | Err(error) => worker::console_error!("security: could not start: {error}"), |
| 784 | } |
| 785 | } |
| 786 | |
| 787 | #[cfg(test)] |
| 788 | mod tests { |
| 789 | use super::*; |
| 790 | |
| 791 | fn secret(fingerprint: &str, test_value: Option<&str>) -> NewSecret { |
| 792 | NewSecret { |
| 793 | fingerprint: fingerprint.into(), |
| 794 | kind: "aws_access_key".into(), |
| 795 | path: "a.env".into(), |
| 796 | line: 1, |
| 797 | commit: "c".into(), |
| 798 | preview: "AKIA…".into(), |
| 799 | test_value: test_value.map(str::to_owned), |
| 800 | pattern_id: None, |
| 801 | pattern_name: None, |
| 802 | } |
| 803 | } |
| 804 | |
| 805 | #[test] |
| 806 | fn dismissed_secrets_and_test_values_go_through() { |
| 807 | let known = vec![ |
| 808 | ("allowed".to_owned(), "sec_1".to_owned(), "allowed".to_owned()), |
| 809 | ("resolved".to_owned(), "sec_2".to_owned(), "resolved".to_owned()), |
| 810 | ("blocked".to_owned(), "sec_3".to_owned(), "blocked".to_owned()), |
| 811 | ]; |
| 812 | let secrets = [secret("allowed", None), secret("resolved", None), secret("example", Some("it says it is an example")), secret("real", None)]; |
| 813 | assert_eq!(let_through(&known, &secrets), ["allowed", "example"]); |
| 814 | } |
| 815 | |
| 816 | #[test] |
| 817 | fn a_push_says_when_it_landed_unscanned() { |
| 818 | let pushed: Pushed = serde_json::from_value(serde_json::json!({ |
| 819 | "repoId": "rep_1", "ref": "refs/heads/import", "after": "abc", "defaultBranch": false, "unscanned": true |
| 820 | })) |
| 821 | .unwrap(); |
| 822 | assert!(pushed.unscanned && pushed.before.is_none() && pushed.git_ref == "refs/heads/import"); |
| 823 | let ordinary: Pushed = serde_json::from_value(serde_json::json!({ "repoId": "rep_1", "ref": "refs/heads/main", "after": "abc", "defaultBranch": true })).unwrap(); |
| 824 | assert!(!ordinary.unscanned); |
| 825 | } |
| 826 | |
| 827 | #[test] |
| 828 | fn owners_are_told_what_landed_and_what_to_do() { |
| 829 | let one = history::landed_secrets_intro("acme", "rocket", "import", 1); |
| 830 | assert!(one.contains("A push to import in acme/rocket") && one.contains("a secret that looks real") && one.contains("Rotate")); |
| 831 | assert!(history::landed_secrets_intro("acme", "rocket", "main", 3).contains("3 secrets that look real")); |
| 832 | } |
| 833 | |
| 834 | #[test] |
| 835 | fn dismissing_a_secret_takes_admin_and_a_dependency_write() { |
| 836 | assert_eq!(Security::capability_for("sec_1"), Capability::SecurityAlerts); |
| 837 | assert_eq!(Security::capability_for("vul_1"), Capability::SecurityAlerts); |
| 838 | } |
| 839 | } |