g1t/services/security/src/history.rs
| 1 | //! Scanning a repository's history for secrets once, in the background, a |
| 2 | //! page of commits at a time, metered to its workspace; and the new commits |
| 3 | //! of a push too large to scan before it was stored, after it landed. |
| 4 | |
| 5 | use g1t_contracts::billing::{CheckLimitArgs, Limit, LimitState, NotePendingArgs}; |
| 6 | use g1t_contracts::identity::NotifyOwnersArgs; |
| 7 | use g1t_contracts::security::{HistoryPage, ScanHistoryArgs, SecretStatus}; |
| 8 | use g1t_contracts::Outcome; |
| 9 | use worker::Result; |
| 10 | |
| 11 | use crate::Security; |
| 12 | use crate::store::RepoRow; |
| 13 | |
| 14 | /// Commits read per call to the repos service. |
| 15 | const PAGE: u32 = 25; |
| 16 | /// Pages of a large push scanned as soon as it is heard of; the sweep |
| 17 | /// continues the rest. |
| 18 | pub const PUSH_PAGES_AT_ONCE: u32 = 4; |
| 19 | /// A push's scan stops after this many pages (25 000 commits): beyond it, |
| 20 | /// a rescan of the whole history is the way to look. |
| 21 | const MAX_PUSH_PAGES: i64 = 1_000; |
| 22 | // What scanning costs g1t, from Cloudflare's published prices on the |
| 23 | // Workers Paid plan (October 2026), the same as billing's `scan_cpu` and |
| 24 | // `scan_rows` meters: |
| 25 | // |
| 26 | // - Worker CPU time: $0.02 per million CPU milliseconds, so 0.02 millionths |
| 27 | // of a dollar per millisecond. |
| 28 | // - D1 rows written: $1.00 per million, so 1 millionth of a dollar a row. |
| 29 | // Rows read ($0.001 per million) come to nothing measurable. |
| 30 | // - Requests: the scan's calls between g1t's services go over service |
| 31 | // bindings, which Cloudflare does not charge as requests. |
| 32 | // - Artifacts: its operations are priced for create, push, pull and clone; |
| 33 | // reading a git object through the binding is none of those. |
| 34 | // - OSV, which dependency checks query, is free. |
| 35 | // |
| 36 | // So a scan costs the CPU it takes and the rows it writes. The CPU per |
| 37 | // object read is an estimate: decoding the object and running every |
| 38 | // secret pattern over it, generously rounded up. Billing charges the |
| 39 | // total at cost plus its margin once the month is over. |
| 40 | |
| 41 | /// Worker CPU, in millionths of a dollar per millisecond. |
| 42 | pub const MICROS_PER_CPU_MS: f64 = 0.02; |
| 43 | /// One D1 row written, in millionths of a dollar. |
| 44 | pub const MICROS_PER_ROW_WRITTEN: f64 = 1.0; |
| 45 | /// CPU one git object read takes in a history scan, in milliseconds. |
| 46 | pub const CPU_MS_PER_READ: f64 = 5.0; |
| 47 | |
| 48 | /// What a page of history scanning cost g1t, in millionths of a dollar, |
| 49 | /// rounded up: the CPU of its reads, and the rows it writes (where the |
| 50 | /// scan stands, the month's usage, and each secret found). |
| 51 | pub fn history_page_cost(reads: u32, secrets: usize) -> i64 { |
| 52 | let cpu = f64::from(reads) * CPU_MS_PER_READ * MICROS_PER_CPU_MS; |
| 53 | let rows = (2 + secrets) as f64 * MICROS_PER_ROW_WRITTEN; |
| 54 | (cpu + rows).ceil() as i64 |
| 55 | } |
| 56 | |
| 57 | /// What the email about secrets in a push that landed unscanned says. |
| 58 | pub fn landed_secrets_intro(namespace: &str, name: &str, branch: &str, count: usize) -> String { |
| 59 | let what = if count == 1 { "a secret that looks real".to_owned() } else { format!("{count} secrets that look real") }; |
| 60 | format!( |
| 61 | "A push to {branch} in {namespace}/{name} was too large to check before it was stored, so g1t scanned it after it landed and found {what}. \ |
| 62 | Rotate each one with whoever issued it, then mark the alert revoked, or dismiss it if it is not a real secret." |
| 63 | ) |
| 64 | } |
| 65 | |
| 66 | impl Security { |
| 67 | /// Whether the workspace's usage has reached its limit, which stops |
| 68 | /// background work. Unknown counts as not. |
| 69 | async fn over_limit(&self, workspace: &str) -> bool { |
| 70 | let limit: Result<Outcome<Limit>> = |
| 71 | g1t_kit::call(&self.billing, "check_limit", &CheckLimitArgs { workspace: workspace.to_owned() }).await; |
| 72 | matches!(limit, Ok(Outcome::Ok(limit)) if limit.state == LimitState::Stopped) |
| 73 | } |
| 74 | |
| 75 | /// Records what scanning cost, and tells billing the month's total so |
| 76 | /// the workspace's limit counts it. |
| 77 | pub async fn meter(&self, workspace: &str, reads: u32, commits: u32, osv_calls: u32, cost_micros: i64) -> Result<()> { |
| 78 | let total = self.store.meter(workspace, reads, commits, osv_calls, cost_micros).await?; |
| 79 | let noted: Result<bool> = g1t_kit::call( |
| 80 | &self.billing, |
| 81 | "note_pending", |
| 82 | &NotePendingArgs { workspace: workspace.to_owned(), source: "security".to_owned(), cost_micros: total, detail: None }, |
| 83 | ) |
| 84 | .await; |
| 85 | if let Err(error) = noted { |
| 86 | worker::console_error!("security: usage for {workspace} not noted: {error}"); |
| 87 | } |
| 88 | Ok(()) |
| 89 | } |
| 90 | |
| 91 | /// Scans up to `pages` pages of the new commits of a push that reached |
| 92 | /// the store unscanned, from where its scan stopped. What it finds is |
| 93 | /// recorded open (it has landed), never blocking anything; a secret that |
| 94 | /// looks real is emailed to the workspace's owners once per push. |
| 95 | pub async fn advance_push_scan(&self, id: &str, pages: u32) -> Result<()> { |
| 96 | let Some(scan) = self.store.push_scan(id).await? else { |
| 97 | return Ok(()); |
| 98 | }; |
| 99 | let Some(repo) = self.store.repo(&scan.repo_id).await? else { |
| 100 | return self.store.advance_push_scan(id, None, 0, 0).await; |
| 101 | }; |
| 102 | if self.over_limit(&repo.namespace).await { |
| 103 | // Picked up again by the sweep once the workspace is under it. |
| 104 | return Ok(()); |
| 105 | } |
| 106 | let mut cursor = scan.cursor.clone(); |
| 107 | let mut real = 0usize; |
| 108 | for pages_done in (scan.pages + 1)..=(scan.pages + i64::from(pages)) { |
| 109 | let page: HistoryPage = g1t_kit::call( |
| 110 | &self.repos, |
| 111 | "scan_history", |
| 112 | &ScanHistoryArgs { |
| 113 | repo_id: repo.repo_id.clone(), |
| 114 | after: cursor.clone(), |
| 115 | limit: PAGE, |
| 116 | from: Some(scan.head.clone()), |
| 117 | until: scan.base.clone(), |
| 118 | }, |
| 119 | ) |
| 120 | .await?; |
| 121 | let fingerprints: Vec<String> = page.secrets.iter().map(|secret| secret.fingerprint.clone()).collect(); |
| 122 | let fresh: Vec<String> = { |
| 123 | let known = self.store.known(&repo.repo_id, &fingerprints).await?; |
| 124 | page.secrets |
| 125 | .iter() |
| 126 | .filter(|secret| secret.test_value.is_none()) |
| 127 | .filter(|secret| !known.iter().any(|(fingerprint, _, _)| *fingerprint == secret.fingerprint)) |
| 128 | .map(|secret| secret.fingerprint.clone()) |
| 129 | .collect() |
| 130 | }; |
| 131 | real += fresh.len(); |
| 132 | self.store.landed(&repo.repo_id, &fingerprints).await?; |
| 133 | self.store |
| 134 | .add_secrets(&repo.repo_id, &page.secrets, SecretStatus::Open, "history", scan.pusher.as_deref()) |
| 135 | .await?; |
| 136 | let cost = history_page_cost(page.reads, page.secrets.len()); |
| 137 | self.meter(&repo.namespace, page.reads, page.commits, 0, cost).await?; |
| 138 | let next = page.next.filter(|_| pages_done < MAX_PUSH_PAGES); |
| 139 | self.store |
| 140 | .advance_push_scan(id, next.as_deref(), page.commits, page.secrets.len() as u32) |
| 141 | .await?; |
| 142 | match next { |
| 143 | Some(next) => cursor = Some(next), |
| 144 | None => break, |
| 145 | } |
| 146 | } |
| 147 | if real > 0 { |
| 148 | self.tell_owners_of_landed_secrets(&repo, &scan.git_ref, real).await; |
| 149 | } |
| 150 | Ok(()) |
| 151 | } |
| 152 | |
| 153 | /// Emails a workspace's owners, who are Admins of every repository in |
| 154 | /// it, that a push which landed unscanned holds secrets that look real. |
| 155 | async fn tell_owners_of_landed_secrets(&self, repo: &RepoRow, git_ref: &str, count: usize) { |
| 156 | let branch = git_ref.strip_prefix("refs/heads/").unwrap_or(git_ref); |
| 157 | let args = NotifyOwnersArgs { |
| 158 | workspace: repo.namespace.clone(), |
| 159 | subject: format!("Secrets found in a push to {}/{}", repo.namespace, repo.name), |
| 160 | intro: landed_secrets_intro(&repo.namespace, &repo.name, branch, count), |
| 161 | action: "Review the alerts".to_owned(), |
| 162 | link: format!("https://g1t.sh/{}/{}/security?tab=secrets", repo.namespace, repo.name), |
| 163 | footer: "You get this because you own this workspace on g1t. Very large pushes are scanned for secrets after they land: https://docs.g1t.sh/guides/security/".to_owned(), |
| 164 | }; |
| 165 | if let Err(error) = g1t_kit::call::<_, u32>(&self.identity, "notify_owners", &args).await { |
| 166 | worker::console_error!("security: owners of {} not told of landed secrets: {error}", repo.namespace); |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | /// Scans up to `pages` pages of a repository's history from where the |
| 171 | /// last scan stopped. |
| 172 | pub async fn advance_history(&self, repo: &RepoRow, pages: u32) -> Result<()> { |
| 173 | if repo.history == "done" { |
| 174 | return Ok(()); |
| 175 | } |
| 176 | if self.over_limit(&repo.namespace).await { |
| 177 | return self.store.set_history(&repo.repo_id, "stopped", repo.history_cursor.as_deref(), 0).await; |
| 178 | } |
| 179 | let mut cursor = repo.history_cursor.clone(); |
| 180 | for _ in 0..pages { |
| 181 | let page: HistoryPage = g1t_kit::call( |
| 182 | &self.repos, |
| 183 | "scan_history", |
| 184 | &ScanHistoryArgs { repo_id: repo.repo_id.clone(), after: cursor.clone(), limit: PAGE, from: None, until: None }, |
| 185 | ) |
| 186 | .await?; |
| 187 | let fingerprints: Vec<String> = page.secrets.iter().map(|secret| secret.fingerprint.clone()).collect(); |
| 188 | self.store.landed(&repo.repo_id, &fingerprints).await?; |
| 189 | self.store.add_secrets(&repo.repo_id, &page.secrets, SecretStatus::Open, "history", None).await?; |
| 190 | let cost = history_page_cost(page.reads, page.secrets.len()); |
| 191 | self.meter(&repo.namespace, page.reads, page.commits, 0, cost).await?; |
| 192 | match page.next { |
| 193 | Some(next) => { |
| 194 | self.store.set_history(&repo.repo_id, "running", Some(&next), page.commits).await?; |
| 195 | cursor = Some(next); |
| 196 | } |
| 197 | None => return self.store.set_history(&repo.repo_id, "done", None, page.commits).await, |
| 198 | } |
| 199 | } |
| 200 | Ok(()) |
| 201 | } |
| 202 | } |