g1t/services/security/src/history.rs
| 1 | //! Scanning a repository's history for secrets once, in the background, a |
| 2 | //! page of commits at a time, metered to its workspace. |
| 3 | |
| 4 | use g1t_contracts::billing::{CheckLimitArgs, Limit, LimitState, NotePendingArgs}; |
| 5 | use g1t_contracts::security::{HistoryPage, ScanHistoryArgs, SecretStatus}; |
| 6 | use g1t_contracts::Outcome; |
| 7 | use worker::Result; |
| 8 | |
| 9 | use crate::Security; |
| 10 | use crate::store::RepoRow; |
| 11 | |
| 12 | /// Commits read per call to the repos service. |
| 13 | const PAGE: u32 = 25; |
| 14 | // What scanning costs g1t, from Cloudflare's published prices on the |
| 15 | // Workers Paid plan (October 2026), the same as billing's `scan_cpu` and |
| 16 | // `scan_rows` meters: |
| 17 | // |
| 18 | // - Worker CPU time: $0.02 per million CPU milliseconds, so 0.02 millionths |
| 19 | // of a dollar per millisecond. |
| 20 | // - D1 rows written: $1.00 per million, so 1 millionth of a dollar a row. |
| 21 | // Rows read ($0.001 per million) come to nothing measurable. |
| 22 | // - Requests: the scan's calls between g1t's services go over service |
| 23 | // bindings, which Cloudflare does not charge as requests. |
| 24 | // - Artifacts: its operations are priced for create, push, pull and clone; |
| 25 | // reading a git object through the binding is none of those. |
| 26 | // - OSV, which dependency checks query, is free. |
| 27 | // |
| 28 | // So a scan costs the CPU it takes and the rows it writes. The CPU per |
| 29 | // object read is an estimate: decoding the object and running every |
| 30 | // secret pattern over it, generously rounded up. Billing charges the |
| 31 | // total at cost plus its margin once the month is over. |
| 32 | |
| 33 | /// Worker CPU, in millionths of a dollar per millisecond. |
| 34 | pub const MICROS_PER_CPU_MS: f64 = 0.02; |
| 35 | /// One D1 row written, in millionths of a dollar. |
| 36 | pub const MICROS_PER_ROW_WRITTEN: f64 = 1.0; |
| 37 | /// CPU one git object read takes in a history scan, in milliseconds. |
| 38 | pub const CPU_MS_PER_READ: f64 = 5.0; |
| 39 | |
| 40 | /// What a page of history scanning cost g1t, in millionths of a dollar, |
| 41 | /// rounded up: the CPU of its reads, and the rows it writes (where the |
| 42 | /// scan stands, the month's usage, and each secret found). |
| 43 | pub fn history_page_cost(reads: u32, secrets: usize) -> i64 { |
| 44 | let cpu = f64::from(reads) * CPU_MS_PER_READ * MICROS_PER_CPU_MS; |
| 45 | let rows = (2 + secrets) as f64 * MICROS_PER_ROW_WRITTEN; |
| 46 | (cpu + rows).ceil() as i64 |
| 47 | } |
| 48 | |
| 49 | impl Security { |
| 50 | /// Whether the workspace's usage has reached its limit, which stops |
| 51 | /// background work. Unknown counts as not. |
| 52 | async fn over_limit(&self, workspace: &str) -> bool { |
| 53 | let limit: Result<Outcome<Limit>> = |
| 54 | g1t_kit::call(&self.billing, "check_limit", &CheckLimitArgs { workspace: workspace.to_owned() }).await; |
| 55 | matches!(limit, Ok(Outcome::Ok(limit)) if limit.state == LimitState::Stopped) |
| 56 | } |
| 57 | |
| 58 | /// Records what scanning cost, and tells billing the month's total so |
| 59 | /// the workspace's limit counts it. |
| 60 | pub async fn meter(&self, workspace: &str, reads: u32, commits: u32, osv_calls: u32, cost_micros: i64) -> Result<()> { |
| 61 | let total = self.store.meter(workspace, reads, commits, osv_calls, cost_micros).await?; |
| 62 | let noted: Result<bool> = g1t_kit::call( |
| 63 | &self.billing, |
| 64 | "note_pending", |
| 65 | &NotePendingArgs { workspace: workspace.to_owned(), source: "security".to_owned(), cost_micros: total }, |
| 66 | ) |
| 67 | .await; |
| 68 | if let Err(error) = noted { |
| 69 | worker::console_error!("security: usage for {workspace} not noted: {error}"); |
| 70 | } |
| 71 | Ok(()) |
| 72 | } |
| 73 | |
| 74 | /// Scans up to `pages` pages of a repository's history from where the |
| 75 | /// last scan stopped. |
| 76 | pub async fn advance_history(&self, repo: &RepoRow, pages: u32) -> Result<()> { |
| 77 | if repo.history == "done" { |
| 78 | return Ok(()); |
| 79 | } |
| 80 | if self.over_limit(&repo.namespace).await { |
| 81 | return self.store.set_history(&repo.repo_id, "stopped", repo.history_cursor.as_deref(), 0).await; |
| 82 | } |
| 83 | let mut cursor = repo.history_cursor.clone(); |
| 84 | for _ in 0..pages { |
| 85 | let page: HistoryPage = g1t_kit::call( |
| 86 | &self.repos, |
| 87 | "scan_history", |
| 88 | &ScanHistoryArgs { repo_id: repo.repo_id.clone(), after: cursor.clone(), limit: PAGE }, |
| 89 | ) |
| 90 | .await?; |
| 91 | let fingerprints: Vec<String> = page.secrets.iter().map(|secret| secret.fingerprint.clone()).collect(); |
| 92 | self.store.landed(&repo.repo_id, &fingerprints).await?; |
| 93 | self.store.add_secrets(&repo.repo_id, &page.secrets, SecretStatus::Open, "history", None).await?; |
| 94 | let cost = history_page_cost(page.reads, page.secrets.len()); |
| 95 | self.meter(&repo.namespace, page.reads, page.commits, 0, cost).await?; |
| 96 | match page.next { |
| 97 | Some(next) => { |
| 98 | self.store.set_history(&repo.repo_id, "running", Some(&next), page.commits).await?; |
| 99 | cursor = Some(next); |
| 100 | } |
| 101 | None => return self.store.set_history(&repo.repo_id, "done", None, page.commits).await, |
| 102 | } |
| 103 | } |
| 104 | Ok(()) |
| 105 | } |
| 106 | } |