g1t/services/repos/src/secret_scan.rs
Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API | 1 | //! Looking for secrets in git: in what a push adds, before it is stored |
| 2 | //! (push protection), and in a repository's history, a page at a time, for | |
| 3 | //! the security service. Also finds the lockfiles it reads dependencies | |
| 4 | //! from. What counts as a secret is `g1t_scan`'s business. | |
| 5 | ||
| 6 | use std::cell::Cell; | |
| 7 | use std::collections::{HashSet, VecDeque}; | |
| 8 | ||
| 9 | use futures_util::future::try_join_all; | |
| 10 | use g1t_contracts::User; | |
| 11 | use g1t_contracts::repos::{EntryKind, RepoPath}; | |
| 12 | use g1t_contracts::security::{ | |
| 13 | FindLockfilesArgs, HistoryPage, LockfileText, Lockfiles, NewSecret, PushBlockedArgs, PushVerdict, | |
| 14 | ScanHistoryArgs, | |
| 15 | }; | |
| 16 | use g1t_scan::lockfiles::Lockfile; | |
| 17 | use g1t_scan::pack::{ObjectKind, Pack, TreeItem, encode_tree, pack_start}; | |
| 18 | use g1t_scan::protection::{self, Blocked}; | |
| 19 | use worker::{Response, Result}; | |
| 20 | ||
| 21 | use crate::registry::store_key; | |
| 22 | use crate::store::{GitRepo, GitStore}; | |
| 23 | ||
| 24 | /// Where people allow a secret: the project's Security page. | |
| 25 | const SITE: &str = "https://g1t.sh"; | |
| 26 | /// A push adding more commits than this is scanned for this many of them. | |
| 27 | const MAX_PUSH_COMMITS: usize = 300; | |
| 28 | /// Files compared per commit, at most. | |
| 29 | const MAX_FILES_PER_COMMIT: usize = 300; | |
| 30 | /// Bases fetched from the store for a thin pack, at most. | |
| 31 | const MAX_BASES: usize = 500; | |
| 32 | /// Pushes larger than this are let through unscanned. | |
| 33 | const MAX_SCANNED_PUSH: usize = 24 * 1024 * 1024; | |
| 34 | const READS_AT_ONCE: usize = 16; | |
| 35 | /// Directories never searched for lockfiles. | |
| 36 | const SKIPPED_DIRECTORIES: [&str; 8] = ["node_modules", "vendor", "target", ".git", "dist", "build", "third_party", ".venv"]; | |
| 37 | const MAX_LOCKFILES: usize = 40; | |
| 38 | const MAX_LOCKFILE_DEPTH: usize = 4; | |
| 39 | const MAX_LOCKFILE_BYTES: usize = 16 * 1024 * 1024; | |
| 40 | ||
| 41 | fn mode(kind: EntryKind) -> &'static str { | |
| 42 | match kind { | |
| 43 | EntryKind::Tree => "40000", | |
| 44 | EntryKind::Blob => "100644", | |
| 45 | EntryKind::Exec => "100755", | |
| 46 | EntryKind::Symlink => "120000", | |
| 47 | EntryKind::Gitlink => "160000", | |
| 48 | } | |
| 49 | } | |
| 50 | ||
| 51 | /// Objects for a walk: the pushed pack's first, then the repository's. | |
| 52 | struct Objects<'a, R: GitRepo> { | |
| 53 | pack: &'a Pack, | |
| 54 | repo: &'a R, | |
| 55 | reads: Cell<u32>, | |
| 56 | } | |
| 57 | ||
| 58 | impl<R: GitRepo> Objects<'_, R> { | |
| 59 | async fn tree(&self, id: &str) -> Result<Vec<TreeItem>> { | |
| 60 | if let Some(items) = self.pack.tree(id) { | |
| 61 | return Ok(items); | |
| 62 | } | |
| 63 | self.reads.set(self.reads.get() + 1); | |
| 64 | Ok(self | |
| 65 | .repo | |
| 66 | .read_tree(id) | |
| 67 | .await? | |
| 68 | .unwrap_or_default() | |
| 69 | .into_iter() | |
| 70 | .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash }) | |
| 71 | .collect()) | |
| 72 | } | |
| 73 | ||
| 74 | async fn blob(&self, id: &str) -> Result<Option<Vec<u8>>> { | |
| 75 | if let Some(bytes) = self.pack.blob(id) { | |
| 76 | return Ok(Some(bytes.to_vec())); | |
| 77 | } | |
| 78 | self.reads.set(self.reads.get() + 1); | |
| 79 | self.repo.read_blob(id).await | |
| 80 | } | |
| 81 | ||
| 82 | async fn commit_tree(&self, id: &str) -> Result<Option<String>> { | |
| 83 | if let Some(commit) = self.pack.commit(id) { | |
| 84 | return Ok(Some(commit.tree)); | |
| 85 | } | |
| 86 | self.reads.set(self.reads.get() + 1); | |
| 87 | Ok(self.repo.log(id, 1).await?.into_iter().next().map(|commit| commit.tree_hash)) | |
| 88 | } | |
| 89 | } | |
| 90 | ||
| 91 | /// A file that differs between two trees: its path, the blob it was and | |
| 92 | /// the blob it is. | |
| 93 | struct Change { | |
| 94 | path: String, | |
| 95 | old: Option<String>, | |
| 96 | new: String, | |
| 97 | } | |
| 98 | ||
| 99 | /// The regular files whose content differs between two trees. Each level | |
| 100 | /// is read at once; identical subtrees are skipped by id. | |
| 101 | async fn changed_files<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: String) -> Result<Vec<Change>> { | |
| 102 | let mut changes = Vec::new(); | |
| 103 | let mut level = vec![(String::new(), old_root, new_root)]; | |
| 104 | while !level.is_empty() && changes.len() < MAX_FILES_PER_COMMIT { | |
| 105 | let read = try_join_all(level.iter().map(|(_, old, new)| async move { | |
| 106 | let old = match old { | |
| 107 | Some(old) => objects.tree(old).await?, | |
| 108 | None => Vec::new(), | |
| 109 | }; | |
| 110 | Ok::<_, worker::Error>((old, objects.tree(new).await?)) | |
| 111 | })) | |
| 112 | .await?; | |
| 113 | let mut next = Vec::new(); | |
| 114 | for ((prefix, _, _), (old, new)) in level.iter().zip(read) { | |
| 115 | for item in &new { | |
| 116 | let before = old.iter().find(|entry| entry.name == item.name); | |
| 117 | if before.is_some_and(|before| before.id == item.id) { | |
| 118 | continue; | |
| 119 | } | |
| 120 | let path = format!("{prefix}{}", item.name); | |
| 121 | if item.is_tree() { | |
| 122 | next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), item.id.clone())); | |
| 123 | } else if item.is_file() && changes.len() < MAX_FILES_PER_COMMIT { | |
| 124 | changes.push(Change { | |
| 125 | path, | |
| 126 | old: before.filter(|b| b.is_file()).map(|b| b.id.clone()), | |
| 127 | new: item.id.clone(), | |
| 128 | }); | |
| 129 | } | |
| 130 | } | |
| 131 | } | |
| 132 | level = next; | |
| 133 | } | |
| 134 | Ok(changes) | |
| 135 | } | |
| 136 | ||
| 137 | /// The secrets each change adds, found `READS_AT_ONCE` files at a time. | |
| 138 | async fn scan_changes<R: GitRepo>(objects: &Objects<'_, R>, commit: &str, changes: Vec<Change>) -> Result<Vec<NewSecret>> { | |
| 139 | let mut found = Vec::new(); | |
| 140 | let changes: Vec<Change> = changes | |
| 141 | .into_iter() | |
| 142 | .filter(|change| !g1t_scan::secrets::skipped_path(&change.path)) | |
| 143 | .collect(); | |
| 144 | for batch in changes.chunks(READS_AT_ONCE) { | |
| 145 | let read = try_join_all(batch.iter().map(|change| async move { | |
| 146 | let new = objects.blob(&change.new).await?; | |
| 147 | let old = match (&change.old, &new) { | |
| 148 | (Some(old), Some(_)) => objects.blob(old).await?, | |
| 149 | _ => None, | |
| 150 | }; | |
| 151 | Ok::<_, worker::Error>((new, old)) | |
| 152 | })) | |
| 153 | .await?; | |
| 154 | for (change, (new, old)) in batch.iter().zip(read) { | |
| 155 | let Some(new) = new else { continue }; | |
| 156 | for hit in protection::scan_change(&change.path, old.as_deref(), &new) { | |
| 157 | found.push(NewSecret { | |
| 158 | fingerprint: hit.fingerprint(), | |
| 159 | kind: hit.kind.id().to_owned(), | |
| 160 | path: change.path.clone(), | |
| 161 | line: hit.line, | |
| 162 | commit: commit.to_owned(), | |
| 163 | preview: hit.preview(), | |
| 164 | }); | |
| 165 | } | |
| 166 | } | |
| 167 | } | |
| 168 | Ok(found) | |
| 169 | } | |
| 170 | ||
| 171 | /// Fetches what a thin pack's deltas are based on from the repository. | |
| 172 | async fn supply_bases<R: GitRepo>(pack: &mut Pack, repo: &R) -> Result<()> { | |
| 173 | for _ in 0..3 { | |
| 174 | let missing = pack.missing_bases(); | |
| 175 | if missing.is_empty() { | |
| 176 | return Ok(()); | |
| 177 | } | |
| 178 | let found = try_join_all(missing.iter().take(MAX_BASES).map(|id| async move { | |
| 179 | // A base is nearly always a blob; failing that, a tree. | |
| 180 | if let Ok(Some(bytes)) = repo.read_blob(id).await { | |
| 181 | return Ok::<_, worker::Error>(Some((ObjectKind::Blob, bytes))); | |
| 182 | } | |
| 183 | Ok(repo.read_tree(id).await.ok().flatten().map(|entries| { | |
| 184 | let items: Vec<TreeItem> = entries | |
| 185 | .into_iter() | |
| 186 | .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash }) | |
| 187 | .collect(); | |
| 188 | (ObjectKind::Tree, encode_tree(&items)) | |
| 189 | })) | |
| 190 | })) | |
| 191 | .await?; | |
| 192 | let mut progress = false; | |
| 193 | for (id, object) in missing.iter().zip(found) { | |
| 194 | if let Some((kind, data)) = object { | |
| 195 | pack.supply(id, kind, data); | |
| 196 | progress = true; | |
| 197 | } | |
| 198 | } | |
| 199 | if !progress { | |
| 200 | return Ok(()); | |
| 201 | } | |
| 202 | } | |
| 203 | Ok(()) | |
| 204 | } | |
| 205 | ||
| 206 | /// The secrets the commits in a push add, each secret once. Fails open: a | |
| 207 | /// pack that cannot be read is let through, and said so in the logs. | |
| 208 | pub async fn scan_push<R: GitRepo>(repo: &R, body: &[u8]) -> Result<Vec<NewSecret>> { | |
| 209 | // The request is already in memory; reading a pack this large as well | |
| 210 | // could run the worker out of it, which would fail the push outright. | |
| 211 | if body.len() > MAX_SCANNED_PUSH { | |
| 212 | worker::console_error!("a push of {} bytes was not scanned for secrets", body.len()); | |
| 213 | return Ok(Vec::new()); | |
| 214 | } | |
| 215 | let Some(start) = pack_start(body) else { | |
| 216 | return Ok(Vec::new()); | |
| 217 | }; | |
| 218 | let mut pack = match Pack::parse(&body[start..]) { | |
| 219 | Ok(pack) => pack, | |
| 220 | Err(problem) => { | |
| 221 | worker::console_error!("push not scanned for secrets: {problem}"); | |
| 222 | return Ok(Vec::new()); | |
| 223 | } | |
| 224 | }; | |
| 225 | supply_bases(&mut pack, repo).await?; | |
| 226 | if pack.unresolved() > 0 { | |
| 227 | worker::console_error!("{} objects of a push could not be resolved for scanning", pack.unresolved()); | |
| 228 | } | |
| 229 | let objects = Objects { pack: &pack, repo, reads: Cell::new(0) }; | |
| 230 | let commits: Vec<String> = pack.commits().iter().take(MAX_PUSH_COMMITS).cloned().collect(); | |
| 231 | let mut found = Vec::new(); | |
| 232 | let mut seen_blobs = HashSet::new(); | |
| 233 | let mut seen_secrets = HashSet::new(); | |
| 234 | for id in commits { | |
| 235 | let Some(commit) = pack.commit(&id) else { continue }; | |
| 236 | let old_tree = match commit.parents.first() { | |
| 237 | Some(parent) => objects.commit_tree(parent).await?, | |
| 238 | None => None, | |
| 239 | }; | |
| 240 | // Only content the push brings is new; a blob the repository has | |
| 241 | // was looked at when it arrived. | |
| 242 | let changes: Vec<Change> = changed_files(&objects, old_tree, commit.tree) | |
| 243 | .await? | |
| 244 | .into_iter() | |
| 245 | .filter(|change| pack.contains(&change.new) && seen_blobs.insert((change.path.clone(), change.new.clone()))) | |
| 246 | .collect(); | |
| 247 | for secret in scan_changes(&objects, &id, changes).await? { | |
| 248 | if seen_secrets.insert(secret.fingerprint.clone()) { | |
| 249 | found.push(secret); | |
| 250 | } | |
| 251 | } | |
| 252 | } | |
| 253 | Ok(found) | |
| 254 | } | |
| 255 | ||
| 256 | impl<S: GitStore> crate::Repos<S> { | |
| 257 | /// Push protection: the response refusing a push that adds secrets | |
| 258 | /// nobody has allowed, or `None` to let it through. | |
| 259 | pub(crate) async fn protect(&self, path: &RepoPath, pusher: Option<&User>, body: &[u8]) -> Result<Option<Response>> { | |
| 260 | let Some(repo) = self.registry.by_path(path).await? else { | |
| 261 | return Ok(None); | |
| 262 | }; | |
| 263 | let git = self.store.open(&store_key(&repo)).await?; | |
| 264 | let found = scan_push(&git, body).await?; | |
| 265 | if found.is_empty() { | |
| 266 | return Ok(None); | |
| 267 | } | |
| 268 | // A pull request's findings belong to the repository it was made from. | |
| 269 | let owner = match &repo.fork_of { | |
| 270 | Some(id) => self.registry.by_id(id).await?.unwrap_or(repo.clone()), | |
| 271 | None => repo.clone(), | |
| 272 | }; | |
| 273 | let owner_path = RepoPath { namespace: owner.namespace.clone(), name: owner.name.clone() }; | |
| 274 | let verdict = match &self.security { | |
| 275 | Some(security) => g1t_kit::call::<_, PushVerdict>( | |
| 276 | security, | |
| 277 | "push_blocked", | |
| 278 | &PushBlockedArgs { | |
| 279 | repo_id: owner.id.clone(), | |
| 280 | path: owner_path.clone(), | |
| 281 | pusher: pusher.map(|user| user.username.clone()), | |
| 282 | secrets: found.clone(), | |
| 283 | }, | |
| 284 | ) | |
| 285 | .await | |
| 286 | .unwrap_or_else(|error| { | |
| 287 | worker::console_error!("push_blocked failed: {error}"); | |
| 288 | PushVerdict::default() | |
| 289 | }), | |
| 290 | None => PushVerdict::default(), | |
| 291 | }; | |
| 292 | let blocked: Vec<Blocked> = found | |
| 293 | .iter() | |
| 294 | .filter(|secret| !verdict.allowed.contains(&secret.fingerprint)) | |
| 295 | .filter_map(|secret| { | |
| 296 | let kind = g1t_scan::secrets::SecretKind::parse(&secret.kind)?; | |
| 297 | let id = verdict.ids.iter().find(|(fingerprint, _)| *fingerprint == secret.fingerprint); | |
| 298 | Some(Blocked { | |
| 299 | kind, | |
| 300 | path: secret.path.clone(), | |
| 301 | line: secret.line, | |
| 302 | commit: secret.commit.clone(), | |
| 303 | allow_url: id.map(|(_, id)| { | |
| 304 | format!("{SITE}/{}/{}/security?tab=secrets&finding={id}", owner_path.namespace, owner_path.name) | |
| 305 | }), | |
| 306 | }) | |
| 307 | }) | |
| 308 | .collect(); | |
| 309 | if blocked.is_empty() { | |
| 310 | return Ok(None); | |
| 311 | } | |
| 312 | Ok(Some(crate::git_http::declined( | |
| 313 | body, | |
| 314 | &protection::reason(&blocked), | |
| 315 | &protection::explain(&blocked), | |
| 316 | )?)) | |
| 317 | } | |
| 318 | ||
| 319 | /// A page of the default branch's history, scanned for secrets. | |
| 320 | pub(crate) async fn scan_history(&self, a: ScanHistoryArgs) -> Result<HistoryPage> { | |
| 321 | let Some(repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 322 | return Ok(HistoryPage::default()); | |
| 323 | }; | |
| 324 | let git = self.store.open(&store_key(&repo)).await?; | |
| 325 | let limit = a.limit.clamp(1, 100); | |
| 326 | let start = a.after.unwrap_or_else(|| repo.default_branch.clone()); | |
| 327 | let mut commits = git.log(&start, limit + 1).await?; | |
| 328 | let next = (commits.len() > limit as usize).then(|| commits.pop().map(|commit| commit.hash)).flatten(); | |
| 329 | let empty = Pack::default(); | |
| 330 | let objects = Objects { pack: &empty, repo: &git, reads: Cell::new(1) }; | |
| 331 | let mut page = HistoryPage { next, ..HistoryPage::default() }; | |
| 332 | let mut seen = HashSet::new(); | |
| 333 | for (index, commit) in commits.iter().enumerate() { | |
| 334 | let old_tree = match commit.parents.first() { | |
| 335 | Some(parent) => match commits.get(index + 1).filter(|older| older.hash == *parent) { | |
| 336 | Some(older) => Some(older.tree_hash.clone()), | |
| 337 | None => objects.commit_tree(parent).await?, | |
| 338 | }, | |
| 339 | None => None, | |
| 340 | }; | |
| 341 | let changes = changed_files(&objects, old_tree, commit.tree_hash.clone()).await?; | |
| 342 | for secret in scan_changes(&objects, &commit.hash, changes).await? { | |
| 343 | if seen.insert(secret.fingerprint.clone()) { | |
| 344 | page.secrets.push(secret); | |
| 345 | } | |
| 346 | } | |
| 347 | page.commits += 1; | |
| 348 | } | |
| 349 | page.reads = objects.reads.get(); | |
| 350 | Ok(page) | |
| 351 | } | |
| 352 | ||
| 353 | /// The lockfiles on the default branch, outside vendored directories. | |
| 354 | pub(crate) async fn find_lockfiles(&self, a: FindLockfilesArgs) -> Result<Lockfiles> { | |
| 355 | let Some(repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 356 | return Ok(Lockfiles::default()); | |
| 357 | }; | |
| 358 | let git = self.store.open(&store_key(&repo)).await?; | |
| 359 | let Some(head) = git.log(&repo.default_branch, 1).await?.into_iter().next() else { | |
| 360 | return Ok(Lockfiles::default()); | |
| 361 | }; | |
| 362 | let mut found = Vec::new(); | |
| 363 | let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone(), 0usize)]); | |
| 364 | while let Some((prefix, tree, depth)) = queue.pop_front() { | |
| 365 | for entry in git.read_tree(&tree).await?.unwrap_or_default() { | |
| 366 | match entry.kind { | |
| 367 | EntryKind::Tree if depth < MAX_LOCKFILE_DEPTH && !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => { | |
| 368 | queue.push_back((format!("{prefix}{}/", entry.name), entry.hash, depth + 1)); | |
| 369 | } | |
| 370 | EntryKind::Blob if Lockfile::for_path(&entry.name).is_some() && found.len() < MAX_LOCKFILES => { | |
| 371 | found.push((format!("{prefix}{}", entry.name), entry.hash)); | |
| 372 | } | |
| 373 | _ => {} | |
| 374 | } | |
| 375 | } | |
| 376 | } | |
| 377 | let texts = try_join_all(found.iter().map(|(_, hash)| git.read_blob(hash))).await?; | |
| 378 | let files = found | |
| 379 | .into_iter() | |
| 380 | .zip(texts) | |
| 381 | .filter_map(|((path, _), bytes)| { | |
| 382 | let bytes = bytes.filter(|bytes| bytes.len() <= MAX_LOCKFILE_BYTES)?; | |
| 383 | Some(LockfileText { path, text: String::from_utf8(bytes).ok()? }) | |
| 384 | }) | |
| 385 | .collect(); | |
| 386 | Ok(Lockfiles { commit: Some(head.hash), files }) | |
| 387 | } | |
| 388 | } | |
| 389 | ||
| 390 | #[cfg(test)] | |
| 391 | mod tests { | |
| 392 | use std::collections::HashMap; | |
| 393 | use std::future::Future; | |
| 394 | use std::pin::pin; | |
| 395 | use std::task::{Context, Poll, Waker}; | |
| 396 | ||
| 397 | use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature, TreeEntry}; | |
| 398 | use g1t_scan::pack::{ObjectKind, TreeItem, encode_tree, object_id}; | |
| 399 | ||
| 400 | use super::*; | |
| 401 | use crate::store::Scope; | |
| 402 | ||
| 403 | /// Runs a future that never waits, as every call to the fake store is. | |
| 404 | fn run<F: Future>(future: F) -> F::Output { | |
| 405 | match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) { | |
| 406 | Poll::Ready(output) => output, | |
| 407 | Poll::Pending => panic!("the fake store never waits"), | |
| 408 | } | |
| 409 | } | |
| 410 | ||
| 411 | /// A repository held in memory. | |
| 412 | #[derive(Default)] | |
| 413 | struct FakeRepo { | |
| 414 | blobs: HashMap<String, Vec<u8>>, | |
| 415 | trees: HashMap<String, Vec<TreeEntry>>, | |
| 416 | commits: HashMap<String, Commit>, | |
| 417 | } | |
| 418 | ||
| 419 | impl GitRepo for FakeRepo { | |
| 420 | async fn access(&self, _scope: Scope) -> Result<GitAccess> { | |
| 421 | unimplemented!() | |
| 422 | } | |
| 423 | async fn branches(&self) -> Result<Vec<Branch>> { | |
| 424 | Ok(Vec::new()) | |
| 425 | } | |
| 426 | async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> { | |
| 427 | Ok(self.commits.get(git_ref).cloned().into_iter().collect()) | |
| 428 | } | |
| 429 | async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> { | |
| 430 | Ok(self.commits.get(commit_hash).map(|commit| commit.parents.clone())) | |
| 431 | } | |
| 432 | async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> { | |
| 433 | Ok(self.trees.get(tree_hash).cloned()) | |
| 434 | } | |
| 435 | async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> { | |
| 436 | Ok(self.blobs.get(blob_hash).cloned()) | |
| 437 | } | |
| 438 | async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> { | |
| 439 | Ok(None) | |
| 440 | } | |
| 441 | async fn fork(&self, _target_key: &str) -> Result<()> { | |
| 442 | Ok(()) | |
| 443 | } | |
| 444 | } | |
| 445 | ||
| 446 | /// Zlib with one stored (uncompressed) block, which is all a pack needs. | |
| 447 | fn zlib(data: &[u8]) -> Vec<u8> { | |
| 448 | let mut out = vec![0x78, 0x01, 0x01]; | |
| 449 | let length = data.len() as u16; | |
| 450 | out.extend_from_slice(&length.to_le_bytes()); | |
| 451 | out.extend_from_slice(&(!length).to_le_bytes()); | |
| 452 | out.extend_from_slice(data); | |
| 453 | let (mut a, mut b) = (1u32, 0u32); | |
| 454 | for byte in data { | |
| 455 | a = (a + u32::from(*byte)) % 65521; | |
| 456 | b = (b + a) % 65521; | |
| 457 | } | |
| 458 | out.extend_from_slice(&((b << 16) | a).to_be_bytes()); | |
| 459 | out | |
| 460 | } | |
| 461 | ||
| 462 | fn header(code: u8, size: usize) -> Vec<u8> { | |
| 463 | let mut out = Vec::new(); | |
| 464 | let mut byte = (code << 4) | (size & 15) as u8; | |
| 465 | let mut rest = size >> 4; | |
| 466 | while rest > 0 { | |
| 467 | out.push(byte | 0x80); | |
| 468 | byte = (rest & 0x7f) as u8; | |
| 469 | rest >>= 7; | |
| 470 | } | |
| 471 | out.push(byte); | |
| 472 | out | |
| 473 | } | |
| 474 | ||
| 475 | fn raw_id(id: &str) -> Vec<u8> { | |
| 476 | id.as_bytes() | |
| 477 | .chunks(2) | |
| 478 | .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) | |
| 479 | .collect() | |
| 480 | } | |
| 481 | ||
| 482 | enum Entry { | |
| 483 | Whole(ObjectKind, Vec<u8>), | |
| 484 | /// A ref-delta: base id and delta. | |
| 485 | Delta(String, Vec<u8>), | |
| 486 | } | |
| 487 | ||
| 488 | /// A receive-pack request: one command, then the pack. | |
| 489 | fn push(entries: &[Entry]) -> Vec<u8> { | |
| 490 | let command = b"0000000000000000000000000000000000000000 4807077b296e6edbf410d55e72749d3e1170c291 refs/heads/main\0report-status side-band-64k\n"; | |
| 491 | let mut body = format!("{:04x}", command.len() + 4).into_bytes(); | |
| 492 | body.extend_from_slice(command); | |
| 493 | body.extend_from_slice(b"0000PACK"); | |
| 494 | body.extend_from_slice(&2u32.to_be_bytes()); | |
| 495 | body.extend_from_slice(&(entries.len() as u32).to_be_bytes()); | |
| 496 | for entry in entries { | |
| 497 | match entry { | |
| 498 | Entry::Whole(kind, data) => { | |
| 499 | let code = match kind { | |
| 500 | ObjectKind::Commit => 1, | |
| 501 | ObjectKind::Tree => 2, | |
| 502 | ObjectKind::Blob => 3, | |
| 503 | ObjectKind::Tag => 4, | |
| 504 | }; | |
| 505 | body.extend(header(code, data.len())); | |
| 506 | body.extend(zlib(data)); | |
| 507 | } | |
| 508 | Entry::Delta(base, delta) => { | |
| 509 | body.extend(header(7, delta.len())); | |
| 510 | body.extend(raw_id(base)); | |
| 511 | body.extend(zlib(delta)); | |
| 512 | } | |
| 513 | } | |
| 514 | } | |
| 515 | body.extend_from_slice(&[0u8; 20]); | |
| 516 | body | |
| 517 | } | |
| 518 | ||
| 519 | fn key() -> String { | |
| 520 | format!("AK{}", "IAZ7Q4N2XWLM3KDTRV") | |
| 521 | } | |
| 522 | ||
| 523 | fn commit(tree: &str, parent: Option<&str>) -> Vec<u8> { | |
| 524 | let parent = parent.map(|parent| format!("parent {parent}\n")).unwrap_or_default(); | |
| 525 | format!("tree {tree}\n{parent}author A <a@example.com> 0 +0000\ncommitter A <a@example.com> 0 +0000\n\nchange\n").into_bytes() | |
| 526 | } | |
| 527 | ||
| 528 | #[test] | |
| 529 | fn a_first_push_with_a_secret_is_found_by_file_and_line() { | |
| 530 | let blob = format!("REGION=eu\nAWS_KEY={}\n", key()).into_bytes(); | |
| 531 | let blob_id = object_id(ObjectKind::Blob, &blob); | |
| 532 | let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "config.env".into(), id: blob_id }]); | |
| 533 | let tree_id = object_id(ObjectKind::Tree, &tree); | |
| 534 | let body = push(&[ | |
| 535 | Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), | |
| 536 | Entry::Whole(ObjectKind::Tree, tree), | |
| 537 | Entry::Whole(ObjectKind::Blob, blob), | |
| 538 | ]); | |
| 539 | let found = run(scan_push(&FakeRepo::default(), &body)).unwrap(); | |
| 540 | assert_eq!(found.len(), 1); | |
| 541 | assert_eq!((found[0].path.as_str(), found[0].line, found[0].kind.as_str()), ("config.env", 2, "aws_access_key")); | |
| 542 | assert!(found[0].preview.starts_with("AKIA") && !found[0].preview.contains(&key())); | |
| 543 | } | |
| 544 | ||
| 545 | #[test] | |
| 546 | fn a_thin_push_reports_only_the_lines_it_adds() { | |
| 547 | // The repository already has a file with a key in it (decided on | |
| 548 | // before); the push appends a line holding a second key. | |
| 549 | let old = format!("first={}\n", key()).into_bytes(); | |
| 550 | let old_id = object_id(ObjectKind::Blob, &old); | |
| 551 | let second = format!("AK{}", "IAQ9W8E7R6T5Y4U3I2"); | |
| 552 | let new = [old.clone(), format!("second={second}\n").into_bytes()].concat(); | |
| 553 | let base_tree = vec![TreeEntry { name: "app.env".into(), hash: old_id.clone(), kind: EntryKind::Blob }]; | |
| 554 | let base_tree_id = object_id(ObjectKind::Tree, &encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: old_id.clone() }])); | |
| 555 | let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned(); | |
| 556 | let mut repo = FakeRepo::default(); | |
| 557 | repo.blobs.insert(old_id.clone(), old.clone()); | |
| 558 | repo.trees.insert(base_tree_id.clone(), base_tree); | |
| 559 | repo.commits.insert( | |
| 560 | parent_id.clone(), | |
| 561 | Commit { | |
| 562 | hash: parent_id.clone(), | |
| 563 | tree_hash: base_tree_id, | |
| 564 | message: String::new(), | |
| 565 | author: Signature { name: "A".into(), email: "a@example.com".into() }, | |
| 566 | parents: Vec::new(), | |
| 567 | authored_at: String::new(), | |
| 568 | }, | |
| 569 | ); | |
| 570 | // A delta: copy the old file whole, then insert the new line. | |
| 571 | let added = format!("second={second}\n").into_bytes(); | |
| 572 | let mut delta = vec![old.len() as u8, new.len() as u8, 0x80 | 0x10, old.len() as u8, added.len() as u8]; | |
| 573 | delta.extend_from_slice(&added); | |
| 574 | let new_id = object_id(ObjectKind::Blob, &new); | |
| 575 | let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: new_id }]); | |
| 576 | let tree_id = object_id(ObjectKind::Tree, &tree); | |
| 577 | let body = push(&[ | |
| 578 | Entry::Whole(ObjectKind::Commit, commit(&tree_id, Some(&parent_id))), | |
| 579 | Entry::Whole(ObjectKind::Tree, tree), | |
| 580 | Entry::Delta(old_id, delta), | |
| 581 | ]); | |
| 582 | let found = run(scan_push(&repo, &body)).unwrap(); | |
| 583 | assert_eq!(found.len(), 1, "{found:?}"); | |
| 584 | assert_eq!((found[0].path.as_str(), found[0].line), ("app.env", 2)); | |
| 585 | } | |
| 586 | ||
| 587 | #[test] | |
| 588 | fn a_push_without_secrets_or_a_pack_finds_nothing() { | |
| 589 | let blob = b"fn main() {}\n".to_vec(); | |
| 590 | let blob_id = object_id(ObjectKind::Blob, &blob); | |
| 591 | let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "main.rs".into(), id: blob_id }]); | |
| 592 | let tree_id = object_id(ObjectKind::Tree, &tree); | |
| 593 | let body = push(&[ | |
| 594 | Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), | |
| 595 | Entry::Whole(ObjectKind::Tree, tree), | |
| 596 | Entry::Whole(ObjectKind::Blob, blob), | |
| 597 | ]); | |
| 598 | assert!(run(scan_push(&FakeRepo::default(), &body)).unwrap().is_empty()); | |
| 599 | // A deletion sends commands and no pack. | |
| 600 | assert!(run(scan_push(&FakeRepo::default(), b"0000")).unwrap().is_empty()); | |
| 601 | } | |
| 602 | } |