Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/repos/src/secret_scan.rs

740 lines31,828 bytesCodeBlame
1//! Looking for secrets in git: in what a push adds, before it is stored
2//! (push protection), and in a repository's history, a page at a time, for
3//! the security service. Also finds the lockfiles it reads dependencies
4//! from. What counts as a secret is `g1t_scan`'s business.
5//!
6//! Push protection also keeps a person's private address out of what they
7//! push, when they asked g1t to (see [`exposed_address`]).
8
9use std::cell::Cell;
10use std::collections::{HashSet, VecDeque};
11
12use futures_util::future::try_join_all;
13use g1t_contracts::User;
14use g1t_contracts::accounts::{CommitIdentityArgs, PushEmailGuard, mask_email};
15use g1t_contracts::repos::{EntryKind, Repo, RepoPath};
16use g1t_contracts::security::{
17 FindLockfilesArgs, HistoryPage, LockfileText, Lockfiles, NewSecret, PushBlockedArgs, PushVerdict,
18 ScanHistoryArgs,
19};
20use g1t_scan::lockfiles::Lockfile;
21use g1t_scan::pack::{ObjectKind, Pack, TreeItem, encode_tree, pack_start};
22use g1t_scan::protection::{self, Blocked};
23use worker::{Response, Result};
24
25use crate::registry::store_key;
26use crate::store::{GitRepo, GitStore};
27
28/// Where people allow a secret: the project's Security page.
29const SITE: &str = "https://g1t.sh";
30/// A push adding more commits than this is scanned for this many of them.
31const MAX_PUSH_COMMITS: usize = 300;
32/// Files compared per commit, at most.
33const MAX_FILES_PER_COMMIT: usize = 300;
34/// Bases fetched from the store for a thin pack, at most.
35const MAX_BASES: usize = 500;
36/// The largest push that is read whole and scanned. A larger one is
37/// declined, since it cannot be checked (git_http.rs `LargePushes`).
38pub const MAX_SCANNED_PUSH: usize = 24 * 1024 * 1024;
39/// What marks an error as a push too large to scan.
40const UNSCANNABLE: &str = "push-unscannable:";
41
42/// Whether an error says the push was too large to scan.
43pub fn unscannable(error: &worker::Error) -> bool {
44 error.to_string().contains(UNSCANNABLE)
45}
46const READS_AT_ONCE: usize = 16;
47/// Directories never searched for lockfiles.
48const SKIPPED_DIRECTORIES: [&str; 8] = ["node_modules", "vendor", "target", ".git", "dist", "build", "third_party", ".venv"];
49const MAX_LOCKFILES: usize = 40;
50const MAX_LOCKFILE_DEPTH: usize = 4;
51const MAX_LOCKFILE_BYTES: usize = 16 * 1024 * 1024;
52
53fn mode(kind: EntryKind) -> &'static str {
54 match kind {
55 EntryKind::Tree => "40000",
56 EntryKind::Blob => "100644",
57 EntryKind::Exec => "100755",
58 EntryKind::Symlink => "120000",
59 EntryKind::Gitlink => "160000",
60 }
61}
62
63/// Objects for a walk: the pushed pack's first, then the repository's.
64struct Objects<'a, R: GitRepo> {
65 pack: &'a Pack,
66 repo: &'a R,
67 reads: Cell<u32>,
68}
69
70impl<R: GitRepo> Objects<'_, R> {
71 async fn tree(&self, id: &str) -> Result<Vec<TreeItem>> {
72 if let Some(items) = self.pack.tree(id) {
73 return Ok(items);
74 }
75 self.reads.set(self.reads.get() + 1);
76 Ok(self
77 .repo
78 .read_tree(id)
79 .await?
80 .unwrap_or_default()
81 .into_iter()
82 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
83 .collect())
84 }
85
86 async fn blob(&self, id: &str) -> Result<Option<Vec<u8>>> {
87 if let Some(bytes) = self.pack.blob(id) {
88 return Ok(Some(bytes.to_vec()));
89 }
90 self.reads.set(self.reads.get() + 1);
91 self.repo.read_blob(id).await
92 }
93
94 async fn commit_tree(&self, id: &str) -> Result<Option<String>> {
95 if let Some(commit) = self.pack.commit(id) {
96 return Ok(Some(commit.tree));
97 }
98 self.reads.set(self.reads.get() + 1);
99 Ok(self.repo.log(id, 1).await?.into_iter().next().map(|commit| commit.tree_hash))
100 }
101}
102
103/// A file that differs between two trees: its path, the blob it was and
104/// the blob it is.
105struct Change {
106 path: String,
107 old: Option<String>,
108 new: String,
109}
110
111/// The regular files whose content differs between two trees. Each level
112/// is read at once; identical subtrees are skipped by id.
113async fn changed_files<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: String) -> Result<Vec<Change>> {
114 let mut changes = Vec::new();
115 let mut level = vec![(String::new(), old_root, new_root)];
116 while !level.is_empty() && changes.len() < MAX_FILES_PER_COMMIT {
117 let read = try_join_all(level.iter().map(|(_, old, new)| async move {
118 let old = match old {
119 Some(old) => objects.tree(old).await?,
120 None => Vec::new(),
121 };
122 Ok::<_, worker::Error>((old, objects.tree(new).await?))
123 }))
124 .await?;
125 let mut next = Vec::new();
126 for ((prefix, _, _), (old, new)) in level.iter().zip(read) {
127 for item in &new {
128 let before = old.iter().find(|entry| entry.name == item.name);
129 if before.is_some_and(|before| before.id == item.id) {
130 continue;
131 }
132 let path = format!("{prefix}{}", item.name);
133 if item.is_tree() {
134 next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), item.id.clone()));
135 } else if item.is_file() && changes.len() < MAX_FILES_PER_COMMIT {
136 changes.push(Change {
137 path,
138 old: before.filter(|b| b.is_file()).map(|b| b.id.clone()),
139 new: item.id.clone(),
140 });
141 }
142 }
143 }
144 level = next;
145 }
146 Ok(changes)
147}
148
149/// The secrets each change adds, found `READS_AT_ONCE` files at a time.
150async fn scan_changes<R: GitRepo>(objects: &Objects<'_, R>, commit: &str, changes: Vec<Change>) -> Result<Vec<NewSecret>> {
151 let mut found = Vec::new();
152 let changes: Vec<Change> = changes
153 .into_iter()
154 .filter(|change| !g1t_scan::secrets::skipped_path(&change.path))
155 .collect();
156 for batch in changes.chunks(READS_AT_ONCE) {
157 let read = try_join_all(batch.iter().map(|change| async move {
158 let new = objects.blob(&change.new).await?;
159 let old = match (&change.old, &new) {
160 (Some(old), Some(_)) => objects.blob(old).await?,
161 _ => None,
162 };
163 Ok::<_, worker::Error>((new, old))
164 }))
165 .await?;
166 for (change, (new, old)) in batch.iter().zip(read) {
167 let Some(new) = new else { continue };
168 for hit in protection::scan_change(&change.path, old.as_deref(), &new) {
169 found.push(NewSecret {
170 fingerprint: hit.fingerprint(),
171 kind: hit.kind.id().to_owned(),
172 path: change.path.clone(),
173 line: hit.line,
174 commit: commit.to_owned(),
175 preview: hit.preview(),
176 test_value: hit.test_value().map(str::to_owned),
177 });
178 }
179 }
180 }
181 Ok(found)
182}
183
184/// Fetches what a thin pack's deltas are based on from the repository.
185async fn supply_bases<R: GitRepo>(pack: &mut Pack, repo: &R) -> Result<()> {
186 for _ in 0..3 {
187 let missing = pack.missing_bases();
188 if missing.is_empty() {
189 return Ok(());
190 }
191 let found = try_join_all(missing.iter().take(MAX_BASES).map(|id| async move {
192 // A base is nearly always a blob; failing that, a tree.
193 if let Ok(Some(bytes)) = repo.read_blob(id).await {
194 return Ok::<_, worker::Error>(Some((ObjectKind::Blob, bytes)));
195 }
196 Ok(repo.read_tree(id).await.ok().flatten().map(|entries| {
197 let items: Vec<TreeItem> = entries
198 .into_iter()
199 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
200 .collect();
201 (ObjectKind::Tree, encode_tree(&items))
202 }))
203 }))
204 .await?;
205 let mut progress = false;
206 for (id, object) in missing.iter().zip(found) {
207 if let Some((kind, data)) = object {
208 pack.supply(id, kind, data);
209 progress = true;
210 }
211 }
212 if !progress {
213 return Ok(());
214 }
215 }
216 Ok(())
217}
218
219/// The secrets the commits in a push add, each secret once. A push too
220/// large to read is an error ([`unscannable`]): it is declined, never let
221/// through unread. A pack that cannot be read for another reason is let
222/// through, and said so in the logs; the store will judge it.
223pub async fn scan_push<R: GitRepo>(repo: &R, body: &[u8]) -> Result<Vec<NewSecret>> {
224 if body.len() > MAX_SCANNED_PUSH {
225 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {} bytes", body.len())));
226 }
227 let Some(start) = pack_start(body) else {
228 return Ok(Vec::new());
229 };
230 let mut pack = match Pack::parse(&body[start..]) {
231 Ok(pack) => pack,
232 Err(problem) if problem.contains("too large") => {
233 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {problem}")));
234 }
235 Err(problem) => {
236 worker::console_error!("push not scanned for secrets: {problem}");
237 return Ok(Vec::new());
238 }
239 };
240 supply_bases(&mut pack, repo).await?;
241 if pack.unresolved() > 0 {
242 worker::console_error!("{} objects of a push could not be resolved for scanning", pack.unresolved());
243 }
244 let objects = Objects { pack: &pack, repo, reads: Cell::new(0) };
245 let commits: Vec<String> = pack.commits().iter().take(MAX_PUSH_COMMITS).cloned().collect();
246 let mut found = Vec::new();
247 let mut seen_blobs = HashSet::new();
248 let mut seen_secrets = HashSet::new();
249 for id in commits {
250 let Some(commit) = pack.commit(&id) else { continue };
251 let old_tree = match commit.parents.first() {
252 Some(parent) => objects.commit_tree(parent).await?,
253 None => None,
254 };
255 // Only content the push brings is new; a blob the repository has
256 // was looked at when it arrived.
257 let changes: Vec<Change> = changed_files(&objects, old_tree, commit.tree)
258 .await?
259 .into_iter()
260 .filter(|change| pack.contains(&change.new) && seen_blobs.insert((change.path.clone(), change.new.clone())))
261 .collect();
262 for secret in scan_changes(&objects, &id, changes).await? {
263 if seen_secrets.insert(secret.fingerprint.clone()) {
264 found.push(secret);
265 }
266 }
267 }
268 Ok(found)
269}
270
271/// A commit in a push that would publish one of the pusher's own
272/// addresses while they keep it private: its id and the address. Only the
273/// commits the push adds are read; anyone else's address is no concern
274/// here. A pack that cannot be read is let through.
275pub fn exposed_address(body: &[u8], guard: &PushEmailGuard) -> Option<(String, String)> {
276 if body.len() > MAX_SCANNED_PUSH {
277 return None;
278 }
279 let pack = Pack::parse(&body[pack_start(body)?..]).ok()?;
280 pack.commits().iter().find_map(|id| {
281 let commit = pack.commit(id)?;
282 [commit.author_email, commit.committer_email]
283 .into_iter()
284 .flatten()
285 .find(|email| guard.exposes(email))
286 .map(|email| (id.clone(), email))
287 })
288}
289
290/// What git shows a person whose push would publish their private address.
291pub fn exposed_message(commit: &str, email: &str, noreply: &str) -> Vec<String> {
292 let short: String = commit.chars().take(7).collect();
293 vec![
294 format!(
295 "push declined: commit {short} would publish {} while your email is private.",
296 mask_email(&email.to_lowercase())
297 ),
298 format!("Commit with {noreply} (git config user.email {noreply}) and amend,"),
299 format!("or change this in {}/settings/emails.", SITE.trim_start_matches("https://")),
300 ]
301}
302
303impl<S: GitStore> crate::Repos<S> {
304 /// What a push by `pusher` must not publish: their own addresses, when
305 /// they keep them private and block such pushes. An agent's push is
306 /// its person's. `None` when nothing is guarded, or identity cannot say.
307 async fn push_email_guard(&self, pusher: Option<&User>) -> Option<PushEmailGuard> {
308 let pusher = pusher?;
309 let person = pusher.acting.as_ref().map_or(pusher.id.clone(), |acting| acting.on_behalf_of.id.clone());
310 let identity = self.identity.as_ref()?;
311 g1t_kit::call::<_, Option<PushEmailGuard>>(identity, "push_email_guard", &CommitIdentityArgs { user_id: person })
312 .await
313 .unwrap_or_else(|error| {
314 worker::console_error!("push_email_guard failed: {error}");
315 None
316 })
317 }
318
319 /// Push protection: the response refusing a push that adds secrets
320 /// nobody has allowed, or that would publish the pusher's private
321 /// address, or `None` to let it through.
322 /// `repo` is the repository pushed to, as the request read it.
323 pub(crate) async fn protect(&self, repo: &Repo, pusher: Option<&User>, body: &[u8]) -> Result<Option<Response>> {
324 // Asking identity about the pusher's address and scanning the push
325 // do not depend on each other, so they happen at once.
326 let scan = async {
327 let git = self.store.open(&store_key(repo)).await?;
328 scan_push(&git, body).await
329 };
330 let (guard, found) = futures_util::future::join(self.push_email_guard(pusher), scan).await;
331 if let Some(guard) = guard
332 && let Some((commit, email)) = exposed_address(body, &guard)
333 {
334 return Ok(Some(crate::git_http::declined(
335 body,
336 "push would publish a private email",
337 &exposed_message(&commit, &email, &guard.noreply),
338 )?));
339 }
340 let found = match found {
341 Err(error) if unscannable(&error) => {
342 let (reason, messages) = crate::git_http::size_refusal(&crate::git_http::SizeViolation::Unscannable {
343 size: body.len() as u64,
344 cap: MAX_SCANNED_PUSH,
345 });
346 return Ok(Some(crate::git_http::declined(body, &reason, &messages)?));
347 }
348 found => found?,
349 };
350 if found.is_empty() {
351 return Ok(None);
352 }
353 // A pull request's findings belong to the repository it was made from.
354 let owner = match &repo.fork_of {
355 Some(id) => self.registry.by_id(id).await?.unwrap_or(repo.clone()),
356 None => repo.clone(),
357 };
358 let owner_path = RepoPath { namespace: owner.namespace.clone(), name: owner.name.clone() };
359 let verdict = match &self.security {
360 Some(security) => g1t_kit::call::<_, PushVerdict>(
361 security,
362 "push_blocked",
363 &PushBlockedArgs {
364 repo_id: owner.id.clone(),
365 path: owner_path.clone(),
366 pusher: pusher.map(|user| user.username.clone()),
367 secrets: found.clone(),
368 },
369 )
370 .await
371 .unwrap_or_else(|error| {
372 worker::console_error!("push_blocked failed: {error}");
373 PushVerdict::default()
374 }),
375 None => PushVerdict::default(),
376 };
377 let blocked: Vec<Blocked> = found
378 .iter()
379 .filter(|secret| !verdict.allowed.contains(&secret.fingerprint))
380 // A likely test value is recorded, never a reason to refuse.
381 .filter(|secret| secret.test_value.is_none())
382 .filter_map(|secret| {
383 let kind = g1t_scan::secrets::SecretKind::parse(&secret.kind)?;
384 let id = verdict.ids.iter().find(|(fingerprint, _)| *fingerprint == secret.fingerprint);
385 Some(Blocked {
386 kind,
387 path: secret.path.clone(),
388 line: secret.line,
389 commit: secret.commit.clone(),
390 allow_url: id.map(|(_, id)| {
391 format!("{SITE}/{}/{}/security?tab=secrets&finding={id}", owner_path.namespace, owner_path.name)
392 }),
393 })
394 })
395 .collect();
396 if blocked.is_empty() {
397 return Ok(None);
398 }
399 Ok(Some(crate::git_http::declined(
400 body,
401 &protection::reason(&blocked),
402 &protection::explain(&blocked),
403 )?))
404 }
405
406 /// A page of the default branch's history, scanned for secrets.
407 pub(crate) async fn scan_history(&self, a: ScanHistoryArgs) -> Result<HistoryPage> {
408 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
409 return Ok(HistoryPage::default());
410 };
411 let git = self.store.open(&store_key(&repo)).await?;
412 let limit = a.limit.clamp(1, 100);
413 // A page of a pushed range starts at its newest commit, and the
414 // history of the default branch at its head.
415 let start = a.after.or(a.from).unwrap_or_else(|| repo.default_branch.clone());
416 let mut commits = git.log(&start, limit + 1).await?;
417 let mut next = (commits.len() > limit as usize).then(|| commits.pop().map(|commit| commit.hash)).flatten();
418 // A range ends where the branch was before the push.
419 if let Some(until) = a.until.as_deref()
420 && let Some(at) = commits.iter().position(|commit| commit.hash == until)
421 {
422 commits.truncate(at);
423 next = None;
424 }
425 if a.until.is_some() && next.as_deref() == a.until.as_deref() {
426 next = None;
427 }
428 let empty = Pack::default();
429 let objects = Objects { pack: &empty, repo: &git, reads: Cell::new(1) };
430 let mut page = HistoryPage { next, ..HistoryPage::default() };
431 let mut seen = HashSet::new();
432 for (index, commit) in commits.iter().enumerate() {
433 let old_tree = match commit.parents.first() {
434 Some(parent) => match commits.get(index + 1).filter(|older| older.hash == *parent) {
435 Some(older) => Some(older.tree_hash.clone()),
436 None => objects.commit_tree(parent).await?,
437 },
438 None => None,
439 };
440 let changes = changed_files(&objects, old_tree, commit.tree_hash.clone()).await?;
441 for secret in scan_changes(&objects, &commit.hash, changes).await? {
442 if seen.insert(secret.fingerprint.clone()) {
443 page.secrets.push(secret);
444 }
445 }
446 page.commits += 1;
447 }
448 page.reads = objects.reads.get();
449 Ok(page)
450 }
451
452 /// The lockfiles on the default branch, outside vendored directories.
453 pub(crate) async fn find_lockfiles(&self, a: FindLockfilesArgs) -> Result<Lockfiles> {
454 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
455 return Ok(Lockfiles::default());
456 };
457 let git = self.store.open(&store_key(&repo)).await?;
458 let Some(head) = git.log(&repo.default_branch, 1).await?.into_iter().next() else {
459 return Ok(Lockfiles::default());
460 };
461 let mut found = Vec::new();
462 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone(), 0usize)]);
463 while let Some((prefix, tree, depth)) = queue.pop_front() {
464 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
465 match entry.kind {
466 EntryKind::Tree if depth < MAX_LOCKFILE_DEPTH && !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => {
467 queue.push_back((format!("{prefix}{}/", entry.name), entry.hash, depth + 1));
468 }
469 EntryKind::Blob if Lockfile::for_path(&entry.name).is_some() && found.len() < MAX_LOCKFILES => {
470 found.push((format!("{prefix}{}", entry.name), entry.hash));
471 }
472 _ => {}
473 }
474 }
475 }
476 let texts = try_join_all(found.iter().map(|(_, hash)| git.read_blob(hash))).await?;
477 let files = found
478 .into_iter()
479 .zip(texts)
480 .filter_map(|((path, _), bytes)| {
481 let bytes = bytes.filter(|bytes| bytes.len() <= MAX_LOCKFILE_BYTES)?;
482 Some(LockfileText { path, text: String::from_utf8(bytes).ok()? })
483 })
484 .collect();
485 Ok(Lockfiles { commit: Some(head.hash), files })
486 }
487}
488
489#[cfg(test)]
490mod tests {
491 use std::collections::HashMap;
492 use std::future::Future;
493 use std::pin::pin;
494 use std::task::{Context, Poll, Waker};
495
496 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature, TreeEntry};
497 use g1t_scan::pack::{ObjectKind, TreeItem, encode_tree, object_id};
498
499 use super::*;
500 use crate::store::Scope;
501
502 /// Runs a future that never waits, as every call to the fake store is.
503 fn run<F: Future>(future: F) -> F::Output {
504 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
505 Poll::Ready(output) => output,
506 Poll::Pending => panic!("the fake store never waits"),
507 }
508 }
509
510 /// A repository held in memory.
511 #[derive(Default)]
512 struct FakeRepo {
513 blobs: HashMap<String, Vec<u8>>,
514 trees: HashMap<String, Vec<TreeEntry>>,
515 commits: HashMap<String, Commit>,
516 }
517
518 impl GitRepo for FakeRepo {
519 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
520 unimplemented!()
521 }
522 async fn branches(&self) -> Result<Vec<Branch>> {
523 Ok(Vec::new())
524 }
525 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
526 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
527 }
528 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
529 Ok(self.commits.get(commit_hash).map(|commit| commit.parents.clone()))
530 }
531 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
532 Ok(self.trees.get(tree_hash).cloned())
533 }
534 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
535 Ok(self.blobs.get(blob_hash).cloned())
536 }
537 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
538 Ok(None)
539 }
540 async fn fork(&self, _target_key: &str) -> Result<()> {
541 Ok(())
542 }
543 }
544
545 /// Zlib with one stored (uncompressed) block, which is all a pack needs.
546 fn zlib(data: &[u8]) -> Vec<u8> {
547 let mut out = vec![0x78, 0x01, 0x01];
548 let length = data.len() as u16;
549 out.extend_from_slice(&length.to_le_bytes());
550 out.extend_from_slice(&(!length).to_le_bytes());
551 out.extend_from_slice(data);
552 let (mut a, mut b) = (1u32, 0u32);
553 for byte in data {
554 a = (a + u32::from(*byte)) % 65521;
555 b = (b + a) % 65521;
556 }
557 out.extend_from_slice(&((b << 16) | a).to_be_bytes());
558 out
559 }
560
561 fn header(code: u8, size: usize) -> Vec<u8> {
562 let mut out = Vec::new();
563 let mut byte = (code << 4) | (size & 15) as u8;
564 let mut rest = size >> 4;
565 while rest > 0 {
566 out.push(byte | 0x80);
567 byte = (rest & 0x7f) as u8;
568 rest >>= 7;
569 }
570 out.push(byte);
571 out
572 }
573
574 fn raw_id(id: &str) -> Vec<u8> {
575 id.as_bytes()
576 .chunks(2)
577 .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
578 .collect()
579 }
580
581 enum Entry {
582 Whole(ObjectKind, Vec<u8>),
583 /// A ref-delta: base id and delta.
584 Delta(String, Vec<u8>),
585 }
586
587 /// A receive-pack request: one command, then the pack.
588 fn push(entries: &[Entry]) -> Vec<u8> {
589 let command = b"0000000000000000000000000000000000000000 4807077b296e6edbf410d55e72749d3e1170c291 refs/heads/main\0report-status side-band-64k\n";
590 let mut body = format!("{:04x}", command.len() + 4).into_bytes();
591 body.extend_from_slice(command);
592 body.extend_from_slice(b"0000PACK");
593 body.extend_from_slice(&2u32.to_be_bytes());
594 body.extend_from_slice(&(entries.len() as u32).to_be_bytes());
595 for entry in entries {
596 match entry {
597 Entry::Whole(kind, data) => {
598 let code = match kind {
599 ObjectKind::Commit => 1,
600 ObjectKind::Tree => 2,
601 ObjectKind::Blob => 3,
602 ObjectKind::Tag => 4,
603 };
604 body.extend(header(code, data.len()));
605 body.extend(zlib(data));
606 }
607 Entry::Delta(base, delta) => {
608 body.extend(header(7, delta.len()));
609 body.extend(raw_id(base));
610 body.extend(zlib(delta));
611 }
612 }
613 }
614 body.extend_from_slice(&[0u8; 20]);
615 body
616 }
617
618 fn key() -> String {
619 format!("AK{}", "IAZ7Q4N2XWLM3KDTRV")
620 }
621
622 fn commit(tree: &str, parent: Option<&str>) -> Vec<u8> {
623 let parent = parent.map(|parent| format!("parent {parent}\n")).unwrap_or_default();
624 format!("tree {tree}\n{parent}author A <a@example.com> 0 +0000\ncommitter A <a@example.com> 0 +0000\n\nchange\n").into_bytes()
625 }
626
627 #[test]
628 fn a_first_push_with_a_secret_is_found_by_file_and_line() {
629 let blob = format!("REGION=eu\nAWS_KEY={}\n", key()).into_bytes();
630 let blob_id = object_id(ObjectKind::Blob, &blob);
631 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "config.env".into(), id: blob_id }]);
632 let tree_id = object_id(ObjectKind::Tree, &tree);
633 let body = push(&[
634 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
635 Entry::Whole(ObjectKind::Tree, tree),
636 Entry::Whole(ObjectKind::Blob, blob),
637 ]);
638 let found = run(scan_push(&FakeRepo::default(), &body)).unwrap();
639 assert_eq!(found.len(), 1);
640 assert_eq!((found[0].path.as_str(), found[0].line, found[0].kind.as_str()), ("config.env", 2, "aws_access_key"));
641 assert!(found[0].preview.starts_with("AKIA") && !found[0].preview.contains(&key()));
642 }
643
644 #[test]
645 fn a_thin_push_reports_only_the_lines_it_adds() {
646 // The repository already has a file with a key in it (decided on
647 // before); the push appends a line holding a second key.
648 let old = format!("first={}\n", key()).into_bytes();
649 let old_id = object_id(ObjectKind::Blob, &old);
650 let second = format!("AK{}", "IAQ9W8E7R6T5Y4U3I2");
651 let new = [old.clone(), format!("second={second}\n").into_bytes()].concat();
652 let base_tree = vec![TreeEntry { name: "app.env".into(), hash: old_id.clone(), kind: EntryKind::Blob }];
653 let base_tree_id = object_id(ObjectKind::Tree, &encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: old_id.clone() }]));
654 let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned();
655 let mut repo = FakeRepo::default();
656 repo.blobs.insert(old_id.clone(), old.clone());
657 repo.trees.insert(base_tree_id.clone(), base_tree);
658 repo.commits.insert(
659 parent_id.clone(),
660 Commit {
661 hash: parent_id.clone(),
662 tree_hash: base_tree_id,
663 message: String::new(),
664 author: Signature { name: "A".into(), email: "a@example.com".into() },
665 parents: Vec::new(),
666 authored_at: String::new(),
667 },
668 );
669 // A delta: copy the old file whole, then insert the new line.
670 let added = format!("second={second}\n").into_bytes();
671 let mut delta = vec![old.len() as u8, new.len() as u8, 0x80 | 0x10, old.len() as u8, added.len() as u8];
672 delta.extend_from_slice(&added);
673 let new_id = object_id(ObjectKind::Blob, &new);
674 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: new_id }]);
675 let tree_id = object_id(ObjectKind::Tree, &tree);
676 let body = push(&[
677 Entry::Whole(ObjectKind::Commit, commit(&tree_id, Some(&parent_id))),
678 Entry::Whole(ObjectKind::Tree, tree),
679 Entry::Delta(old_id, delta),
680 ]);
681 let found = run(scan_push(&repo, &body)).unwrap();
682 assert_eq!(found.len(), 1, "{found:?}");
683 assert_eq!((found[0].path.as_str(), found[0].line), ("app.env", 2));
684 }
685
686 #[test]
687 fn a_push_too_large_to_read_is_never_let_through_unread() {
688 let body = vec![0u8; MAX_SCANNED_PUSH + 1];
689 let error = run(scan_push(&FakeRepo::default(), &body)).unwrap_err();
690 assert!(unscannable(&error));
691 let (reason, messages) = crate::git_http::size_refusal(&crate::git_http::SizeViolation::Unscannable {
692 size: body.len() as u64,
693 cap: MAX_SCANNED_PUSH,
694 });
695 assert_eq!(reason, "the push is too large to check for secrets");
696 assert!(messages.iter().any(|line| line.contains("100.0 MB")));
697 assert!(messages.iter().any(|line| line.contains("Push in parts")));
698 }
699
700 #[test]
701 fn a_push_without_secrets_or_a_pack_finds_nothing() {
702 let blob = b"fn main() {}\n".to_vec();
703 let blob_id = object_id(ObjectKind::Blob, &blob);
704 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "main.rs".into(), id: blob_id }]);
705 let tree_id = object_id(ObjectKind::Tree, &tree);
706 let body = push(&[
707 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
708 Entry::Whole(ObjectKind::Tree, tree),
709 Entry::Whole(ObjectKind::Blob, blob),
710 ]);
711 assert!(run(scan_push(&FakeRepo::default(), &body)).unwrap().is_empty());
712 // A deletion sends commands and no pack.
713 assert!(run(scan_push(&FakeRepo::default(), b"0000")).unwrap().is_empty());
714 }
715
716 #[test]
717 fn a_push_carrying_the_pushers_private_address_is_declined_with_a_masked_address() {
718 let tree = encode_tree(&[]);
719 let tree_id = object_id(ObjectKind::Tree, &tree);
720 let mine = format!("tree {tree_id}
721author S <Sam@Gmail.com> 0 +0000
722committer S <sam@gmail.com> 0 +0000
723
724x
725").into_bytes();
726 let mine_id = object_id(ObjectKind::Commit, &mine);
727 let guard = PushEmailGuard { emails: vec!["sam@gmail.com".into()], noreply: "1abc2def+sam@users.noreply.g1t.sh".into() };
728 let body = push(&[Entry::Whole(ObjectKind::Commit, mine), Entry::Whole(ObjectKind::Tree, tree.clone())]);
729 let (found, email) = exposed_address(&body, &guard).unwrap();
730 assert_eq!(found, mine_id);
731 let message = exposed_message(&found, &email, &guard.noreply);
732 assert!(message[0].starts_with(&format!("push declined: commit {} would publish s***@gmail.com", &mine_id[..7])));
733 assert!(message[1].contains("git config user.email 1abc2def+sam@users.noreply.g1t.sh"));
734 assert!(message[2].contains("g1t.sh/settings/emails"));
735 // Someone else's commits, and no pack at all, go through.
736 let theirs = push(&[Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), Entry::Whole(ObjectKind::Tree, tree)]);
737 assert_eq!(exposed_address(&theirs, &guard), None);
738 assert_eq!(exposed_address(b"0000", &guard), None);
739 }
740}