Skip to content
Sign up

flagon-io/g1t

public
0
1,271 linesCodeBlameRaw
1//! Looking for secrets in git: in what a push adds, before it is stored
2//! (push protection), and in a repository's history, a page at a time, for
3//! the security service. Also finds the lockfiles it reads dependencies
4//! from. What counts as a secret is `g1t_scan`'s business.
5//!
6//! Push protection also keeps a person's private address out of what they
7//! push, when they asked g1t to (see [`exposed_address`]).
8//!
9//! Custom patterns (the security suite's) are looked for alongside the
10//! built-in formats, in pushes, history and files committed through g1t
11//! itself; the security service says which apply ([`Repos::patterns_for`]).
12//! It can also run a pattern over the default branch for a dry run
13//! ([`Repos::match_pattern`]), and ask a landed secret's issuer whether it
14//! still works ([`Repos::check_secret`]), without the value ever leaving
15//! this service except to that issuer.
16
17use std::cell::Cell;
18use std::collections::{HashSet, VecDeque};
19
20use futures_util::future::try_join_all;
21use g1t_contracts::User;
22use g1t_contracts::accounts::{CommitIdentityArgs, PushEmailGuard, mask_email};
23use g1t_contracts::repos::{EntryKind, Repo, RepoPath};
24use g1t_contracts::security::{
25 FindLockfilesArgs, HistoryPage, LockfileText, Lockfiles, NewSecret, PushBlockedArgs, PushVerdict,
26 ScanHistoryArgs,
27};
28use g1t_contracts::security_suite::{CheckSecretArgs, MatchPatternArgs, PatternMatch, PatternMatches, PatternSpec, PatternsForArgs, SecretValidity};
29use g1t_scan::custom::{self, Compiled};
30use g1t_scan::lockfiles::Lockfile;
31use g1t_scan::pack::{ObjectKind, Pack, TreeItem, encode_tree};
32use g1t_scan::protection::{self, Blocked};
33use worker::Result;
34
35use crate::registry::store_key;
36use crate::store::{GitRepo, GitStore};
37
38/// Where people allow a secret: the project's Security page.
39const SITE: &str = "https://g1t.sh";
40/// A push adding more commits than this is scanned for this many of them.
41const MAX_PUSH_COMMITS: usize = 300;
42/// Files compared per commit, at most.
43const MAX_FILES_PER_COMMIT: usize = 300;
44/// Bases fetched from the store for a thin pack, at most, in all rounds
45/// together. Each is one or two store reads, each with a Cache API look.
46const MAX_BASES: usize = 200;
47/// Bases asked for at a time (two reads each when the pack does not say
48/// whether a base is a blob or a tree).
49const BASES_AT_ONCE: usize = 16;
50/// The largest push that is read whole and scanned. A larger one is
51/// declined, since it cannot be checked (git_http.rs `LargePushes`).
52pub const MAX_SCANNED_PUSH: usize = 24 * 1024 * 1024;
53/// What marks an error as a push too large to scan.
54const UNSCANNABLE: &str = "push-unscannable:";
55
56/// Whether an error says the push was too large to scan.
57pub fn unscannable(error: &worker::Error) -> bool {
58 error.to_string().contains(UNSCANNABLE)
59}
60const READS_AT_ONCE: usize = 16;
61/// Directories never searched for lockfiles.
62const SKIPPED_DIRECTORIES: [&str; 8] = ["node_modules", "vendor", "target", ".git", "dist", "build", "third_party", ".venv"];
63const MAX_LOCKFILES: usize = 40;
64const MAX_LOCKFILE_DEPTH: usize = 4;
65const MAX_LOCKFILE_BYTES: usize = 16 * 1024 * 1024;
66
67fn mode(kind: EntryKind) -> &'static str {
68 match kind {
69 EntryKind::Tree => "40000",
70 EntryKind::Blob => "100644",
71 EntryKind::Exec => "100755",
72 EntryKind::Symlink => "120000",
73 EntryKind::Gitlink => "160000",
74 }
75}
76
77/// Objects for a walk: the pushed pack's first, then the repository's.
78pub(crate) struct Objects<'a, R: GitRepo> {
79 pub(crate) pack: &'a Pack,
80 pub(crate) repo: &'a R,
81 pub(crate) reads: Cell<u32>,
82}
83
84impl<R: GitRepo> Objects<'_, R> {
85 pub(crate) async fn tree(&self, id: &str) -> Result<Vec<TreeItem>> {
86 if let Some(items) = self.pack.tree(id) {
87 return Ok(items);
88 }
89 self.reads.set(self.reads.get() + 1);
90 Ok(self
91 .repo
92 .read_tree(id)
93 .await?
94 .unwrap_or_default()
95 .into_iter()
96 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
97 .collect())
98 }
99
100 async fn blob(&self, id: &str) -> Result<Option<Vec<u8>>> {
101 if let Some(bytes) = self.pack.blob(id) {
102 return Ok(Some(bytes.to_vec()));
103 }
104 self.reads.set(self.reads.get() + 1);
105 self.repo.read_blob(id).await
106 }
107
108 pub(crate) async fn commit_tree(&self, id: &str) -> Result<Option<String>> {
109 if let Some(commit) = self.pack.commit(id) {
110 return Ok(Some(commit.tree));
111 }
112 self.reads.set(self.reads.get() + 1);
113 Ok(self.repo.log(id, 1).await?.into_iter().next().map(|commit| commit.tree_hash))
114 }
115}
116
117/// A file that differs between two trees: its path, the blob it was and
118/// the blob it is.
119struct Change {
120 path: String,
121 old: Option<String>,
122 new: String,
123}
124
125/// The regular files whose content differs between two trees. Each level
126/// is read at once; identical subtrees are skipped by id.
127async fn changed_files<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: String) -> Result<Vec<Change>> {
128 let mut changes = Vec::new();
129 let mut level = vec![(String::new(), old_root, new_root)];
130 while !level.is_empty() && changes.len() < MAX_FILES_PER_COMMIT {
131 let read = try_join_all(level.iter().map(|(_, old, new)| async move {
132 let old = match old {
133 Some(old) => objects.tree(old).await?,
134 None => Vec::new(),
135 };
136 Ok::<_, worker::Error>((old, objects.tree(new).await?))
137 }))
138 .await?;
139 let mut next = Vec::new();
140 for ((prefix, _, _), (old, new)) in level.iter().zip(read) {
141 for item in &new {
142 let before = old.iter().find(|entry| entry.name == item.name);
143 if before.is_some_and(|before| before.id == item.id) {
144 continue;
145 }
146 let path = format!("{prefix}{}", item.name);
147 if item.is_tree() {
148 next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), item.id.clone()));
149 } else if item.is_file() && changes.len() < MAX_FILES_PER_COMMIT {
150 changes.push(Change {
151 path,
152 old: before.filter(|b| b.is_file()).map(|b| b.id.clone()),
153 new: item.id.clone(),
154 });
155 }
156 }
157 }
158 level = next;
159 }
160 Ok(changes)
161}
162
163/// The scanner's custom patterns, compiled; any that no longer compile are
164/// skipped.
165pub fn compiled(patterns: &[PatternSpec]) -> Vec<Compiled> {
166 let specs: Vec<custom::PatternSpec> = patterns
167 .iter()
168 .map(|spec| custom::PatternSpec {
169 id: spec.id.clone(),
170 name: spec.name.clone(),
171 pattern: spec.pattern.clone(),
172 before: spec.before.clone(),
173 after: spec.after.clone(),
174 })
175 .collect();
176 custom::compile_all(&specs)
177}
178
179/// What a custom pattern found, as the security service records it.
180fn custom_secret(hit: custom::CustomHit, path: &str, commit: &str) -> NewSecret {
181 NewSecret {
182 fingerprint: hit.fingerprint(),
183 kind: custom::KIND.to_owned(),
184 path: path.to_owned(),
185 line: hit.line,
186 commit: commit.to_owned(),
187 preview: hit.preview(),
188 test_value: None,
189 pattern_id: Some(hit.pattern_id),
190 pattern_name: Some(hit.pattern_name),
191 }
192}
193
194/// How a sentence names a secret found: its format, or its pattern.
195pub fn secret_label(secret: &NewSecret) -> Option<String> {
196 if secret.kind == custom::KIND {
197 return Some(custom::label(secret.pattern_name.as_deref().unwrap_or("custom")));
198 }
199 g1t_scan::secrets::SecretKind::parse(&secret.kind).map(|kind| kind.label().to_owned())
200}
201
202/// The secrets a new file holds, built-in and custom, for a commit made
203/// through g1t rather than pushed.
204pub fn scan_file(path: &str, bytes: &[u8], commit: &str, patterns: &[Compiled]) -> Vec<NewSecret> {
205 let mut found: Vec<NewSecret> = protection::scan_change(path, None, bytes)
206 .into_iter()
207 .map(|hit| NewSecret {
208 fingerprint: hit.fingerprint(),
209 kind: hit.kind.id().to_owned(),
210 path: path.to_owned(),
211 line: hit.line,
212 commit: commit.to_owned(),
213 preview: hit.preview(),
214 test_value: hit.test_value().map(str::to_owned),
215 pattern_id: None,
216 pattern_name: None,
217 })
218 .collect();
219 found.extend(protection::scan_change_custom(path, None, bytes, patterns).into_iter().map(|hit| custom_secret(hit, path, commit)));
220 found
221}
222
223/// The secrets each change adds, found `READS_AT_ONCE` files at a time.
224async fn scan_changes<R: GitRepo>(objects: &Objects<'_, R>, commit: &str, changes: Vec<Change>, patterns: &[Compiled]) -> Result<Vec<NewSecret>> {
225 let mut found = Vec::new();
226 let changes: Vec<Change> = changes
227 .into_iter()
228 .filter(|change| !g1t_scan::secrets::skipped_path(&change.path))
229 .collect();
230 for batch in changes.chunks(READS_AT_ONCE) {
231 // A change's old and new contents at once: the new is nearly always
232 // in the pack, the old in the repository.
233 let read = try_join_all(batch.iter().map(|change| async move {
234 let old = async {
235 match &change.old {
236 Some(old) => objects.blob(old).await,
237 None => Ok(None),
238 }
239 };
240 let (new, old) = futures_util::future::try_join(objects.blob(&change.new), old).await?;
241 Ok::<_, worker::Error>((new, old))
242 }))
243 .await?;
244 for (change, (new, old)) in batch.iter().zip(read) {
245 let Some(new) = new else { continue };
246 for hit in protection::scan_change(&change.path, old.as_deref(), &new) {
247 found.push(NewSecret {
248 fingerprint: hit.fingerprint(),
249 kind: hit.kind.id().to_owned(),
250 path: change.path.clone(),
251 line: hit.line,
252 commit: commit.to_owned(),
253 preview: hit.preview(),
254 test_value: hit.test_value().map(str::to_owned),
255 pattern_id: None,
256 pattern_name: None,
257 });
258 }
259 for hit in protection::scan_change_custom(&change.path, old.as_deref(), &new, patterns) {
260 found.push(custom_secret(hit, &change.path, commit));
261 }
262 }
263 }
264 Ok(found)
265}
266
267/// What [`supply_bases`] did for a pack.
268#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
269pub struct Bases {
270 /// Bases the pack's deltas needed from outside it when it arrived: 0
271 /// for a pack sent whole, as g1t asks for (`no-thin`, git_http.rs).
272 pub missing: usize,
273 /// Bases asked of the store, in every round.
274 pub asked: usize,
275 /// Bases still missing at the end: objects that stay unresolved.
276 pub left: usize,
277}
278
279impl Bases {
280 /// Whether the pack came thin, its deltas based on objects outside it.
281 pub fn thin(&self) -> bool {
282 self.missing > 0
283 }
284}
285
286/// Fetches what a thin pack's deltas are based on from the repository.
287/// A base the pack's own trees name is read as what they say it is; any
288/// other is asked for as a blob and as a tree together, and whichever it
289/// is answers.
290///
291/// g1t asks for packs without outside bases (`no-thin`), so this is for
292/// clients that send thin ones anyway, and it is bounded: [`MAX_BASES`] in
293/// all, over up to three rounds for delta chains, [`BASES_AT_ONCE`] at a
294/// time. Asking for hundreds at once made a large thin push fan out into
295/// as many store reads, and Cache API looks beside them, all in flight
296/// together; the reads that failed were tried again and counted against
297/// the store's breaker (store.rs `invoke`), which then turned every read
298/// after them away, so the push was answered 503. A store that says it is
299/// busy stops the reading here. Bases left missing leave their objects
300/// unresolved, and the checks go on without them, as for any pack the
301/// store cannot complete.
302pub(crate) async fn supply_bases<R: GitRepo>(pack: &mut Pack, repo: &R) -> Result<Bases> {
303 let mut bases = Bases { missing: pack.missing_bases().len(), ..Bases::default() };
304 let mut busy = false;
305 for _ in 0..3 {
306 let missing = pack.missing_bases();
307 if missing.is_empty() || busy || bases.asked >= MAX_BASES {
308 break;
309 }
310 let named = pack.named_kinds();
311 // A read that fails is a base not found, unless the store is busy,
312 // which ends the reading.
313 let settle = |read: Result<Option<(ObjectKind, Vec<u8>)>>| match read {
314 Ok(found) => Ok(found),
315 Err(error) if crate::resilience::busy(&error.to_string()).is_some() => Err(()),
316 Err(_) => Ok(None),
317 };
318 let blob = async |id: &str| settle(repo.read_blob(id).await.map(|found| found.map(|bytes| (ObjectKind::Blob, bytes))));
319 let tree = async |id: &str| {
320 settle(repo.read_tree(id).await.map(|found| {
321 found.map(|entries| {
322 let items: Vec<TreeItem> = entries
323 .into_iter()
324 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
325 .collect();
326 (ObjectKind::Tree, encode_tree(&items))
327 })
328 }))
329 };
330 let wanted: Vec<&String> = missing.iter().take(MAX_BASES - bases.asked).collect();
331 let mut progress = false;
332 for batch in wanted.chunks(BASES_AT_ONCE) {
333 let found = futures_util::future::join_all(batch.iter().map(|id| async {
334 match named.get(id.as_str()) {
335 Some(ObjectKind::Tree) => tree(id).await,
336 Some(_) => blob(id).await,
337 None => {
338 let (as_blob, as_tree) = futures_util::future::join(blob(id), tree(id)).await;
339 match (as_blob, as_tree) {
340 (Ok(Some(found)), _) | (_, Ok(Some(found))) => Ok(Some(found)),
341 (Err(()), _) | (_, Err(())) => Err(()),
342 _ => Ok(None),
343 }
344 }
345 }
346 }))
347 .await;
348 bases.asked += batch.len();
349 for (id, object) in batch.iter().zip(found) {
350 match object {
351 Ok(Some((kind, data))) => {
352 pack.supply(id, kind, data);
353 progress = true;
354 }
355 Ok(None) => {}
356 Err(()) => busy = true,
357 }
358 }
359 if busy {
360 break;
361 }
362 }
363 if !progress {
364 break;
365 }
366 }
367 bases.left = pack.missing_bases().len();
368 Ok(bases)
369}
370
371/// The secrets the commits in a push add, each secret once. A push too
372/// large to read is an error ([`unscannable`]): it is declined, never let
373/// through unread. A pack that cannot be read for another reason is let
374/// through, and said so in the logs; the store will judge it.
375#[cfg(test)]
376pub async fn scan_push<R: GitRepo>(repo: &R, body: &[u8], patterns: &[Compiled]) -> Result<Vec<NewSecret>> {
377 if body.len() > MAX_SCANNED_PUSH {
378 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {} bytes", body.len())));
379 }
380 let mut pack = crate::push_checks::read_pack(body);
381 if let Ok(pack) = &mut pack {
382 supply_bases(pack, repo).await?;
383 }
384 scan_pack(repo, body.len(), &pack, patterns).await
385}
386
387/// [`scan_push`] for a push of `size` bytes whose pack was read already,
388/// with its bases supplied (push_checks.rs).
389pub(crate) async fn scan_pack<R: GitRepo>(repo: &R, size: usize, pack: &std::result::Result<Pack, String>, patterns: &[Compiled]) -> Result<Vec<NewSecret>> {
390 if size > MAX_SCANNED_PUSH {
391 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {size} bytes")));
392 }
393 let pack = match pack {
394 Ok(pack) => pack,
395 Err(problem) if problem.contains("too large") => {
396 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {problem}")));
397 }
398 Err(problem) => {
399 worker::console_error!("push not scanned for secrets: {problem}");
400 return Ok(Vec::new());
401 }
402 };
403 if pack.unresolved() > 0 {
404 worker::console_error!("{} objects of a push could not be resolved for scanning", pack.unresolved());
405 }
406 let objects = Objects { pack, repo, reads: Cell::new(0) };
407 let objects = &objects;
408 let commits: Vec<(String, g1t_scan::pack::CommitInfo)> = pack
409 .commits()
410 .iter()
411 .take(MAX_PUSH_COMMITS)
412 .filter_map(|id| Some((id.clone(), pack.commit(id)?)))
413 .collect();
414 // The trees of the parents the pack does not hold, all read up front:
415 // usually the one commit the push builds on.
416 let mut outside: Vec<&String> = commits
417 .iter()
418 .filter_map(|(_, commit)| commit.parents.first())
419 .filter(|parent| !pack.contains(parent))
420 .collect();
421 outside.sort();
422 outside.dedup();
423 let outside_trees: std::collections::HashMap<&String, Option<String>> = outside
424 .iter()
425 .copied()
426 .zip(try_join_all(outside.iter().map(|parent| objects.commit_tree(parent))).await?)
427 .collect();
428 let outside_trees = &outside_trees;
429 // What each commit changes, all read at once.
430 let changed = try_join_all(commits.iter().map(|(_, commit)| async move {
431 let old_tree = match commit.parents.first() {
432 Some(parent) => match outside_trees.get(parent) {
433 Some(tree) => tree.clone(),
434 None => objects.commit_tree(parent).await?,
435 },
436 None => None,
437 };
438 changed_files(objects, old_tree, commit.tree.clone()).await
439 }))
440 .await?;
441 let mut found = Vec::new();
442 let mut seen_blobs = HashSet::new();
443 let mut seen_secrets = HashSet::new();
444 for ((id, _), changes) in commits.iter().zip(changed) {
445 // Only content the push brings is new; a blob the repository has
446 // was looked at when it arrived.
447 let changes: Vec<Change> = changes
448 .into_iter()
449 .filter(|change| pack.contains(&change.new) && seen_blobs.insert((change.path.clone(), change.new.clone())))
450 .collect();
451 for secret in scan_changes(objects, id, changes, patterns).await? {
452 if seen_secrets.insert(secret.fingerprint.clone()) {
453 found.push(secret);
454 }
455 }
456 }
457 Ok(found)
458}
459
460/// A commit in a push that would publish one of the pusher's own
461/// addresses while they keep it private: its id and the address. Only the
462/// commits the push adds are read; anyone else's address is no concern
463/// here. A pack that cannot be read is let through.
464#[cfg(test)]
465pub fn exposed_address(body: &[u8], guard: &PushEmailGuard) -> Option<(String, String)> {
466 if body.len() > MAX_SCANNED_PUSH {
467 return None;
468 }
469 exposed_in(&Pack::parse(&body[g1t_scan::pack::pack_start(body)?..]).ok()?, guard)
470}
471
472/// [`exposed_address`] for a pack read already.
473pub(crate) fn exposed_in(pack: &Pack, guard: &PushEmailGuard) -> Option<(String, String)> {
474 pack.commits().iter().find_map(|id| {
475 let commit = pack.commit(id)?;
476 [commit.author_email, commit.committer_email]
477 .into_iter()
478 .flatten()
479 .find(|email| guard.exposes(email))
480 .map(|email| (id.clone(), email))
481 })
482}
483
484/// What git shows a person whose push would publish their private address.
485pub fn exposed_message(commit: &str, email: &str, noreply: &str) -> Vec<String> {
486 let short: String = commit.chars().take(7).collect();
487 vec![
488 format!(
489 "push declined: commit {short} would publish {} while your email is private.",
490 mask_email(&email.to_lowercase())
491 ),
492 format!("Commit with {noreply} (git config user.email {noreply}) and amend,"),
493 format!("or change this in {}/settings/emails.", SITE.trim_start_matches("https://")),
494 ]
495}
496
497impl<S: GitStore> crate::Repos<S> {
498 /// What a push by `pusher` must not publish: their own addresses, when
499 /// they keep them private and block such pushes. An agent's push is
500 /// its person's. `None` when nothing is guarded, or identity cannot say.
501 pub(crate) async fn push_email_guard(&self, pusher: Option<&User>) -> Option<PushEmailGuard> {
502 let pusher = pusher?;
503 let person = pusher.acting.as_ref().map_or(pusher.id.clone(), |acting| acting.on_behalf_of.id.clone());
504 let identity = self.identity.as_ref()?;
505 g1t_kit::call::<_, Option<PushEmailGuard>>(identity, "push_email_guard", &CommitIdentityArgs { user_id: person })
506 .await
507 .unwrap_or_else(|error| {
508 worker::console_error!("push_email_guard failed: {error}");
509 None
510 })
511 }
512
513 /// The custom patterns the security service says `repo` is scanned
514 /// with; none when it cannot say.
515 pub(crate) async fn patterns_for(&self, repo: &Repo) -> Vec<PatternSpec> {
516 let Some(security) = &self.security else { return Vec::new() };
517 g1t_kit::call(
518 security,
519 "patterns_for",
520 &PatternsForArgs { repo_id: repo.id.clone(), namespace: repo.namespace.clone(), private: Some(repo.is_private) },
521 )
522 .await
523 .unwrap_or_else(|error| {
524 worker::console_error!("patterns_for failed: {error}");
525 Vec::new()
526 })
527 }
528
529 /// Of `found` in a change to `owner`, the secrets nobody let through:
530 /// the security service records them all and says which were allowed.
531 pub(crate) async fn blocked(&self, owner: &Repo, pusher: Option<&User>, found: Vec<NewSecret>) -> Vec<Blocked> {
532 let owner_path = RepoPath { namespace: owner.namespace.clone(), name: owner.name.clone() };
533 let verdict = match &self.security {
534 Some(security) => g1t_kit::call::<_, PushVerdict>(
535 security,
536 "push_blocked",
537 &PushBlockedArgs {
538 repo_id: owner.id.clone(),
539 path: owner_path.clone(),
540 pusher: pusher.map(|user| user.username.clone()),
541 secrets: found.clone(),
542 private: Some(owner.is_private),
543 },
544 )
545 .await
546 .unwrap_or_else(|error| {
547 worker::console_error!("push_blocked failed: {error}");
548 PushVerdict::default()
549 }),
550 None => PushVerdict::default(),
551 };
552 found
553 .iter()
554 .filter(|secret| !verdict.allowed.contains(&secret.fingerprint))
555 // A likely test value is recorded, never a reason to refuse.
556 .filter(|secret| secret.test_value.is_none())
557 .filter_map(|secret| {
558 let label = secret_label(secret)?;
559 let id = verdict.ids.iter().find(|(fingerprint, _)| *fingerprint == secret.fingerprint);
560 Some(Blocked {
561 label,
562 path: secret.path.clone(),
563 line: secret.line,
564 commit: secret.commit.clone(),
565 // Where it can be bypassed with a reason, or allowed.
566 allow_url: id.map(|(_, id)| {
567 format!("{SITE}/{}/{}/security/secret-scanning/{id}", owner_path.namespace, owner_path.name)
568 }),
569 })
570 })
571 .collect()
572 }
573
574 /// Push protection for a file committed through g1t (`commit_file`):
575 /// the refusal, naming each secret and where to bypass it, or `None`.
576 pub(crate) async fn protect_file(&self, repo: &Repo, actor: &User, path: &str, content: &[u8], commit: &str) -> Option<String> {
577 let patterns = compiled(&self.patterns_for(repo).await);
578 let found = scan_file(path, content, commit, &patterns);
579 if found.is_empty() {
580 return None;
581 }
582 let blocked = self.blocked(repo, Some(actor), found).await;
583 if blocked.is_empty() {
584 return None;
585 }
586 Some(protection::explain(&blocked).join("\n"))
587 }
588
589 /// A page of the default branch's history, scanned for secrets.
590 pub(crate) async fn scan_history(&self, a: ScanHistoryArgs) -> Result<HistoryPage> {
591 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
592 return Ok(HistoryPage::default());
593 };
594 let git = self.store.open(&store_key(&repo)).await?;
595 let limit = a.limit.clamp(1, 100);
596 // A page of a pushed range starts at its newest commit, and the
597 // history of the default branch at its head.
598 let start = a.after.or(a.from).unwrap_or_else(|| repo.default_branch.clone());
599 let mut commits = git.log(&start, limit + 1).await?;
600 let mut next = (commits.len() > limit as usize).then(|| commits.pop().map(|commit| commit.hash)).flatten();
601 // A range ends where the branch was before the push.
602 if let Some(until) = a.until.as_deref()
603 && let Some(at) = commits.iter().position(|commit| commit.hash == until)
604 {
605 commits.truncate(at);
606 next = None;
607 }
608 if a.until.is_some() && next.as_deref() == a.until.as_deref() {
609 next = None;
610 }
611 let empty = Pack::default();
612 let objects = Objects { pack: &empty, repo: &git, reads: Cell::new(1) };
613 let patterns = compiled(&a.patterns);
614 let mut page = HistoryPage { next, ..HistoryPage::default() };
615 let mut seen = HashSet::new();
616 for (index, commit) in commits.iter().enumerate() {
617 let old_tree = match commit.parents.first() {
618 Some(parent) => match commits.get(index + 1).filter(|older| older.hash == *parent) {
619 Some(older) => Some(older.tree_hash.clone()),
620 None => objects.commit_tree(parent).await?,
621 },
622 None => None,
623 };
624 let changes = changed_files(&objects, old_tree, commit.tree_hash.clone()).await?;
625 for secret in scan_changes(&objects, &commit.hash, changes, &patterns).await? {
626 if seen.insert(secret.fingerprint.clone()) {
627 page.secrets.push(secret);
628 }
629 }
630 page.commits += 1;
631 }
632 page.reads = objects.reads.get();
633 Ok(page)
634 }
635
636 /// The lockfiles on the default branch, outside vendored directories.
637 pub(crate) async fn find_lockfiles(&self, a: FindLockfilesArgs) -> Result<Lockfiles> {
638 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
639 return Ok(Lockfiles::default());
640 };
641 let git = self.store.open(&store_key(&repo)).await?;
642 let at = a.git_ref.as_deref().unwrap_or(&repo.default_branch);
643 let Some(head) = git.log(at, 1).await?.into_iter().next() else {
644 return Ok(Lockfiles::default());
645 };
646 let mut found = Vec::new();
647 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone(), 0usize)]);
648 while let Some((prefix, tree, depth)) = queue.pop_front() {
649 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
650 match entry.kind {
651 EntryKind::Tree if depth < MAX_LOCKFILE_DEPTH && !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => {
652 queue.push_back((format!("{prefix}{}/", entry.name), entry.hash, depth + 1));
653 }
654 EntryKind::Blob if Lockfile::for_path(&entry.name).is_some() && found.len() < MAX_LOCKFILES => {
655 found.push((format!("{prefix}{}", entry.name), entry.hash));
656 }
657 _ => {}
658 }
659 }
660 }
661 let texts = try_join_all(found.iter().map(|(_, hash)| git.read_blob(hash))).await?;
662 let files = found
663 .into_iter()
664 .zip(texts)
665 .filter_map(|((path, _), bytes)| {
666 let bytes = bytes.filter(|bytes| bytes.len() <= MAX_LOCKFILE_BYTES)?;
667 Some(LockfileText { path, text: String::from_utf8(bytes).ok()? })
668 })
669 .collect();
670 Ok(Lockfiles { commit: Some(head.hash), files })
671 }
672
673 /// A dry run of a custom pattern over the default branch's files, up to
674 /// [`MATCH_FILES`] files and [`MATCH_BYTES`] of text, skipping what
675 /// secret scanning skips. Nothing is recorded.
676 pub(crate) async fn match_pattern(&self, a: MatchPatternArgs) -> Result<PatternMatches> {
677 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
678 return Ok(PatternMatches::default());
679 };
680 let patterns = compiled(std::slice::from_ref(&a.pattern));
681 let Some(pattern) = patterns.first() else {
682 return Ok(PatternMatches::default());
683 };
684 let git = self.store.open(&store_key(&repo)).await?;
685 let Some(head) = git.log(&repo.default_branch, 1).await?.into_iter().next() else {
686 return Ok(PatternMatches::default());
687 };
688 let mut result = PatternMatches { commit: Some(head.hash.clone()), ..PatternMatches::default() };
689 let mut files = Vec::new();
690 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone())]);
691 while let Some((prefix, tree)) = queue.pop_front() {
692 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
693 let path = format!("{prefix}{}", entry.name);
694 match entry.kind {
695 EntryKind::Tree if !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => queue.push_back((format!("{path}/"), entry.hash)),
696 EntryKind::Blob | EntryKind::Exec if !g1t_scan::secrets::skipped_path(&path) => {
697 if files.len() == MATCH_FILES {
698 result.truncated = true;
699 } else {
700 files.push((path, entry.hash));
701 }
702 }
703 _ => {}
704 }
705 }
706 }
707 let mut bytes = 0usize;
708 let limit = a.limit.clamp(1, 200) as usize;
709 for batch in files.chunks(READS_AT_ONCE) {
710 if bytes > MATCH_BYTES || result.matches.len() >= limit {
711 result.truncated = true;
712 break;
713 }
714 let read = try_join_all(batch.iter().map(|(_, hash)| git.read_blob(hash))).await?;
715 for ((path, _), blob) in batch.iter().zip(read) {
716 let Some(blob) = blob else { continue };
717 bytes += blob.len();
718 result.files_scanned += 1;
719 let Some(text) = protection::text_of(path, &blob) else { continue };
720 let lines: Vec<&str> = text.lines().collect();
721 for hit in custom::scan_lines(text, std::slice::from_ref(pattern), |_| true) {
722 if result.matches.len() >= limit {
723 result.truncated = true;
724 break;
725 }
726 let line = lines.get(hit.line as usize - 1).copied().unwrap_or_default();
727 result.matches.push(PatternMatch { path: path.clone(), line: hit.line, preview: custom::masked_line(line, &hit.value) });
728 }
729 }
730 }
731 Ok(result)
732 }
733
734 /// Asks a landed secret's issuer whether it still works: finds it again
735 /// by its fingerprint at `commit`:`path` near `line`, and makes the
736 /// issuer's own read-only check over HTTPS. The value goes nowhere else.
737 pub(crate) async fn check_secret(&self, a: CheckSecretArgs) -> Result<SecretValidity> {
738 let unknown = |detail: &str| SecretValidity { validity: "unknown".to_owned(), detail: Some(detail.to_owned()) };
739 let Some(kind) = g1t_scan::secrets::SecretKind::parse(&a.kind) else {
740 return Ok(SecretValidity { validity: "unsupported".to_owned(), detail: None });
741 };
742 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
743 return Ok(unknown("no such repository"));
744 };
745 let git = self.store.open(&store_key(&repo)).await?;
746 let Some(bytes) = git.read_file(&a.commit, &a.path).await? else {
747 return Ok(unknown("the file is not at that commit"));
748 };
749 let Some(text) = protection::text_of(&a.path, &bytes) else {
750 return Ok(unknown("the file cannot be read as text"));
751 };
752 let near = |line: u32| line + 2 >= a.line && line <= a.line + 2;
753 let Some(hit) = g1t_scan::secrets::scan_lines(text, near).into_iter().find(|hit| hit.fingerprint() == a.fingerprint) else {
754 return Ok(unknown("the secret is no longer where it was found"));
755 };
756 let Some(probe) = g1t_scan::validity::check_for(kind, &hit.value) else {
757 return Ok(SecretValidity { validity: "unsupported".to_owned(), detail: None });
758 };
759 let headers = worker::Headers::new();
760 headers.set("user-agent", "g1t secret validity check (+https://docs.g1t.sh/guides/security/secret-protection/)")?;
761 for (name, value) in &probe.headers {
762 headers.set(name, value)?;
763 }
764 let mut init = worker::RequestInit::new();
765 init.with_method(if probe.method == "POST" { worker::Method::Post } else { worker::Method::Get }).with_headers(headers);
766 if let Some(body) = &probe.body {
767 init.with_body(Some(body.clone().into()));
768 }
769 let answer = async {
770 let mut response = worker::Fetch::Request(worker::Request::new_with_init(probe.url, &init)?).send().await?;
771 let status = response.status_code();
772 let body = if probe.reader == g1t_scan::validity::Reader::SlackOk { response.text().await.unwrap_or_default() } else { String::new() };
773 Ok::<_, worker::Error>((status, body))
774 }
775 .await;
776 Ok(match answer {
777 Ok((status, body)) => {
778 let validity = g1t_scan::validity::read(probe.reader, kind, status, &body);
779 SecretValidity {
780 validity: validity.as_str().to_owned(),
781 detail: (validity == g1t_scan::validity::Validity::Unknown).then(|| format!("the issuer answered {status}")),
782 }
783 }
784 Err(error) => unknown(&format!("the issuer could not be reached: {error}")),
785 })
786 }
787}
788
789/// Files a dry run reads, at most, and text in all.
790const MATCH_FILES: usize = 2_000;
791const MATCH_BYTES: usize = 20 * 1024 * 1024;
792
793#[cfg(test)]
794mod tests {
795 use std::cell::Cell;
796 use std::collections::HashMap;
797 use std::future::Future;
798 use std::pin::pin;
799 use std::task::{Context, Poll, Waker};
800
801 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature, TreeEntry};
802 use g1t_scan::pack::{ObjectKind, TreeItem, encode_tree, object_id};
803
804 use super::*;
805 use crate::store::Scope;
806
807 /// Runs a future that never waits, as every call to the fake store is.
808 fn run<F: Future>(future: F) -> F::Output {
809 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
810 Poll::Ready(output) => output,
811 Poll::Pending => panic!("the fake store never waits"),
812 }
813 }
814
815 /// A repository held in memory.
816 #[derive(Default)]
817 struct FakeRepo {
818 blobs: HashMap<String, Vec<u8>>,
819 trees: HashMap<String, Vec<TreeEntry>>,
820 commits: HashMap<String, Commit>,
821 /// How often each kind of read was asked for.
822 blob_reads: Cell<u32>,
823 tree_reads: Cell<u32>,
824 log_reads: Cell<u32>,
825 /// Each read waits once before it answers, as a store's does, so
826 /// that reads asked for together are in flight together.
827 waits: bool,
828 in_flight: Cell<u32>,
829 most_in_flight: Cell<u32>,
830 /// Every read fails as a busy store's does.
831 busy: bool,
832 }
833
834 impl FakeRepo {
835 async fn reading(&self) -> Result<()> {
836 if self.busy {
837 return Err(crate::resilience::Busy { rate_limited: true, retry_after: 5, read_only: false }.error("readBlob"));
838 }
839 if self.waits {
840 self.in_flight.set(self.in_flight.get() + 1);
841 self.most_in_flight.set(self.most_in_flight.get().max(self.in_flight.get()));
842 YieldOnce(false).await;
843 self.in_flight.set(self.in_flight.get() - 1);
844 }
845 Ok(())
846 }
847 }
848
849 /// Waits once, then is ready.
850 struct YieldOnce(bool);
851
852 impl Future for YieldOnce {
853 type Output = ();
854 fn poll(mut self: std::pin::Pin<&mut Self>, context: &mut Context<'_>) -> Poll<()> {
855 if self.0 {
856 return Poll::Ready(());
857 }
858 self.0 = true;
859 context.waker().wake_by_ref();
860 Poll::Pending
861 }
862 }
863
864 /// Runs a future to its end, polling it until it is ready.
865 fn run_waiting<F: Future>(future: F) -> F::Output {
866 let mut future = pin!(future);
867 loop {
868 if let Poll::Ready(output) = future.as_mut().poll(&mut Context::from_waker(Waker::noop())) {
869 return output;
870 }
871 }
872 }
873
874 impl GitRepo for FakeRepo {
875 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
876 unimplemented!()
877 }
878 async fn branches(&self) -> Result<Vec<Branch>> {
879 Ok(Vec::new())
880 }
881 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
882 self.log_reads.set(self.log_reads.get() + 1);
883 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
884 }
885 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
886 Ok(self.commits.get(commit_hash).map(|commit| commit.parents.clone()))
887 }
888 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
889 self.tree_reads.set(self.tree_reads.get() + 1);
890 self.reading().await?;
891 Ok(self.trees.get(tree_hash).cloned())
892 }
893 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
894 self.blob_reads.set(self.blob_reads.get() + 1);
895 self.reading().await?;
896 Ok(self.blobs.get(blob_hash).cloned())
897 }
898 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
899 Ok(None)
900 }
901 async fn fork(&self, _target_key: &str) -> Result<()> {
902 Ok(())
903 }
904 }
905
906 /// Zlib with one stored (uncompressed) block, which is all a pack needs.
907 fn zlib(data: &[u8]) -> Vec<u8> {
908 let mut out = vec![0x78, 0x01, 0x01];
909 let length = data.len() as u16;
910 out.extend_from_slice(&length.to_le_bytes());
911 out.extend_from_slice(&(!length).to_le_bytes());
912 out.extend_from_slice(data);
913 let (mut a, mut b) = (1u32, 0u32);
914 for byte in data {
915 a = (a + u32::from(*byte)) % 65521;
916 b = (b + a) % 65521;
917 }
918 out.extend_from_slice(&((b << 16) | a).to_be_bytes());
919 out
920 }
921
922 fn header(code: u8, size: usize) -> Vec<u8> {
923 let mut out = Vec::new();
924 let mut byte = (code << 4) | (size & 15) as u8;
925 let mut rest = size >> 4;
926 while rest > 0 {
927 out.push(byte | 0x80);
928 byte = (rest & 0x7f) as u8;
929 rest >>= 7;
930 }
931 out.push(byte);
932 out
933 }
934
935 fn raw_id(id: &str) -> Vec<u8> {
936 id.as_bytes()
937 .chunks(2)
938 .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
939 .collect()
940 }
941
942 enum Entry {
943 Whole(ObjectKind, Vec<u8>),
944 /// A ref-delta: base id and delta.
945 Delta(String, Vec<u8>),
946 }
947
948 /// A receive-pack request: one command, then the pack.
949 fn push(entries: &[Entry]) -> Vec<u8> {
950 let command = b"0000000000000000000000000000000000000000 4807077b296e6edbf410d55e72749d3e1170c291 refs/heads/main\0report-status side-band-64k\n";
951 let mut body = format!("{:04x}", command.len() + 4).into_bytes();
952 body.extend_from_slice(command);
953 body.extend_from_slice(b"0000PACK");
954 body.extend_from_slice(&2u32.to_be_bytes());
955 body.extend_from_slice(&(entries.len() as u32).to_be_bytes());
956 for entry in entries {
957 match entry {
958 Entry::Whole(kind, data) => {
959 let code = match kind {
960 ObjectKind::Commit => 1,
961 ObjectKind::Tree => 2,
962 ObjectKind::Blob => 3,
963 ObjectKind::Tag => 4,
964 };
965 body.extend(header(code, data.len()));
966 body.extend(zlib(data));
967 }
968 Entry::Delta(base, delta) => {
969 body.extend(header(7, delta.len()));
970 body.extend(raw_id(base));
971 body.extend(zlib(delta));
972 }
973 }
974 }
975 body.extend_from_slice(&[0u8; 20]);
976 body
977 }
978
979 fn key() -> String {
980 format!("AK{}", "IAZ7Q4N2XWLM3KDTRV")
981 }
982
983 fn commit(tree: &str, parent: Option<&str>) -> Vec<u8> {
984 let parent = parent.map(|parent| format!("parent {parent}\n")).unwrap_or_default();
985 format!("tree {tree}\n{parent}author A <a@example.com> 0 +0000\ncommitter A <a@example.com> 0 +0000\n\nchange\n").into_bytes()
986 }
987
988 #[test]
989 fn a_first_push_with_a_secret_is_found_by_file_and_line() {
990 let blob = format!("REGION=eu\nAWS_KEY={}\n", key()).into_bytes();
991 let blob_id = object_id(ObjectKind::Blob, &blob);
992 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "config.env".into(), id: blob_id }]);
993 let tree_id = object_id(ObjectKind::Tree, &tree);
994 let body = push(&[
995 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
996 Entry::Whole(ObjectKind::Tree, tree),
997 Entry::Whole(ObjectKind::Blob, blob),
998 ]);
999 let found = run(scan_push(&FakeRepo::default(), &body, &[])).unwrap();
1000 assert_eq!(found.len(), 1);
1001 assert_eq!((found[0].path.as_str(), found[0].line, found[0].kind.as_str()), ("config.env", 2, "aws_access_key"));
1002 assert!(found[0].preview.starts_with("AKIA") && !found[0].preview.contains(&key()));
1003 }
1004
1005 #[test]
1006 fn a_thin_push_reports_only_the_lines_it_adds() {
1007 // The repository already has a file with a key in it (decided on
1008 // before); the push appends a line holding a second key.
1009 let old = format!("first={}\n", key()).into_bytes();
1010 let old_id = object_id(ObjectKind::Blob, &old);
1011 let second = format!("AK{}", "IAQ9W8E7R6T5Y4U3I2");
1012 let new = [old.clone(), format!("second={second}\n").into_bytes()].concat();
1013 let base_tree = vec![TreeEntry { name: "app.env".into(), hash: old_id.clone(), kind: EntryKind::Blob }];
1014 let base_tree_id = object_id(ObjectKind::Tree, &encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: old_id.clone() }]));
1015 let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned();
1016 let mut repo = FakeRepo::default();
1017 repo.blobs.insert(old_id.clone(), old.clone());
1018 repo.trees.insert(base_tree_id.clone(), base_tree);
1019 repo.commits.insert(
1020 parent_id.clone(),
1021 Commit {
1022 hash: parent_id.clone(),
1023 tree_hash: base_tree_id,
1024 message: String::new(),
1025 author: Signature { name: "A".into(), email: "a@example.com".into() },
1026 parents: Vec::new(),
1027 authored_at: String::new(),
1028 },
1029 );
1030 // A delta: copy the old file whole, then insert the new line.
1031 let added = format!("second={second}\n").into_bytes();
1032 let mut delta = vec![old.len() as u8, new.len() as u8, 0x80 | 0x10, old.len() as u8, added.len() as u8];
1033 delta.extend_from_slice(&added);
1034 let new_id = object_id(ObjectKind::Blob, &new);
1035 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: new_id }]);
1036 let tree_id = object_id(ObjectKind::Tree, &tree);
1037 let body = push(&[
1038 Entry::Whole(ObjectKind::Commit, commit(&tree_id, Some(&parent_id))),
1039 Entry::Whole(ObjectKind::Tree, tree),
1040 Entry::Delta(old_id, delta),
1041 ]);
1042 let found = run(scan_push(&repo, &body, &[])).unwrap();
1043 assert_eq!(found.len(), 1, "{found:?}");
1044 assert_eq!((found[0].path.as_str(), found[0].line), ("app.env", 2));
1045 }
1046
1047 #[test]
1048 fn a_base_is_asked_for_as_what_the_pack_names_it_or_both_ways_at_once() {
1049 // A file's old version, which no tree in the pack names: asked for
1050 // as a blob and as a tree together, and found as a blob.
1051 let old = b"one
1052".to_vec();
1053 let old_id = object_id(ObjectKind::Blob, &old);
1054 let mut repo = FakeRepo::default();
1055 repo.blobs.insert(old_id.clone(), old.clone());
1056 let mut delta = vec![old.len() as u8, (old.len() + 4) as u8, 0x80 | 0x10, old.len() as u8, 4];
1057 delta.extend_from_slice(b"two
1058");
1059 let body = push(&[Entry::Delta(old_id.clone(), delta)]);
1060 let mut pack = crate::push_checks::read_pack(&body).unwrap();
1061 run(supply_bases(&mut pack, &repo)).unwrap();
1062 assert_eq!(pack.unresolved(), 0);
1063 assert_eq!((repo.blob_reads.get(), repo.tree_reads.get()), (1, 1));
1064
1065 // A directory the pack's root tree names as a tree: read as one.
1066 let file = object_id(ObjectKind::Blob, b"x");
1067 let listed = [TreeItem { mode: "100644".into(), name: "a".into(), id: file.clone() }];
1068 let sub = encode_tree(&listed);
1069 let sub_id = object_id(ObjectKind::Tree, &sub);
1070 let mut repo = FakeRepo::default();
1071 repo.trees.insert(sub_id.clone(), vec![TreeEntry { name: "a".into(), hash: file, kind: EntryKind::Blob }]);
1072 let root = encode_tree(&[TreeItem { mode: "40000".into(), name: "src".into(), id: sub_id.clone() }]);
1073 let delta = vec![sub.len() as u8, sub.len() as u8, 0x80 | 0x10, sub.len() as u8];
1074 let body = push(&[Entry::Whole(ObjectKind::Tree, root), Entry::Delta(sub_id, delta)]);
1075 let mut pack = crate::push_checks::read_pack(&body).unwrap();
1076 run(supply_bases(&mut pack, &repo)).unwrap();
1077 assert_eq!(pack.unresolved(), 0);
1078 assert_eq!((repo.blob_reads.get(), repo.tree_reads.get()), (0, 1));
1079 }
1080
1081 #[test]
1082 fn a_pack_sent_whole_is_checked_without_reading_a_base() {
1083 // What git sends when told `no-thin`: a delta's base is in the pack
1084 // with it. Here the second file is a delta on the first.
1085 let first = b"REGION=eu
1086".to_vec();
1087 let first_id = object_id(ObjectKind::Blob, &first);
1088 let added = format!("AWS_KEY={}
1089", key()).into_bytes();
1090 let second = [first.clone(), added.clone()].concat();
1091 let mut delta = vec![first.len() as u8, second.len() as u8, 0x80 | 0x10, first.len() as u8, added.len() as u8];
1092 delta.extend_from_slice(&added);
1093 let tree = encode_tree(&[
1094 TreeItem { mode: "100644".into(), name: "a.env".into(), id: first_id.clone() },
1095 TreeItem { mode: "100644".into(), name: "b.env".into(), id: object_id(ObjectKind::Blob, &second) },
1096 ]);
1097 let tree_id = object_id(ObjectKind::Tree, &tree);
1098 let body = push(&[
1099 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
1100 Entry::Whole(ObjectKind::Tree, tree),
1101 Entry::Whole(ObjectKind::Blob, first),
1102 Entry::Delta(first_id, delta),
1103 ]);
1104 let repo = FakeRepo::default();
1105 let mut pack = crate::push_checks::read_pack(&body).unwrap();
1106 let bases = run(supply_bases(&mut pack, &repo)).unwrap();
1107 assert_eq!(bases, Bases::default());
1108 assert!(!bases.thin());
1109 assert_eq!(pack.unresolved(), 0);
1110 let found = run(scan_pack(&repo, body.len(), &Ok(pack), &[])).unwrap();
1111 assert_eq!(found.len(), 1);
1112 assert_eq!((found[0].path.as_str(), found[0].line), ("b.env", 2));
1113 // Nothing was read from the store: not a base, not a file.
1114 assert_eq!((repo.blob_reads.get(), repo.tree_reads.get(), repo.log_reads.get()), (0, 0, 0));
1115 }
1116
1117 /// A pack of `count` deltas, each on a different base outside it.
1118 fn thin_pack(count: usize) -> Pack {
1119 let entries: Vec<Entry> = (1..=count)
1120 .map(|at| Entry::Delta(format!("{at:040x}"), vec![1, 2, 0x02, b'h', b'i']))
1121 .collect();
1122 crate::push_checks::read_pack(&push(&entries)).unwrap()
1123 }
1124
1125 #[test]
1126 fn a_large_thin_push_reads_a_bounded_number_of_bases_a_few_at_a_time() {
1127 // 300 bases the store does not have, none named by a tree: before,
1128 // each round asked for 500 at once, two reads each, three rounds.
1129 let mut pack = thin_pack(300);
1130 let repo = FakeRepo { waits: true, ..FakeRepo::default() };
1131 let bases = run_waiting(supply_bases(&mut pack, &repo)).unwrap();
1132 assert_eq!(bases, Bases { missing: 300, asked: MAX_BASES, left: 300 });
1133 assert!(bases.thin());
1134 // Asked once each, as a blob and as a tree, and never more at once
1135 // than a batch's.
1136 assert_eq!((repo.blob_reads.get(), repo.tree_reads.get()), (MAX_BASES as u32, MAX_BASES as u32));
1137 assert_eq!(repo.most_in_flight.get(), 2 * BASES_AT_ONCE as u32);
1138 }
1139
1140 #[test]
1141 fn a_busy_store_stops_the_reading_of_bases() {
1142 let mut pack = thin_pack(100);
1143 let repo = FakeRepo { busy: true, ..FakeRepo::default() };
1144 let bases = run(supply_bases(&mut pack, &repo)).unwrap();
1145 // One batch, then no more: the checks go on, and the store's own
1146 // answer to them says it is busy.
1147 assert_eq!(bases, Bases { missing: 100, asked: BASES_AT_ONCE, left: 100 });
1148 assert_eq!(repo.blob_reads.get(), BASES_AT_ONCE as u32);
1149 }
1150
1151 #[test]
1152 fn the_commit_a_push_builds_on_is_read_once_however_many_commits_build_on_it() {
1153 // Two branches pushed at once, each one commit on the same parent.
1154 let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned();
1155 let base_tree_id = object_id(ObjectKind::Tree, &[]);
1156 let mut repo = FakeRepo::default();
1157 repo.trees.insert(base_tree_id.clone(), Vec::new());
1158 repo.commits.insert(
1159 parent_id.clone(),
1160 Commit {
1161 hash: parent_id.clone(),
1162 tree_hash: base_tree_id,
1163 message: String::new(),
1164 author: Signature { name: "A".into(), email: "a@example.com".into() },
1165 parents: Vec::new(),
1166 authored_at: String::new(),
1167 },
1168 );
1169 let mut entries = Vec::new();
1170 for (name, line) in [("a.env", format!("KEY={}
1171", key())), ("b.txt", "nothing here
1172".to_owned())] {
1173 let blob = line.into_bytes();
1174 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: name.into(), id: object_id(ObjectKind::Blob, &blob) }]);
1175 entries.push(Entry::Whole(ObjectKind::Commit, commit(&object_id(ObjectKind::Tree, &tree), Some(&parent_id))));
1176 entries.push(Entry::Whole(ObjectKind::Tree, tree));
1177 entries.push(Entry::Whole(ObjectKind::Blob, blob));
1178 }
1179 let found = run(scan_push(&repo, &push(&entries), &[])).unwrap();
1180 assert_eq!(found.len(), 1);
1181 assert_eq!(found[0].path, "a.env");
1182 assert_eq!(repo.log_reads.get(), 1);
1183 }
1184
1185 #[test]
1186 fn a_push_too_large_to_read_is_never_let_through_unread() {
1187 let body = vec![0u8; MAX_SCANNED_PUSH + 1];
1188 let error = run(scan_push(&FakeRepo::default(), &body, &[])).unwrap_err();
1189 assert!(unscannable(&error));
1190 let (reason, messages) = crate::git_http::size_refusal(&crate::git_http::SizeViolation::Unscannable {
1191 size: body.len() as u64,
1192 cap: MAX_SCANNED_PUSH,
1193 });
1194 assert_eq!(reason, "the push is too large to check for secrets");
1195 assert!(messages.iter().any(|line| line.contains("100.0 MB")));
1196 assert!(messages.iter().any(|line| line.contains("Push in parts")));
1197 }
1198
1199 #[test]
1200 fn a_push_without_secrets_or_a_pack_finds_nothing() {
1201 let blob = b"fn main() {}\n".to_vec();
1202 let blob_id = object_id(ObjectKind::Blob, &blob);
1203 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "main.rs".into(), id: blob_id }]);
1204 let tree_id = object_id(ObjectKind::Tree, &tree);
1205 let body = push(&[
1206 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
1207 Entry::Whole(ObjectKind::Tree, tree),
1208 Entry::Whole(ObjectKind::Blob, blob),
1209 ]);
1210 assert!(run(scan_push(&FakeRepo::default(), &body, &[])).unwrap().is_empty());
1211 // A deletion sends commands and no pack.
1212 assert!(run(scan_push(&FakeRepo::default(), b"0000", &[])).unwrap().is_empty());
1213 }
1214
1215 #[test]
1216 fn custom_patterns_are_found_in_a_push_and_a_committed_file() {
1217 let patterns = compiled(&[PatternSpec {
1218 id: "pat_1".into(),
1219 name: "Acme key".into(),
1220 pattern: "acme_[0-9a-f]{16}".into(),
1221 before: None,
1222 after: None,
1223 }]);
1224 let blob = b"token: acme_0123456789abcdef\n".to_vec();
1225 let blob_id = object_id(ObjectKind::Blob, &blob);
1226 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "deploy.yml".into(), id: blob_id }]);
1227 let tree_id = object_id(ObjectKind::Tree, &tree);
1228 let body = push(&[
1229 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
1230 Entry::Whole(ObjectKind::Tree, tree),
1231 Entry::Whole(ObjectKind::Blob, blob.clone()),
1232 ]);
1233 let found = run(scan_push(&FakeRepo::default(), &body, &patterns)).unwrap();
1234 assert_eq!(found.len(), 1);
1235 assert_eq!((found[0].kind.as_str(), found[0].pattern_id.as_deref(), found[0].line), ("custom_pattern", Some("pat_1"), 1));
1236 assert_eq!(secret_label(&found[0]).unwrap(), "a match for the custom pattern \"Acme key\"");
1237 assert!(!found[0].preview.contains("0123456789abcdef"));
1238 // Without the pattern, nothing.
1239 assert!(run(scan_push(&FakeRepo::default(), &body, &[])).unwrap().is_empty());
1240 // A file committed through g1t is scanned for both.
1241 let file = format!("{blob}AWS={}\n", key(), blob = String::from_utf8(blob).unwrap());
1242 let found = scan_file(".g1t/workflows/deploy.yml", file.as_bytes(), "c0ffee", &patterns);
1243 let kinds: Vec<&str> = found.iter().map(|secret| secret.kind.as_str()).collect();
1244 assert_eq!(kinds, ["aws_access_key", "custom_pattern"]);
1245 }
1246
1247 #[test]
1248 fn a_push_carrying_the_pushers_private_address_is_declined_with_a_masked_address() {
1249 let tree = encode_tree(&[]);
1250 let tree_id = object_id(ObjectKind::Tree, &tree);
1251 let mine = format!("tree {tree_id}
1252author S <Sam@Gmail.com> 0 +0000
1253committer S <sam@gmail.com> 0 +0000
1254
1255x
1256").into_bytes();
1257 let mine_id = object_id(ObjectKind::Commit, &mine);
1258 let guard = PushEmailGuard { emails: vec!["sam@gmail.com".into()], noreply: "1abc2def+sam@users.noreply.g1t.sh".into() };
1259 let body = push(&[Entry::Whole(ObjectKind::Commit, mine), Entry::Whole(ObjectKind::Tree, tree.clone())]);
1260 let (found, email) = exposed_address(&body, &guard).unwrap();
1261 assert_eq!(found, mine_id);
1262 let message = exposed_message(&found, &email, &guard.noreply);
1263 assert!(message[0].starts_with(&format!("push declined: commit {} would publish s***@gmail.com", &mine_id[..7])));
1264 assert!(message[1].contains("git config user.email 1abc2def+sam@users.noreply.g1t.sh"));
1265 assert!(message[2].contains("g1t.sh/settings/emails"));
1266 // Someone else's commits, and no pack at all, go through.
1267 let theirs = push(&[Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), Entry::Whole(ObjectKind::Tree, tree)]);
1268 assert_eq!(exposed_address(&theirs, &guard), None);
1269 assert_eq!(exposed_address(b"0000", &guard), None);
1270 }
1271}