Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/repos/src/secret_scan.rs

1,008 lines45,100 bytesCodeBlame
1//! Looking for secrets in git: in what a push adds, before it is stored
2//! (push protection), and in a repository's history, a page at a time, for
3//! the security service. Also finds the lockfiles it reads dependencies
4//! from. What counts as a secret is `g1t_scan`'s business.
5//!
6//! Push protection also keeps a person's private address out of what they
7//! push, when they asked g1t to (see [`exposed_address`]).
8//!
9//! Custom patterns (the security suite's) are looked for alongside the
10//! built-in formats, in pushes, history and files committed through g1t
11//! itself; the security service says which apply ([`Repos::patterns_for`]).
12//! It can also run a pattern over the default branch for a dry run
13//! ([`Repos::match_pattern`]), and ask a landed secret's issuer whether it
14//! still works ([`Repos::check_secret`]), without the value ever leaving
15//! this service except to that issuer.
16
17use std::cell::Cell;
18use std::collections::{HashSet, VecDeque};
19
20use futures_util::future::try_join_all;
21use g1t_contracts::User;
22use g1t_contracts::accounts::{CommitIdentityArgs, PushEmailGuard, mask_email};
23use g1t_contracts::repos::{EntryKind, Repo, RepoPath};
24use g1t_contracts::security::{
25 FindLockfilesArgs, HistoryPage, LockfileText, Lockfiles, NewSecret, PushBlockedArgs, PushVerdict,
26 ScanHistoryArgs,
27};
28use g1t_contracts::security_suite::{CheckSecretArgs, MatchPatternArgs, PatternMatch, PatternMatches, PatternSpec, PatternsForArgs, SecretValidity};
29use g1t_scan::custom::{self, Compiled};
30use g1t_scan::lockfiles::Lockfile;
31use g1t_scan::pack::{ObjectKind, Pack, TreeItem, encode_tree, pack_start};
32use g1t_scan::protection::{self, Blocked};
33use worker::{Response, Result};
34
35use crate::registry::store_key;
36use crate::store::{GitRepo, GitStore};
37
38/// Where people allow a secret: the project's Security page.
39const SITE: &str = "https://g1t.sh";
40/// A push adding more commits than this is scanned for this many of them.
41const MAX_PUSH_COMMITS: usize = 300;
42/// Files compared per commit, at most.
43const MAX_FILES_PER_COMMIT: usize = 300;
44/// Bases fetched from the store for a thin pack, at most.
45const MAX_BASES: usize = 500;
46/// The largest push that is read whole and scanned. A larger one is
47/// declined, since it cannot be checked (git_http.rs `LargePushes`).
48pub const MAX_SCANNED_PUSH: usize = 24 * 1024 * 1024;
49/// What marks an error as a push too large to scan.
50const UNSCANNABLE: &str = "push-unscannable:";
51
52/// Whether an error says the push was too large to scan.
53pub fn unscannable(error: &worker::Error) -> bool {
54 error.to_string().contains(UNSCANNABLE)
55}
56const READS_AT_ONCE: usize = 16;
57/// Directories never searched for lockfiles.
58const SKIPPED_DIRECTORIES: [&str; 8] = ["node_modules", "vendor", "target", ".git", "dist", "build", "third_party", ".venv"];
59const MAX_LOCKFILES: usize = 40;
60const MAX_LOCKFILE_DEPTH: usize = 4;
61const MAX_LOCKFILE_BYTES: usize = 16 * 1024 * 1024;
62
63fn mode(kind: EntryKind) -> &'static str {
64 match kind {
65 EntryKind::Tree => "40000",
66 EntryKind::Blob => "100644",
67 EntryKind::Exec => "100755",
68 EntryKind::Symlink => "120000",
69 EntryKind::Gitlink => "160000",
70 }
71}
72
73/// Objects for a walk: the pushed pack's first, then the repository's.
74struct Objects<'a, R: GitRepo> {
75 pack: &'a Pack,
76 repo: &'a R,
77 reads: Cell<u32>,
78}
79
80impl<R: GitRepo> Objects<'_, R> {
81 async fn tree(&self, id: &str) -> Result<Vec<TreeItem>> {
82 if let Some(items) = self.pack.tree(id) {
83 return Ok(items);
84 }
85 self.reads.set(self.reads.get() + 1);
86 Ok(self
87 .repo
88 .read_tree(id)
89 .await?
90 .unwrap_or_default()
91 .into_iter()
92 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
93 .collect())
94 }
95
96 async fn blob(&self, id: &str) -> Result<Option<Vec<u8>>> {
97 if let Some(bytes) = self.pack.blob(id) {
98 return Ok(Some(bytes.to_vec()));
99 }
100 self.reads.set(self.reads.get() + 1);
101 self.repo.read_blob(id).await
102 }
103
104 async fn commit_tree(&self, id: &str) -> Result<Option<String>> {
105 if let Some(commit) = self.pack.commit(id) {
106 return Ok(Some(commit.tree));
107 }
108 self.reads.set(self.reads.get() + 1);
109 Ok(self.repo.log(id, 1).await?.into_iter().next().map(|commit| commit.tree_hash))
110 }
111}
112
113/// A file that differs between two trees: its path, the blob it was and
114/// the blob it is.
115struct Change {
116 path: String,
117 old: Option<String>,
118 new: String,
119}
120
121/// The regular files whose content differs between two trees. Each level
122/// is read at once; identical subtrees are skipped by id.
123async fn changed_files<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: String) -> Result<Vec<Change>> {
124 let mut changes = Vec::new();
125 let mut level = vec![(String::new(), old_root, new_root)];
126 while !level.is_empty() && changes.len() < MAX_FILES_PER_COMMIT {
127 let read = try_join_all(level.iter().map(|(_, old, new)| async move {
128 let old = match old {
129 Some(old) => objects.tree(old).await?,
130 None => Vec::new(),
131 };
132 Ok::<_, worker::Error>((old, objects.tree(new).await?))
133 }))
134 .await?;
135 let mut next = Vec::new();
136 for ((prefix, _, _), (old, new)) in level.iter().zip(read) {
137 for item in &new {
138 let before = old.iter().find(|entry| entry.name == item.name);
139 if before.is_some_and(|before| before.id == item.id) {
140 continue;
141 }
142 let path = format!("{prefix}{}", item.name);
143 if item.is_tree() {
144 next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), item.id.clone()));
145 } else if item.is_file() && changes.len() < MAX_FILES_PER_COMMIT {
146 changes.push(Change {
147 path,
148 old: before.filter(|b| b.is_file()).map(|b| b.id.clone()),
149 new: item.id.clone(),
150 });
151 }
152 }
153 }
154 level = next;
155 }
156 Ok(changes)
157}
158
159/// The scanner's custom patterns, compiled; any that no longer compile are
160/// skipped.
161pub fn compiled(patterns: &[PatternSpec]) -> Vec<Compiled> {
162 let specs: Vec<custom::PatternSpec> = patterns
163 .iter()
164 .map(|spec| custom::PatternSpec {
165 id: spec.id.clone(),
166 name: spec.name.clone(),
167 pattern: spec.pattern.clone(),
168 before: spec.before.clone(),
169 after: spec.after.clone(),
170 })
171 .collect();
172 custom::compile_all(&specs)
173}
174
175/// What a custom pattern found, as the security service records it.
176fn custom_secret(hit: custom::CustomHit, path: &str, commit: &str) -> NewSecret {
177 NewSecret {
178 fingerprint: hit.fingerprint(),
179 kind: custom::KIND.to_owned(),
180 path: path.to_owned(),
181 line: hit.line,
182 commit: commit.to_owned(),
183 preview: hit.preview(),
184 test_value: None,
185 pattern_id: Some(hit.pattern_id),
186 pattern_name: Some(hit.pattern_name),
187 }
188}
189
190/// How a sentence names a secret found: its format, or its pattern.
191pub fn secret_label(secret: &NewSecret) -> Option<String> {
192 if secret.kind == custom::KIND {
193 return Some(custom::label(secret.pattern_name.as_deref().unwrap_or("custom")));
194 }
195 g1t_scan::secrets::SecretKind::parse(&secret.kind).map(|kind| kind.label().to_owned())
196}
197
198/// The secrets a new file holds, built-in and custom, for a commit made
199/// through g1t rather than pushed.
200pub fn scan_file(path: &str, bytes: &[u8], commit: &str, patterns: &[Compiled]) -> Vec<NewSecret> {
201 let mut found: Vec<NewSecret> = protection::scan_change(path, None, bytes)
202 .into_iter()
203 .map(|hit| NewSecret {
204 fingerprint: hit.fingerprint(),
205 kind: hit.kind.id().to_owned(),
206 path: path.to_owned(),
207 line: hit.line,
208 commit: commit.to_owned(),
209 preview: hit.preview(),
210 test_value: hit.test_value().map(str::to_owned),
211 pattern_id: None,
212 pattern_name: None,
213 })
214 .collect();
215 found.extend(protection::scan_change_custom(path, None, bytes, patterns).into_iter().map(|hit| custom_secret(hit, path, commit)));
216 found
217}
218
219/// The secrets each change adds, found `READS_AT_ONCE` files at a time.
220async fn scan_changes<R: GitRepo>(objects: &Objects<'_, R>, commit: &str, changes: Vec<Change>, patterns: &[Compiled]) -> Result<Vec<NewSecret>> {
221 let mut found = Vec::new();
222 let changes: Vec<Change> = changes
223 .into_iter()
224 .filter(|change| !g1t_scan::secrets::skipped_path(&change.path))
225 .collect();
226 for batch in changes.chunks(READS_AT_ONCE) {
227 let read = try_join_all(batch.iter().map(|change| async move {
228 let new = objects.blob(&change.new).await?;
229 let old = match (&change.old, &new) {
230 (Some(old), Some(_)) => objects.blob(old).await?,
231 _ => None,
232 };
233 Ok::<_, worker::Error>((new, old))
234 }))
235 .await?;
236 for (change, (new, old)) in batch.iter().zip(read) {
237 let Some(new) = new else { continue };
238 for hit in protection::scan_change(&change.path, old.as_deref(), &new) {
239 found.push(NewSecret {
240 fingerprint: hit.fingerprint(),
241 kind: hit.kind.id().to_owned(),
242 path: change.path.clone(),
243 line: hit.line,
244 commit: commit.to_owned(),
245 preview: hit.preview(),
246 test_value: hit.test_value().map(str::to_owned),
247 pattern_id: None,
248 pattern_name: None,
249 });
250 }
251 for hit in protection::scan_change_custom(&change.path, old.as_deref(), &new, patterns) {
252 found.push(custom_secret(hit, &change.path, commit));
253 }
254 }
255 }
256 Ok(found)
257}
258
259/// Fetches what a thin pack's deltas are based on from the repository.
260async fn supply_bases<R: GitRepo>(pack: &mut Pack, repo: &R) -> Result<()> {
261 for _ in 0..3 {
262 let missing = pack.missing_bases();
263 if missing.is_empty() {
264 return Ok(());
265 }
266 let found = try_join_all(missing.iter().take(MAX_BASES).map(|id| async move {
267 // A base is nearly always a blob; failing that, a tree.
268 if let Ok(Some(bytes)) = repo.read_blob(id).await {
269 return Ok::<_, worker::Error>(Some((ObjectKind::Blob, bytes)));
270 }
271 Ok(repo.read_tree(id).await.ok().flatten().map(|entries| {
272 let items: Vec<TreeItem> = entries
273 .into_iter()
274 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
275 .collect();
276 (ObjectKind::Tree, encode_tree(&items))
277 }))
278 }))
279 .await?;
280 let mut progress = false;
281 for (id, object) in missing.iter().zip(found) {
282 if let Some((kind, data)) = object {
283 pack.supply(id, kind, data);
284 progress = true;
285 }
286 }
287 if !progress {
288 return Ok(());
289 }
290 }
291 Ok(())
292}
293
294/// The secrets the commits in a push add, each secret once. A push too
295/// large to read is an error ([`unscannable`]): it is declined, never let
296/// through unread. A pack that cannot be read for another reason is let
297/// through, and said so in the logs; the store will judge it.
298pub async fn scan_push<R: GitRepo>(repo: &R, body: &[u8], patterns: &[Compiled]) -> Result<Vec<NewSecret>> {
299 if body.len() > MAX_SCANNED_PUSH {
300 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {} bytes", body.len())));
301 }
302 let Some(start) = pack_start(body) else {
303 return Ok(Vec::new());
304 };
305 let mut pack = match Pack::parse(&body[start..]) {
306 Ok(pack) => pack,
307 Err(problem) if problem.contains("too large") => {
308 return Err(worker::Error::RustError(format!("{UNSCANNABLE} {problem}")));
309 }
310 Err(problem) => {
311 worker::console_error!("push not scanned for secrets: {problem}");
312 return Ok(Vec::new());
313 }
314 };
315 supply_bases(&mut pack, repo).await?;
316 if pack.unresolved() > 0 {
317 worker::console_error!("{} objects of a push could not be resolved for scanning", pack.unresolved());
318 }
319 let objects = Objects { pack: &pack, repo, reads: Cell::new(0) };
320 let commits: Vec<String> = pack.commits().iter().take(MAX_PUSH_COMMITS).cloned().collect();
321 let mut found = Vec::new();
322 let mut seen_blobs = HashSet::new();
323 let mut seen_secrets = HashSet::new();
324 for id in commits {
325 let Some(commit) = pack.commit(&id) else { continue };
326 let old_tree = match commit.parents.first() {
327 Some(parent) => objects.commit_tree(parent).await?,
328 None => None,
329 };
330 // Only content the push brings is new; a blob the repository has
331 // was looked at when it arrived.
332 let changes: Vec<Change> = changed_files(&objects, old_tree, commit.tree)
333 .await?
334 .into_iter()
335 .filter(|change| pack.contains(&change.new) && seen_blobs.insert((change.path.clone(), change.new.clone())))
336 .collect();
337 for secret in scan_changes(&objects, &id, changes, patterns).await? {
338 if seen_secrets.insert(secret.fingerprint.clone()) {
339 found.push(secret);
340 }
341 }
342 }
343 Ok(found)
344}
345
346/// A commit in a push that would publish one of the pusher's own
347/// addresses while they keep it private: its id and the address. Only the
348/// commits the push adds are read; anyone else's address is no concern
349/// here. A pack that cannot be read is let through.
350pub fn exposed_address(body: &[u8], guard: &PushEmailGuard) -> Option<(String, String)> {
351 if body.len() > MAX_SCANNED_PUSH {
352 return None;
353 }
354 let pack = Pack::parse(&body[pack_start(body)?..]).ok()?;
355 pack.commits().iter().find_map(|id| {
356 let commit = pack.commit(id)?;
357 [commit.author_email, commit.committer_email]
358 .into_iter()
359 .flatten()
360 .find(|email| guard.exposes(email))
361 .map(|email| (id.clone(), email))
362 })
363}
364
365/// What git shows a person whose push would publish their private address.
366pub fn exposed_message(commit: &str, email: &str, noreply: &str) -> Vec<String> {
367 let short: String = commit.chars().take(7).collect();
368 vec![
369 format!(
370 "push declined: commit {short} would publish {} while your email is private.",
371 mask_email(&email.to_lowercase())
372 ),
373 format!("Commit with {noreply} (git config user.email {noreply}) and amend,"),
374 format!("or change this in {}/settings/emails.", SITE.trim_start_matches("https://")),
375 ]
376}
377
378impl<S: GitStore> crate::Repos<S> {
379 /// What a push by `pusher` must not publish: their own addresses, when
380 /// they keep them private and block such pushes. An agent's push is
381 /// its person's. `None` when nothing is guarded, or identity cannot say.
382 async fn push_email_guard(&self, pusher: Option<&User>) -> Option<PushEmailGuard> {
383 let pusher = pusher?;
384 let person = pusher.acting.as_ref().map_or(pusher.id.clone(), |acting| acting.on_behalf_of.id.clone());
385 let identity = self.identity.as_ref()?;
386 g1t_kit::call::<_, Option<PushEmailGuard>>(identity, "push_email_guard", &CommitIdentityArgs { user_id: person })
387 .await
388 .unwrap_or_else(|error| {
389 worker::console_error!("push_email_guard failed: {error}");
390 None
391 })
392 }
393
394 /// Push protection: the response refusing a push that adds secrets
395 /// nobody has allowed, or that would publish the pusher's private
396 /// address, or `None` to let it through.
397 /// `repo` is the repository pushed to, as the request read it.
398 pub(crate) async fn protect(&self, repo: &Repo, pusher: Option<&User>, body: &[u8]) -> Result<Option<Response>> {
399 // A pull request's findings belong to the repository it was made from.
400 let owner = match &repo.fork_of {
401 Some(id) => self.registry.by_id(id).await?.unwrap_or(repo.clone()),
402 None => repo.clone(),
403 };
404 // Asking identity about the pusher's address and scanning the push
405 // do not depend on each other, so they happen at once.
406 let scan = async {
407 let patterns = compiled(&self.patterns_for(&owner).await);
408 let git = self.store.open(&store_key(repo)).await?;
409 scan_push(&git, body, &patterns).await
410 };
411 let (guard, found) = futures_util::future::join(self.push_email_guard(pusher), scan).await;
412 if let Some(guard) = guard
413 && let Some((commit, email)) = exposed_address(body, &guard)
414 {
415 return Ok(Some(crate::git_http::declined(
416 body,
417 "push would publish a private email",
418 &exposed_message(&commit, &email, &guard.noreply),
419 )?));
420 }
421 let found = match found {
422 Err(error) if unscannable(&error) => {
423 let (reason, messages) = crate::git_http::size_refusal(&crate::git_http::SizeViolation::Unscannable {
424 size: body.len() as u64,
425 cap: MAX_SCANNED_PUSH,
426 });
427 return Ok(Some(crate::git_http::declined(body, &reason, &messages)?));
428 }
429 found => found?,
430 };
431 if found.is_empty() {
432 return Ok(None);
433 }
434 let blocked = self.blocked(&owner, pusher, found).await;
435 if blocked.is_empty() {
436 return Ok(None);
437 }
438 Ok(Some(crate::git_http::declined(
439 body,
440 &protection::reason(&blocked),
441 &protection::explain(&blocked),
442 )?))
443 }
444
445 /// The custom patterns the security service says `repo` is scanned
446 /// with; none when it cannot say.
447 pub(crate) async fn patterns_for(&self, repo: &Repo) -> Vec<PatternSpec> {
448 let Some(security) = &self.security else { return Vec::new() };
449 g1t_kit::call(
450 security,
451 "patterns_for",
452 &PatternsForArgs { repo_id: repo.id.clone(), namespace: repo.namespace.clone(), private: Some(repo.is_private) },
453 )
454 .await
455 .unwrap_or_else(|error| {
456 worker::console_error!("patterns_for failed: {error}");
457 Vec::new()
458 })
459 }
460
461 /// Of `found` in a change to `owner`, the secrets nobody let through:
462 /// the security service records them all and says which were allowed.
463 pub(crate) async fn blocked(&self, owner: &Repo, pusher: Option<&User>, found: Vec<NewSecret>) -> Vec<Blocked> {
464 let owner_path = RepoPath { namespace: owner.namespace.clone(), name: owner.name.clone() };
465 let verdict = match &self.security {
466 Some(security) => g1t_kit::call::<_, PushVerdict>(
467 security,
468 "push_blocked",
469 &PushBlockedArgs {
470 repo_id: owner.id.clone(),
471 path: owner_path.clone(),
472 pusher: pusher.map(|user| user.username.clone()),
473 secrets: found.clone(),
474 private: Some(owner.is_private),
475 },
476 )
477 .await
478 .unwrap_or_else(|error| {
479 worker::console_error!("push_blocked failed: {error}");
480 PushVerdict::default()
481 }),
482 None => PushVerdict::default(),
483 };
484 found
485 .iter()
486 .filter(|secret| !verdict.allowed.contains(&secret.fingerprint))
487 // A likely test value is recorded, never a reason to refuse.
488 .filter(|secret| secret.test_value.is_none())
489 .filter_map(|secret| {
490 let label = secret_label(secret)?;
491 let id = verdict.ids.iter().find(|(fingerprint, _)| *fingerprint == secret.fingerprint);
492 Some(Blocked {
493 label,
494 path: secret.path.clone(),
495 line: secret.line,
496 commit: secret.commit.clone(),
497 // Where it can be bypassed with a reason, or allowed.
498 allow_url: id.map(|(_, id)| {
499 format!("{SITE}/{}/{}/security/secret-scanning/{id}", owner_path.namespace, owner_path.name)
500 }),
501 })
502 })
503 .collect()
504 }
505
506 /// Push protection for a file committed through g1t (`commit_file`):
507 /// the refusal, naming each secret and where to bypass it, or `None`.
508 pub(crate) async fn protect_file(&self, repo: &Repo, actor: &User, path: &str, content: &[u8], commit: &str) -> Option<String> {
509 let patterns = compiled(&self.patterns_for(repo).await);
510 let found = scan_file(path, content, commit, &patterns);
511 if found.is_empty() {
512 return None;
513 }
514 let blocked = self.blocked(repo, Some(actor), found).await;
515 if blocked.is_empty() {
516 return None;
517 }
518 Some(protection::explain(&blocked).join("\n"))
519 }
520
521 /// A page of the default branch's history, scanned for secrets.
522 pub(crate) async fn scan_history(&self, a: ScanHistoryArgs) -> Result<HistoryPage> {
523 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
524 return Ok(HistoryPage::default());
525 };
526 let git = self.store.open(&store_key(&repo)).await?;
527 let limit = a.limit.clamp(1, 100);
528 // A page of a pushed range starts at its newest commit, and the
529 // history of the default branch at its head.
530 let start = a.after.or(a.from).unwrap_or_else(|| repo.default_branch.clone());
531 let mut commits = git.log(&start, limit + 1).await?;
532 let mut next = (commits.len() > limit as usize).then(|| commits.pop().map(|commit| commit.hash)).flatten();
533 // A range ends where the branch was before the push.
534 if let Some(until) = a.until.as_deref()
535 && let Some(at) = commits.iter().position(|commit| commit.hash == until)
536 {
537 commits.truncate(at);
538 next = None;
539 }
540 if a.until.is_some() && next.as_deref() == a.until.as_deref() {
541 next = None;
542 }
543 let empty = Pack::default();
544 let objects = Objects { pack: &empty, repo: &git, reads: Cell::new(1) };
545 let patterns = compiled(&a.patterns);
546 let mut page = HistoryPage { next, ..HistoryPage::default() };
547 let mut seen = HashSet::new();
548 for (index, commit) in commits.iter().enumerate() {
549 let old_tree = match commit.parents.first() {
550 Some(parent) => match commits.get(index + 1).filter(|older| older.hash == *parent) {
551 Some(older) => Some(older.tree_hash.clone()),
552 None => objects.commit_tree(parent).await?,
553 },
554 None => None,
555 };
556 let changes = changed_files(&objects, old_tree, commit.tree_hash.clone()).await?;
557 for secret in scan_changes(&objects, &commit.hash, changes, &patterns).await? {
558 if seen.insert(secret.fingerprint.clone()) {
559 page.secrets.push(secret);
560 }
561 }
562 page.commits += 1;
563 }
564 page.reads = objects.reads.get();
565 Ok(page)
566 }
567
568 /// The lockfiles on the default branch, outside vendored directories.
569 pub(crate) async fn find_lockfiles(&self, a: FindLockfilesArgs) -> Result<Lockfiles> {
570 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
571 return Ok(Lockfiles::default());
572 };
573 let git = self.store.open(&store_key(&repo)).await?;
574 let at = a.git_ref.as_deref().unwrap_or(&repo.default_branch);
575 let Some(head) = git.log(at, 1).await?.into_iter().next() else {
576 return Ok(Lockfiles::default());
577 };
578 let mut found = Vec::new();
579 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone(), 0usize)]);
580 while let Some((prefix, tree, depth)) = queue.pop_front() {
581 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
582 match entry.kind {
583 EntryKind::Tree if depth < MAX_LOCKFILE_DEPTH && !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => {
584 queue.push_back((format!("{prefix}{}/", entry.name), entry.hash, depth + 1));
585 }
586 EntryKind::Blob if Lockfile::for_path(&entry.name).is_some() && found.len() < MAX_LOCKFILES => {
587 found.push((format!("{prefix}{}", entry.name), entry.hash));
588 }
589 _ => {}
590 }
591 }
592 }
593 let texts = try_join_all(found.iter().map(|(_, hash)| git.read_blob(hash))).await?;
594 let files = found
595 .into_iter()
596 .zip(texts)
597 .filter_map(|((path, _), bytes)| {
598 let bytes = bytes.filter(|bytes| bytes.len() <= MAX_LOCKFILE_BYTES)?;
599 Some(LockfileText { path, text: String::from_utf8(bytes).ok()? })
600 })
601 .collect();
602 Ok(Lockfiles { commit: Some(head.hash), files })
603 }
604
605 /// A dry run of a custom pattern over the default branch's files, up to
606 /// [`MATCH_FILES`] files and [`MATCH_BYTES`] of text, skipping what
607 /// secret scanning skips. Nothing is recorded.
608 pub(crate) async fn match_pattern(&self, a: MatchPatternArgs) -> Result<PatternMatches> {
609 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
610 return Ok(PatternMatches::default());
611 };
612 let patterns = compiled(std::slice::from_ref(&a.pattern));
613 let Some(pattern) = patterns.first() else {
614 return Ok(PatternMatches::default());
615 };
616 let git = self.store.open(&store_key(&repo)).await?;
617 let Some(head) = git.log(&repo.default_branch, 1).await?.into_iter().next() else {
618 return Ok(PatternMatches::default());
619 };
620 let mut result = PatternMatches { commit: Some(head.hash.clone()), ..PatternMatches::default() };
621 let mut files = Vec::new();
622 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone())]);
623 while let Some((prefix, tree)) = queue.pop_front() {
624 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
625 let path = format!("{prefix}{}", entry.name);
626 match entry.kind {
627 EntryKind::Tree if !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => queue.push_back((format!("{path}/"), entry.hash)),
628 EntryKind::Blob | EntryKind::Exec if !g1t_scan::secrets::skipped_path(&path) => {
629 if files.len() == MATCH_FILES {
630 result.truncated = true;
631 } else {
632 files.push((path, entry.hash));
633 }
634 }
635 _ => {}
636 }
637 }
638 }
639 let mut bytes = 0usize;
640 let limit = a.limit.clamp(1, 200) as usize;
641 for batch in files.chunks(READS_AT_ONCE) {
642 if bytes > MATCH_BYTES || result.matches.len() >= limit {
643 result.truncated = true;
644 break;
645 }
646 let read = try_join_all(batch.iter().map(|(_, hash)| git.read_blob(hash))).await?;
647 for ((path, _), blob) in batch.iter().zip(read) {
648 let Some(blob) = blob else { continue };
649 bytes += blob.len();
650 result.files_scanned += 1;
651 let Some(text) = protection::text_of(path, &blob) else { continue };
652 let lines: Vec<&str> = text.lines().collect();
653 for hit in custom::scan_lines(text, std::slice::from_ref(pattern), |_| true) {
654 if result.matches.len() >= limit {
655 result.truncated = true;
656 break;
657 }
658 let line = lines.get(hit.line as usize - 1).copied().unwrap_or_default();
659 result.matches.push(PatternMatch { path: path.clone(), line: hit.line, preview: custom::masked_line(line, &hit.value) });
660 }
661 }
662 }
663 Ok(result)
664 }
665
666 /// Asks a landed secret's issuer whether it still works: finds it again
667 /// by its fingerprint at `commit`:`path` near `line`, and makes the
668 /// issuer's own read-only check over HTTPS. The value goes nowhere else.
669 pub(crate) async fn check_secret(&self, a: CheckSecretArgs) -> Result<SecretValidity> {
670 let unknown = |detail: &str| SecretValidity { validity: "unknown".to_owned(), detail: Some(detail.to_owned()) };
671 let Some(kind) = g1t_scan::secrets::SecretKind::parse(&a.kind) else {
672 return Ok(SecretValidity { validity: "unsupported".to_owned(), detail: None });
673 };
674 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
675 return Ok(unknown("no such repository"));
676 };
677 let git = self.store.open(&store_key(&repo)).await?;
678 let Some(bytes) = git.read_file(&a.commit, &a.path).await? else {
679 return Ok(unknown("the file is not at that commit"));
680 };
681 let Some(text) = protection::text_of(&a.path, &bytes) else {
682 return Ok(unknown("the file cannot be read as text"));
683 };
684 let near = |line: u32| line + 2 >= a.line && line <= a.line + 2;
685 let Some(hit) = g1t_scan::secrets::scan_lines(text, near).into_iter().find(|hit| hit.fingerprint() == a.fingerprint) else {
686 return Ok(unknown("the secret is no longer where it was found"));
687 };
688 let Some(probe) = g1t_scan::validity::check_for(kind, &hit.value) else {
689 return Ok(SecretValidity { validity: "unsupported".to_owned(), detail: None });
690 };
691 let headers = worker::Headers::new();
692 headers.set("user-agent", "g1t secret validity check (+https://docs.g1t.sh/guides/security/secret-protection/)")?;
693 for (name, value) in &probe.headers {
694 headers.set(name, value)?;
695 }
696 let mut init = worker::RequestInit::new();
697 init.with_method(if probe.method == "POST" { worker::Method::Post } else { worker::Method::Get }).with_headers(headers);
698 if let Some(body) = &probe.body {
699 init.with_body(Some(body.clone().into()));
700 }
701 let answer = async {
702 let mut response = worker::Fetch::Request(worker::Request::new_with_init(probe.url, &init)?).send().await?;
703 let status = response.status_code();
704 let body = if probe.reader == g1t_scan::validity::Reader::SlackOk { response.text().await.unwrap_or_default() } else { String::new() };
705 Ok::<_, worker::Error>((status, body))
706 }
707 .await;
708 Ok(match answer {
709 Ok((status, body)) => {
710 let validity = g1t_scan::validity::read(probe.reader, kind, status, &body);
711 SecretValidity {
712 validity: validity.as_str().to_owned(),
713 detail: (validity == g1t_scan::validity::Validity::Unknown).then(|| format!("the issuer answered {status}")),
714 }
715 }
716 Err(error) => unknown(&format!("the issuer could not be reached: {error}")),
717 })
718 }
719}
720
721/// Files a dry run reads, at most, and text in all.
722const MATCH_FILES: usize = 2_000;
723const MATCH_BYTES: usize = 20 * 1024 * 1024;
724
725#[cfg(test)]
726mod tests {
727 use std::collections::HashMap;
728 use std::future::Future;
729 use std::pin::pin;
730 use std::task::{Context, Poll, Waker};
731
732 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature, TreeEntry};
733 use g1t_scan::pack::{ObjectKind, TreeItem, encode_tree, object_id};
734
735 use super::*;
736 use crate::store::Scope;
737
738 /// Runs a future that never waits, as every call to the fake store is.
739 fn run<F: Future>(future: F) -> F::Output {
740 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
741 Poll::Ready(output) => output,
742 Poll::Pending => panic!("the fake store never waits"),
743 }
744 }
745
746 /// A repository held in memory.
747 #[derive(Default)]
748 struct FakeRepo {
749 blobs: HashMap<String, Vec<u8>>,
750 trees: HashMap<String, Vec<TreeEntry>>,
751 commits: HashMap<String, Commit>,
752 }
753
754 impl GitRepo for FakeRepo {
755 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
756 unimplemented!()
757 }
758 async fn branches(&self) -> Result<Vec<Branch>> {
759 Ok(Vec::new())
760 }
761 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
762 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
763 }
764 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
765 Ok(self.commits.get(commit_hash).map(|commit| commit.parents.clone()))
766 }
767 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
768 Ok(self.trees.get(tree_hash).cloned())
769 }
770 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
771 Ok(self.blobs.get(blob_hash).cloned())
772 }
773 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
774 Ok(None)
775 }
776 async fn fork(&self, _target_key: &str) -> Result<()> {
777 Ok(())
778 }
779 }
780
781 /// Zlib with one stored (uncompressed) block, which is all a pack needs.
782 fn zlib(data: &[u8]) -> Vec<u8> {
783 let mut out = vec![0x78, 0x01, 0x01];
784 let length = data.len() as u16;
785 out.extend_from_slice(&length.to_le_bytes());
786 out.extend_from_slice(&(!length).to_le_bytes());
787 out.extend_from_slice(data);
788 let (mut a, mut b) = (1u32, 0u32);
789 for byte in data {
790 a = (a + u32::from(*byte)) % 65521;
791 b = (b + a) % 65521;
792 }
793 out.extend_from_slice(&((b << 16) | a).to_be_bytes());
794 out
795 }
796
797 fn header(code: u8, size: usize) -> Vec<u8> {
798 let mut out = Vec::new();
799 let mut byte = (code << 4) | (size & 15) as u8;
800 let mut rest = size >> 4;
801 while rest > 0 {
802 out.push(byte | 0x80);
803 byte = (rest & 0x7f) as u8;
804 rest >>= 7;
805 }
806 out.push(byte);
807 out
808 }
809
810 fn raw_id(id: &str) -> Vec<u8> {
811 id.as_bytes()
812 .chunks(2)
813 .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
814 .collect()
815 }
816
817 enum Entry {
818 Whole(ObjectKind, Vec<u8>),
819 /// A ref-delta: base id and delta.
820 Delta(String, Vec<u8>),
821 }
822
823 /// A receive-pack request: one command, then the pack.
824 fn push(entries: &[Entry]) -> Vec<u8> {
825 let command = b"0000000000000000000000000000000000000000 4807077b296e6edbf410d55e72749d3e1170c291 refs/heads/main\0report-status side-band-64k\n";
826 let mut body = format!("{:04x}", command.len() + 4).into_bytes();
827 body.extend_from_slice(command);
828 body.extend_from_slice(b"0000PACK");
829 body.extend_from_slice(&2u32.to_be_bytes());
830 body.extend_from_slice(&(entries.len() as u32).to_be_bytes());
831 for entry in entries {
832 match entry {
833 Entry::Whole(kind, data) => {
834 let code = match kind {
835 ObjectKind::Commit => 1,
836 ObjectKind::Tree => 2,
837 ObjectKind::Blob => 3,
838 ObjectKind::Tag => 4,
839 };
840 body.extend(header(code, data.len()));
841 body.extend(zlib(data));
842 }
843 Entry::Delta(base, delta) => {
844 body.extend(header(7, delta.len()));
845 body.extend(raw_id(base));
846 body.extend(zlib(delta));
847 }
848 }
849 }
850 body.extend_from_slice(&[0u8; 20]);
851 body
852 }
853
854 fn key() -> String {
855 format!("AK{}", "IAZ7Q4N2XWLM3KDTRV")
856 }
857
858 fn commit(tree: &str, parent: Option<&str>) -> Vec<u8> {
859 let parent = parent.map(|parent| format!("parent {parent}\n")).unwrap_or_default();
860 format!("tree {tree}\n{parent}author A <a@example.com> 0 +0000\ncommitter A <a@example.com> 0 +0000\n\nchange\n").into_bytes()
861 }
862
863 #[test]
864 fn a_first_push_with_a_secret_is_found_by_file_and_line() {
865 let blob = format!("REGION=eu\nAWS_KEY={}\n", key()).into_bytes();
866 let blob_id = object_id(ObjectKind::Blob, &blob);
867 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "config.env".into(), id: blob_id }]);
868 let tree_id = object_id(ObjectKind::Tree, &tree);
869 let body = push(&[
870 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
871 Entry::Whole(ObjectKind::Tree, tree),
872 Entry::Whole(ObjectKind::Blob, blob),
873 ]);
874 let found = run(scan_push(&FakeRepo::default(), &body, &[])).unwrap();
875 assert_eq!(found.len(), 1);
876 assert_eq!((found[0].path.as_str(), found[0].line, found[0].kind.as_str()), ("config.env", 2, "aws_access_key"));
877 assert!(found[0].preview.starts_with("AKIA") && !found[0].preview.contains(&key()));
878 }
879
880 #[test]
881 fn a_thin_push_reports_only_the_lines_it_adds() {
882 // The repository already has a file with a key in it (decided on
883 // before); the push appends a line holding a second key.
884 let old = format!("first={}\n", key()).into_bytes();
885 let old_id = object_id(ObjectKind::Blob, &old);
886 let second = format!("AK{}", "IAQ9W8E7R6T5Y4U3I2");
887 let new = [old.clone(), format!("second={second}\n").into_bytes()].concat();
888 let base_tree = vec![TreeEntry { name: "app.env".into(), hash: old_id.clone(), kind: EntryKind::Blob }];
889 let base_tree_id = object_id(ObjectKind::Tree, &encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: old_id.clone() }]));
890 let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned();
891 let mut repo = FakeRepo::default();
892 repo.blobs.insert(old_id.clone(), old.clone());
893 repo.trees.insert(base_tree_id.clone(), base_tree);
894 repo.commits.insert(
895 parent_id.clone(),
896 Commit {
897 hash: parent_id.clone(),
898 tree_hash: base_tree_id,
899 message: String::new(),
900 author: Signature { name: "A".into(), email: "a@example.com".into() },
901 parents: Vec::new(),
902 authored_at: String::new(),
903 },
904 );
905 // A delta: copy the old file whole, then insert the new line.
906 let added = format!("second={second}\n").into_bytes();
907 let mut delta = vec![old.len() as u8, new.len() as u8, 0x80 | 0x10, old.len() as u8, added.len() as u8];
908 delta.extend_from_slice(&added);
909 let new_id = object_id(ObjectKind::Blob, &new);
910 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: new_id }]);
911 let tree_id = object_id(ObjectKind::Tree, &tree);
912 let body = push(&[
913 Entry::Whole(ObjectKind::Commit, commit(&tree_id, Some(&parent_id))),
914 Entry::Whole(ObjectKind::Tree, tree),
915 Entry::Delta(old_id, delta),
916 ]);
917 let found = run(scan_push(&repo, &body, &[])).unwrap();
918 assert_eq!(found.len(), 1, "{found:?}");
919 assert_eq!((found[0].path.as_str(), found[0].line), ("app.env", 2));
920 }
921
922 #[test]
923 fn a_push_too_large_to_read_is_never_let_through_unread() {
924 let body = vec![0u8; MAX_SCANNED_PUSH + 1];
925 let error = run(scan_push(&FakeRepo::default(), &body, &[])).unwrap_err();
926 assert!(unscannable(&error));
927 let (reason, messages) = crate::git_http::size_refusal(&crate::git_http::SizeViolation::Unscannable {
928 size: body.len() as u64,
929 cap: MAX_SCANNED_PUSH,
930 });
931 assert_eq!(reason, "the push is too large to check for secrets");
932 assert!(messages.iter().any(|line| line.contains("100.0 MB")));
933 assert!(messages.iter().any(|line| line.contains("Push in parts")));
934 }
935
936 #[test]
937 fn a_push_without_secrets_or_a_pack_finds_nothing() {
938 let blob = b"fn main() {}\n".to_vec();
939 let blob_id = object_id(ObjectKind::Blob, &blob);
940 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "main.rs".into(), id: blob_id }]);
941 let tree_id = object_id(ObjectKind::Tree, &tree);
942 let body = push(&[
943 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
944 Entry::Whole(ObjectKind::Tree, tree),
945 Entry::Whole(ObjectKind::Blob, blob),
946 ]);
947 assert!(run(scan_push(&FakeRepo::default(), &body, &[])).unwrap().is_empty());
948 // A deletion sends commands and no pack.
949 assert!(run(scan_push(&FakeRepo::default(), b"0000", &[])).unwrap().is_empty());
950 }
951
952 #[test]
953 fn custom_patterns_are_found_in_a_push_and_a_committed_file() {
954 let patterns = compiled(&[PatternSpec {
955 id: "pat_1".into(),
956 name: "Acme key".into(),
957 pattern: "acme_[0-9a-f]{16}".into(),
958 before: None,
959 after: None,
960 }]);
961 let blob = b"token: acme_0123456789abcdef\n".to_vec();
962 let blob_id = object_id(ObjectKind::Blob, &blob);
963 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "deploy.yml".into(), id: blob_id }]);
964 let tree_id = object_id(ObjectKind::Tree, &tree);
965 let body = push(&[
966 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
967 Entry::Whole(ObjectKind::Tree, tree),
968 Entry::Whole(ObjectKind::Blob, blob.clone()),
969 ]);
970 let found = run(scan_push(&FakeRepo::default(), &body, &patterns)).unwrap();
971 assert_eq!(found.len(), 1);
972 assert_eq!((found[0].kind.as_str(), found[0].pattern_id.as_deref(), found[0].line), ("custom_pattern", Some("pat_1"), 1));
973 assert_eq!(secret_label(&found[0]).unwrap(), "a match for the custom pattern \"Acme key\"");
974 assert!(!found[0].preview.contains("0123456789abcdef"));
975 // Without the pattern, nothing.
976 assert!(run(scan_push(&FakeRepo::default(), &body, &[])).unwrap().is_empty());
977 // A file committed through g1t is scanned for both.
978 let file = format!("{blob}AWS={}\n", key(), blob = String::from_utf8(blob).unwrap());
979 let found = scan_file(".g1t/workflows/deploy.yml", file.as_bytes(), "c0ffee", &patterns);
980 let kinds: Vec<&str> = found.iter().map(|secret| secret.kind.as_str()).collect();
981 assert_eq!(kinds, ["aws_access_key", "custom_pattern"]);
982 }
983
984 #[test]
985 fn a_push_carrying_the_pushers_private_address_is_declined_with_a_masked_address() {
986 let tree = encode_tree(&[]);
987 let tree_id = object_id(ObjectKind::Tree, &tree);
988 let mine = format!("tree {tree_id}
989author S <Sam@Gmail.com> 0 +0000
990committer S <sam@gmail.com> 0 +0000
991
992x
993").into_bytes();
994 let mine_id = object_id(ObjectKind::Commit, &mine);
995 let guard = PushEmailGuard { emails: vec!["sam@gmail.com".into()], noreply: "1abc2def+sam@users.noreply.g1t.sh".into() };
996 let body = push(&[Entry::Whole(ObjectKind::Commit, mine), Entry::Whole(ObjectKind::Tree, tree.clone())]);
997 let (found, email) = exposed_address(&body, &guard).unwrap();
998 assert_eq!(found, mine_id);
999 let message = exposed_message(&found, &email, &guard.noreply);
1000 assert!(message[0].starts_with(&format!("push declined: commit {} would publish s***@gmail.com", &mine_id[..7])));
1001 assert!(message[1].contains("git config user.email 1abc2def+sam@users.noreply.g1t.sh"));
1002 assert!(message[2].contains("g1t.sh/settings/emails"));
1003 // Someone else's commits, and no pack at all, go through.
1004 let theirs = push(&[Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), Entry::Whole(ObjectKind::Tree, tree)]);
1005 assert_eq!(exposed_address(&theirs, &guard), None);
1006 assert_eq!(exposed_address(b"0000", &guard), None);
1007 }
1008}