Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/repos/src/secret_scan.rs

688 lines29,324 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API1//! Looking for secrets in git: in what a push adds, before it is stored
2//! (push protection), and in a repository's history, a page at a time, for
3//! the security service. Also finds the lockfiles it reads dependencies
4//! from. What counts as a secret is `g1t_scan`'s business.
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look5//!
6//! Push protection also keeps a person's private address out of what they
7//! push, when they asked g1t to (see [`exposed_address`]).
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API8
9use std::cell::Cell;
10use std::collections::{HashSet, VecDeque};
11
12use futures_util::future::try_join_all;
13use g1t_contracts::User;
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look14use g1t_contracts::accounts::{CommitIdentityArgs, PushEmailGuard, mask_email};
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API15use g1t_contracts::repos::{EntryKind, RepoPath};
16use g1t_contracts::security::{
17 FindLockfilesArgs, HistoryPage, LockfileText, Lockfiles, NewSecret, PushBlockedArgs, PushVerdict,
18 ScanHistoryArgs,
19};
20use g1t_scan::lockfiles::Lockfile;
21use g1t_scan::pack::{ObjectKind, Pack, TreeItem, encode_tree, pack_start};
22use g1t_scan::protection::{self, Blocked};
23use worker::{Response, Result};
24
25use crate::registry::store_key;
26use crate::store::{GitRepo, GitStore};
27
28/// Where people allow a secret: the project's Security page.
29const SITE: &str = "https://g1t.sh";
30/// A push adding more commits than this is scanned for this many of them.
31const MAX_PUSH_COMMITS: usize = 300;
32/// Files compared per commit, at most.
33const MAX_FILES_PER_COMMIT: usize = 300;
34/// Bases fetched from the store for a thin pack, at most.
35const MAX_BASES: usize = 500;
36/// Pushes larger than this are let through unscanned.
37const MAX_SCANNED_PUSH: usize = 24 * 1024 * 1024;
38const READS_AT_ONCE: usize = 16;
39/// Directories never searched for lockfiles.
40const SKIPPED_DIRECTORIES: [&str; 8] = ["node_modules", "vendor", "target", ".git", "dist", "build", "third_party", ".venv"];
41const MAX_LOCKFILES: usize = 40;
42const MAX_LOCKFILE_DEPTH: usize = 4;
43const MAX_LOCKFILE_BYTES: usize = 16 * 1024 * 1024;
44
45fn mode(kind: EntryKind) -> &'static str {
46 match kind {
47 EntryKind::Tree => "40000",
48 EntryKind::Blob => "100644",
49 EntryKind::Exec => "100755",
50 EntryKind::Symlink => "120000",
51 EntryKind::Gitlink => "160000",
52 }
53}
54
55/// Objects for a walk: the pushed pack's first, then the repository's.
56struct Objects<'a, R: GitRepo> {
57 pack: &'a Pack,
58 repo: &'a R,
59 reads: Cell<u32>,
60}
61
62impl<R: GitRepo> Objects<'_, R> {
63 async fn tree(&self, id: &str) -> Result<Vec<TreeItem>> {
64 if let Some(items) = self.pack.tree(id) {
65 return Ok(items);
66 }
67 self.reads.set(self.reads.get() + 1);
68 Ok(self
69 .repo
70 .read_tree(id)
71 .await?
72 .unwrap_or_default()
73 .into_iter()
74 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
75 .collect())
76 }
77
78 async fn blob(&self, id: &str) -> Result<Option<Vec<u8>>> {
79 if let Some(bytes) = self.pack.blob(id) {
80 return Ok(Some(bytes.to_vec()));
81 }
82 self.reads.set(self.reads.get() + 1);
83 self.repo.read_blob(id).await
84 }
85
86 async fn commit_tree(&self, id: &str) -> Result<Option<String>> {
87 if let Some(commit) = self.pack.commit(id) {
88 return Ok(Some(commit.tree));
89 }
90 self.reads.set(self.reads.get() + 1);
91 Ok(self.repo.log(id, 1).await?.into_iter().next().map(|commit| commit.tree_hash))
92 }
93}
94
95/// A file that differs between two trees: its path, the blob it was and
96/// the blob it is.
97struct Change {
98 path: String,
99 old: Option<String>,
100 new: String,
101}
102
103/// The regular files whose content differs between two trees. Each level
104/// is read at once; identical subtrees are skipped by id.
105async fn changed_files<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: String) -> Result<Vec<Change>> {
106 let mut changes = Vec::new();
107 let mut level = vec![(String::new(), old_root, new_root)];
108 while !level.is_empty() && changes.len() < MAX_FILES_PER_COMMIT {
109 let read = try_join_all(level.iter().map(|(_, old, new)| async move {
110 let old = match old {
111 Some(old) => objects.tree(old).await?,
112 None => Vec::new(),
113 };
114 Ok::<_, worker::Error>((old, objects.tree(new).await?))
115 }))
116 .await?;
117 let mut next = Vec::new();
118 for ((prefix, _, _), (old, new)) in level.iter().zip(read) {
119 for item in &new {
120 let before = old.iter().find(|entry| entry.name == item.name);
121 if before.is_some_and(|before| before.id == item.id) {
122 continue;
123 }
124 let path = format!("{prefix}{}", item.name);
125 if item.is_tree() {
126 next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), item.id.clone()));
127 } else if item.is_file() && changes.len() < MAX_FILES_PER_COMMIT {
128 changes.push(Change {
129 path,
130 old: before.filter(|b| b.is_file()).map(|b| b.id.clone()),
131 new: item.id.clone(),
132 });
133 }
134 }
135 }
136 level = next;
137 }
138 Ok(changes)
139}
140
141/// The secrets each change adds, found `READS_AT_ONCE` files at a time.
142async fn scan_changes<R: GitRepo>(objects: &Objects<'_, R>, commit: &str, changes: Vec<Change>) -> Result<Vec<NewSecret>> {
143 let mut found = Vec::new();
144 let changes: Vec<Change> = changes
145 .into_iter()
146 .filter(|change| !g1t_scan::secrets::skipped_path(&change.path))
147 .collect();
148 for batch in changes.chunks(READS_AT_ONCE) {
149 let read = try_join_all(batch.iter().map(|change| async move {
150 let new = objects.blob(&change.new).await?;
151 let old = match (&change.old, &new) {
152 (Some(old), Some(_)) => objects.blob(old).await?,
153 _ => None,
154 };
155 Ok::<_, worker::Error>((new, old))
156 }))
157 .await?;
158 for (change, (new, old)) in batch.iter().zip(read) {
159 let Some(new) = new else { continue };
160 for hit in protection::scan_change(&change.path, old.as_deref(), &new) {
161 found.push(NewSecret {
162 fingerprint: hit.fingerprint(),
163 kind: hit.kind.id().to_owned(),
164 path: change.path.clone(),
165 line: hit.line,
166 commit: commit.to_owned(),
167 preview: hit.preview(),
168 });
169 }
170 }
171 }
172 Ok(found)
173}
174
175/// Fetches what a thin pack's deltas are based on from the repository.
176async fn supply_bases<R: GitRepo>(pack: &mut Pack, repo: &R) -> Result<()> {
177 for _ in 0..3 {
178 let missing = pack.missing_bases();
179 if missing.is_empty() {
180 return Ok(());
181 }
182 let found = try_join_all(missing.iter().take(MAX_BASES).map(|id| async move {
183 // A base is nearly always a blob; failing that, a tree.
184 if let Ok(Some(bytes)) = repo.read_blob(id).await {
185 return Ok::<_, worker::Error>(Some((ObjectKind::Blob, bytes)));
186 }
187 Ok(repo.read_tree(id).await.ok().flatten().map(|entries| {
188 let items: Vec<TreeItem> = entries
189 .into_iter()
190 .map(|entry| TreeItem { mode: mode(entry.kind).to_owned(), name: entry.name, id: entry.hash })
191 .collect();
192 (ObjectKind::Tree, encode_tree(&items))
193 }))
194 }))
195 .await?;
196 let mut progress = false;
197 for (id, object) in missing.iter().zip(found) {
198 if let Some((kind, data)) = object {
199 pack.supply(id, kind, data);
200 progress = true;
201 }
202 }
203 if !progress {
204 return Ok(());
205 }
206 }
207 Ok(())
208}
209
210/// The secrets the commits in a push add, each secret once. Fails open: a
211/// pack that cannot be read is let through, and said so in the logs.
212pub async fn scan_push<R: GitRepo>(repo: &R, body: &[u8]) -> Result<Vec<NewSecret>> {
213 // The request is already in memory; reading a pack this large as well
214 // could run the worker out of it, which would fail the push outright.
215 if body.len() > MAX_SCANNED_PUSH {
216 worker::console_error!("a push of {} bytes was not scanned for secrets", body.len());
217 return Ok(Vec::new());
218 }
219 let Some(start) = pack_start(body) else {
220 return Ok(Vec::new());
221 };
222 let mut pack = match Pack::parse(&body[start..]) {
223 Ok(pack) => pack,
224 Err(problem) => {
225 worker::console_error!("push not scanned for secrets: {problem}");
226 return Ok(Vec::new());
227 }
228 };
229 supply_bases(&mut pack, repo).await?;
230 if pack.unresolved() > 0 {
231 worker::console_error!("{} objects of a push could not be resolved for scanning", pack.unresolved());
232 }
233 let objects = Objects { pack: &pack, repo, reads: Cell::new(0) };
234 let commits: Vec<String> = pack.commits().iter().take(MAX_PUSH_COMMITS).cloned().collect();
235 let mut found = Vec::new();
236 let mut seen_blobs = HashSet::new();
237 let mut seen_secrets = HashSet::new();
238 for id in commits {
239 let Some(commit) = pack.commit(&id) else { continue };
240 let old_tree = match commit.parents.first() {
241 Some(parent) => objects.commit_tree(parent).await?,
242 None => None,
243 };
244 // Only content the push brings is new; a blob the repository has
245 // was looked at when it arrived.
246 let changes: Vec<Change> = changed_files(&objects, old_tree, commit.tree)
247 .await?
248 .into_iter()
249 .filter(|change| pack.contains(&change.new) && seen_blobs.insert((change.path.clone(), change.new.clone())))
250 .collect();
251 for secret in scan_changes(&objects, &id, changes).await? {
252 if seen_secrets.insert(secret.fingerprint.clone()) {
253 found.push(secret);
254 }
255 }
256 }
257 Ok(found)
258}
259
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look260/// A commit in a push that would publish one of the pusher's own
261/// addresses while they keep it private: its id and the address. Only the
262/// commits the push adds are read; anyone else's address is no concern
263/// here. A pack that cannot be read is let through.
264pub fn exposed_address(body: &[u8], guard: &PushEmailGuard) -> Option<(String, String)> {
265 if body.len() > MAX_SCANNED_PUSH {
266 return None;
267 }
268 let pack = Pack::parse(&body[pack_start(body)?..]).ok()?;
269 pack.commits().iter().find_map(|id| {
270 let commit = pack.commit(id)?;
271 [commit.author_email, commit.committer_email]
272 .into_iter()
273 .flatten()
274 .find(|email| guard.exposes(email))
275 .map(|email| (id.clone(), email))
276 })
277}
278
279/// What git shows a person whose push would publish their private address.
280pub fn exposed_message(commit: &str, email: &str, noreply: &str) -> Vec<String> {
281 let short: String = commit.chars().take(7).collect();
282 vec![
283 format!(
284 "push declined: commit {short} would publish {} while your email is private.",
285 mask_email(&email.to_lowercase())
286 ),
287 format!("Commit with {noreply} (git config user.email {noreply}) and amend,"),
288 format!("or change this in {}/settings#emails.", SITE.trim_start_matches("https://")),
289 ]
290}
291
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API292impl<S: GitStore> crate::Repos<S> {
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look293 /// What a push by `pusher` must not publish: their own addresses, when
294 /// they keep them private and block such pushes. An agent's push is
295 /// its person's. `None` when nothing is guarded, or identity cannot say.
296 async fn push_email_guard(&self, pusher: Option<&User>) -> Option<PushEmailGuard> {
297 let pusher = pusher?;
298 let person = pusher.acting.as_ref().map_or(pusher.id.clone(), |acting| acting.on_behalf_of.id.clone());
299 let identity = self.identity.as_ref()?;
300 g1t_kit::call::<_, Option<PushEmailGuard>>(identity, "push_email_guard", &CommitIdentityArgs { user_id: person })
301 .await
302 .unwrap_or_else(|error| {
303 worker::console_error!("push_email_guard failed: {error}");
304 None
305 })
306 }
307
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API308 /// Push protection: the response refusing a push that adds secrets
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look309 /// nobody has allowed, or that would publish the pusher's private
310 /// address, or `None` to let it through.
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API311 pub(crate) async fn protect(&self, path: &RepoPath, pusher: Option<&User>, body: &[u8]) -> Result<Option<Response>> {
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look312 if let Some(guard) = self.push_email_guard(pusher).await
313 && let Some((commit, email)) = exposed_address(body, &guard)
314 {
315 return Ok(Some(crate::git_http::declined(
316 body,
317 "push would publish a private email",
318 &exposed_message(&commit, &email, &guard.noreply),
319 )?));
320 }
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API321 let Some(repo) = self.registry.by_path(path).await? else {
322 return Ok(None);
323 };
324 let git = self.store.open(&store_key(&repo)).await?;
325 let found = scan_push(&git, body).await?;
326 if found.is_empty() {
327 return Ok(None);
328 }
329 // A pull request's findings belong to the repository it was made from.
330 let owner = match &repo.fork_of {
331 Some(id) => self.registry.by_id(id).await?.unwrap_or(repo.clone()),
332 None => repo.clone(),
333 };
334 let owner_path = RepoPath { namespace: owner.namespace.clone(), name: owner.name.clone() };
335 let verdict = match &self.security {
336 Some(security) => g1t_kit::call::<_, PushVerdict>(
337 security,
338 "push_blocked",
339 &PushBlockedArgs {
340 repo_id: owner.id.clone(),
341 path: owner_path.clone(),
342 pusher: pusher.map(|user| user.username.clone()),
343 secrets: found.clone(),
344 },
345 )
346 .await
347 .unwrap_or_else(|error| {
348 worker::console_error!("push_blocked failed: {error}");
349 PushVerdict::default()
350 }),
351 None => PushVerdict::default(),
352 };
353 let blocked: Vec<Blocked> = found
354 .iter()
355 .filter(|secret| !verdict.allowed.contains(&secret.fingerprint))
356 .filter_map(|secret| {
357 let kind = g1t_scan::secrets::SecretKind::parse(&secret.kind)?;
358 let id = verdict.ids.iter().find(|(fingerprint, _)| *fingerprint == secret.fingerprint);
359 Some(Blocked {
360 kind,
361 path: secret.path.clone(),
362 line: secret.line,
363 commit: secret.commit.clone(),
364 allow_url: id.map(|(_, id)| {
365 format!("{SITE}/{}/{}/security?tab=secrets&finding={id}", owner_path.namespace, owner_path.name)
366 }),
367 })
368 })
369 .collect();
370 if blocked.is_empty() {
371 return Ok(None);
372 }
373 Ok(Some(crate::git_http::declined(
374 body,
375 &protection::reason(&blocked),
376 &protection::explain(&blocked),
377 )?))
378 }
379
380 /// A page of the default branch's history, scanned for secrets.
381 pub(crate) async fn scan_history(&self, a: ScanHistoryArgs) -> Result<HistoryPage> {
382 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
383 return Ok(HistoryPage::default());
384 };
385 let git = self.store.open(&store_key(&repo)).await?;
386 let limit = a.limit.clamp(1, 100);
387 let start = a.after.unwrap_or_else(|| repo.default_branch.clone());
388 let mut commits = git.log(&start, limit + 1).await?;
389 let next = (commits.len() > limit as usize).then(|| commits.pop().map(|commit| commit.hash)).flatten();
390 let empty = Pack::default();
391 let objects = Objects { pack: &empty, repo: &git, reads: Cell::new(1) };
392 let mut page = HistoryPage { next, ..HistoryPage::default() };
393 let mut seen = HashSet::new();
394 for (index, commit) in commits.iter().enumerate() {
395 let old_tree = match commit.parents.first() {
396 Some(parent) => match commits.get(index + 1).filter(|older| older.hash == *parent) {
397 Some(older) => Some(older.tree_hash.clone()),
398 None => objects.commit_tree(parent).await?,
399 },
400 None => None,
401 };
402 let changes = changed_files(&objects, old_tree, commit.tree_hash.clone()).await?;
403 for secret in scan_changes(&objects, &commit.hash, changes).await? {
404 if seen.insert(secret.fingerprint.clone()) {
405 page.secrets.push(secret);
406 }
407 }
408 page.commits += 1;
409 }
410 page.reads = objects.reads.get();
411 Ok(page)
412 }
413
414 /// The lockfiles on the default branch, outside vendored directories.
415 pub(crate) async fn find_lockfiles(&self, a: FindLockfilesArgs) -> Result<Lockfiles> {
416 let Some(repo) = self.registry.by_id(&a.repo_id).await? else {
417 return Ok(Lockfiles::default());
418 };
419 let git = self.store.open(&store_key(&repo)).await?;
420 let Some(head) = git.log(&repo.default_branch, 1).await?.into_iter().next() else {
421 return Ok(Lockfiles::default());
422 };
423 let mut found = Vec::new();
424 let mut queue = VecDeque::from([(String::new(), head.tree_hash.clone(), 0usize)]);
425 while let Some((prefix, tree, depth)) = queue.pop_front() {
426 for entry in git.read_tree(&tree).await?.unwrap_or_default() {
427 match entry.kind {
428 EntryKind::Tree if depth < MAX_LOCKFILE_DEPTH && !SKIPPED_DIRECTORIES.contains(&entry.name.as_str()) => {
429 queue.push_back((format!("{prefix}{}/", entry.name), entry.hash, depth + 1));
430 }
431 EntryKind::Blob if Lockfile::for_path(&entry.name).is_some() && found.len() < MAX_LOCKFILES => {
432 found.push((format!("{prefix}{}", entry.name), entry.hash));
433 }
434 _ => {}
435 }
436 }
437 }
438 let texts = try_join_all(found.iter().map(|(_, hash)| git.read_blob(hash))).await?;
439 let files = found
440 .into_iter()
441 .zip(texts)
442 .filter_map(|((path, _), bytes)| {
443 let bytes = bytes.filter(|bytes| bytes.len() <= MAX_LOCKFILE_BYTES)?;
444 Some(LockfileText { path, text: String::from_utf8(bytes).ok()? })
445 })
446 .collect();
447 Ok(Lockfiles { commit: Some(head.hash), files })
448 }
449}
450
451#[cfg(test)]
452mod tests {
453 use std::collections::HashMap;
454 use std::future::Future;
455 use std::pin::pin;
456 use std::task::{Context, Poll, Waker};
457
458 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature, TreeEntry};
459 use g1t_scan::pack::{ObjectKind, TreeItem, encode_tree, object_id};
460
461 use super::*;
462 use crate::store::Scope;
463
464 /// Runs a future that never waits, as every call to the fake store is.
465 fn run<F: Future>(future: F) -> F::Output {
466 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
467 Poll::Ready(output) => output,
468 Poll::Pending => panic!("the fake store never waits"),
469 }
470 }
471
472 /// A repository held in memory.
473 #[derive(Default)]
474 struct FakeRepo {
475 blobs: HashMap<String, Vec<u8>>,
476 trees: HashMap<String, Vec<TreeEntry>>,
477 commits: HashMap<String, Commit>,
478 }
479
480 impl GitRepo for FakeRepo {
481 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
482 unimplemented!()
483 }
484 async fn branches(&self) -> Result<Vec<Branch>> {
485 Ok(Vec::new())
486 }
487 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
488 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
489 }
490 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
491 Ok(self.commits.get(commit_hash).map(|commit| commit.parents.clone()))
492 }
493 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
494 Ok(self.trees.get(tree_hash).cloned())
495 }
496 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
497 Ok(self.blobs.get(blob_hash).cloned())
498 }
499 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
500 Ok(None)
501 }
502 async fn fork(&self, _target_key: &str) -> Result<()> {
503 Ok(())
504 }
505 }
506
507 /// Zlib with one stored (uncompressed) block, which is all a pack needs.
508 fn zlib(data: &[u8]) -> Vec<u8> {
509 let mut out = vec![0x78, 0x01, 0x01];
510 let length = data.len() as u16;
511 out.extend_from_slice(&length.to_le_bytes());
512 out.extend_from_slice(&(!length).to_le_bytes());
513 out.extend_from_slice(data);
514 let (mut a, mut b) = (1u32, 0u32);
515 for byte in data {
516 a = (a + u32::from(*byte)) % 65521;
517 b = (b + a) % 65521;
518 }
519 out.extend_from_slice(&((b << 16) | a).to_be_bytes());
520 out
521 }
522
523 fn header(code: u8, size: usize) -> Vec<u8> {
524 let mut out = Vec::new();
525 let mut byte = (code << 4) | (size & 15) as u8;
526 let mut rest = size >> 4;
527 while rest > 0 {
528 out.push(byte | 0x80);
529 byte = (rest & 0x7f) as u8;
530 rest >>= 7;
531 }
532 out.push(byte);
533 out
534 }
535
536 fn raw_id(id: &str) -> Vec<u8> {
537 id.as_bytes()
538 .chunks(2)
539 .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap())
540 .collect()
541 }
542
543 enum Entry {
544 Whole(ObjectKind, Vec<u8>),
545 /// A ref-delta: base id and delta.
546 Delta(String, Vec<u8>),
547 }
548
549 /// A receive-pack request: one command, then the pack.
550 fn push(entries: &[Entry]) -> Vec<u8> {
551 let command = b"0000000000000000000000000000000000000000 4807077b296e6edbf410d55e72749d3e1170c291 refs/heads/main\0report-status side-band-64k\n";
552 let mut body = format!("{:04x}", command.len() + 4).into_bytes();
553 body.extend_from_slice(command);
554 body.extend_from_slice(b"0000PACK");
555 body.extend_from_slice(&2u32.to_be_bytes());
556 body.extend_from_slice(&(entries.len() as u32).to_be_bytes());
557 for entry in entries {
558 match entry {
559 Entry::Whole(kind, data) => {
560 let code = match kind {
561 ObjectKind::Commit => 1,
562 ObjectKind::Tree => 2,
563 ObjectKind::Blob => 3,
564 ObjectKind::Tag => 4,
565 };
566 body.extend(header(code, data.len()));
567 body.extend(zlib(data));
568 }
569 Entry::Delta(base, delta) => {
570 body.extend(header(7, delta.len()));
571 body.extend(raw_id(base));
572 body.extend(zlib(delta));
573 }
574 }
575 }
576 body.extend_from_slice(&[0u8; 20]);
577 body
578 }
579
580 fn key() -> String {
581 format!("AK{}", "IAZ7Q4N2XWLM3KDTRV")
582 }
583
584 fn commit(tree: &str, parent: Option<&str>) -> Vec<u8> {
585 let parent = parent.map(|parent| format!("parent {parent}\n")).unwrap_or_default();
586 format!("tree {tree}\n{parent}author A <a@example.com> 0 +0000\ncommitter A <a@example.com> 0 +0000\n\nchange\n").into_bytes()
587 }
588
589 #[test]
590 fn a_first_push_with_a_secret_is_found_by_file_and_line() {
591 let blob = format!("REGION=eu\nAWS_KEY={}\n", key()).into_bytes();
592 let blob_id = object_id(ObjectKind::Blob, &blob);
593 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "config.env".into(), id: blob_id }]);
594 let tree_id = object_id(ObjectKind::Tree, &tree);
595 let body = push(&[
596 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
597 Entry::Whole(ObjectKind::Tree, tree),
598 Entry::Whole(ObjectKind::Blob, blob),
599 ]);
600 let found = run(scan_push(&FakeRepo::default(), &body)).unwrap();
601 assert_eq!(found.len(), 1);
602 assert_eq!((found[0].path.as_str(), found[0].line, found[0].kind.as_str()), ("config.env", 2, "aws_access_key"));
603 assert!(found[0].preview.starts_with("AKIA") && !found[0].preview.contains(&key()));
604 }
605
606 #[test]
607 fn a_thin_push_reports_only_the_lines_it_adds() {
608 // The repository already has a file with a key in it (decided on
609 // before); the push appends a line holding a second key.
610 let old = format!("first={}\n", key()).into_bytes();
611 let old_id = object_id(ObjectKind::Blob, &old);
612 let second = format!("AK{}", "IAQ9W8E7R6T5Y4U3I2");
613 let new = [old.clone(), format!("second={second}\n").into_bytes()].concat();
614 let base_tree = vec![TreeEntry { name: "app.env".into(), hash: old_id.clone(), kind: EntryKind::Blob }];
615 let base_tree_id = object_id(ObjectKind::Tree, &encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: old_id.clone() }]));
616 let parent_id = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned();
617 let mut repo = FakeRepo::default();
618 repo.blobs.insert(old_id.clone(), old.clone());
619 repo.trees.insert(base_tree_id.clone(), base_tree);
620 repo.commits.insert(
621 parent_id.clone(),
622 Commit {
623 hash: parent_id.clone(),
624 tree_hash: base_tree_id,
625 message: String::new(),
626 author: Signature { name: "A".into(), email: "a@example.com".into() },
627 parents: Vec::new(),
628 authored_at: String::new(),
629 },
630 );
631 // A delta: copy the old file whole, then insert the new line.
632 let added = format!("second={second}\n").into_bytes();
633 let mut delta = vec![old.len() as u8, new.len() as u8, 0x80 | 0x10, old.len() as u8, added.len() as u8];
634 delta.extend_from_slice(&added);
635 let new_id = object_id(ObjectKind::Blob, &new);
636 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "app.env".into(), id: new_id }]);
637 let tree_id = object_id(ObjectKind::Tree, &tree);
638 let body = push(&[
639 Entry::Whole(ObjectKind::Commit, commit(&tree_id, Some(&parent_id))),
640 Entry::Whole(ObjectKind::Tree, tree),
641 Entry::Delta(old_id, delta),
642 ]);
643 let found = run(scan_push(&repo, &body)).unwrap();
644 assert_eq!(found.len(), 1, "{found:?}");
645 assert_eq!((found[0].path.as_str(), found[0].line), ("app.env", 2));
646 }
647
648 #[test]
649 fn a_push_without_secrets_or_a_pack_finds_nothing() {
650 let blob = b"fn main() {}\n".to_vec();
651 let blob_id = object_id(ObjectKind::Blob, &blob);
652 let tree = encode_tree(&[TreeItem { mode: "100644".into(), name: "main.rs".into(), id: blob_id }]);
653 let tree_id = object_id(ObjectKind::Tree, &tree);
654 let body = push(&[
655 Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)),
656 Entry::Whole(ObjectKind::Tree, tree),
657 Entry::Whole(ObjectKind::Blob, blob),
658 ]);
659 assert!(run(scan_push(&FakeRepo::default(), &body)).unwrap().is_empty());
660 // A deletion sends commands and no pack.
661 assert!(run(scan_push(&FakeRepo::default(), b"0000")).unwrap().is_empty());
662 }
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look663
664 #[test]
665 fn a_push_carrying_the_pushers_private_address_is_declined_with_a_masked_address() {
666 let tree = encode_tree(&[]);
667 let tree_id = object_id(ObjectKind::Tree, &tree);
668 let mine = format!("tree {tree_id}
669author S <Sam@Gmail.com> 0 +0000
670committer S <sam@gmail.com> 0 +0000
671
672x
673").into_bytes();
674 let mine_id = object_id(ObjectKind::Commit, &mine);
675 let guard = PushEmailGuard { emails: vec!["sam@gmail.com".into()], noreply: "1abc2def+sam@users.noreply.g1t.sh".into() };
676 let body = push(&[Entry::Whole(ObjectKind::Commit, mine), Entry::Whole(ObjectKind::Tree, tree.clone())]);
677 let (found, email) = exposed_address(&body, &guard).unwrap();
678 assert_eq!(found, mine_id);
679 let message = exposed_message(&found, &email, &guard.noreply);
680 assert!(message[0].starts_with(&format!("push declined: commit {} would publish s***@gmail.com", &mine_id[..7])));
681 assert!(message[1].contains("git config user.email 1abc2def+sam@users.noreply.g1t.sh"));
682 assert!(message[2].contains("g1t.sh/settings#emails"));
683 // Someone else's commits, and no pack at all, go through.
684 let theirs = push(&[Entry::Whole(ObjectKind::Commit, commit(&tree_id, None)), Entry::Whole(ObjectKind::Tree, tree)]);
685 assert_eq!(exposed_address(&theirs, &guard), None);
686 assert_eq!(exposed_address(b"0000", &guard), None);
687 }
Agents get guardrails, run credentials, an audit log, a context hub, repository instructions and mentions; security upkeep; snake_case API688}