Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/crates/scan/src/secrets.rs

713 lines28,831 bytesCodeBlame
1//! Secrets that must never reach a repository: keys and tokens of the
2//! formats their issuers made recognisable on purpose, and private keys.
3//!
4//! Each rule knows a format exactly (its prefix, its alphabet, its length),
5//! so a match is almost always a real credential, or a fake made to look
6//! like one. Fakes in tests and docs are still found, and [`test_value`]
7//! says which look made up (a documented example key, a counting or
8//! repeating value), so they are listed apart and never stop a push. A
9//! person can also mark a line with [`ALLOW_MARKER`], or dismiss a finding
10//! on the project's Security page. Guesswork from entropy alone is left out: it is
11//! what makes scanners noisy, and noise is what makes people bypass them.
12
13use sha2::{Digest, Sha256};
14
15/// Written anywhere on a line, in a comment, says what is on it is not a
16/// real secret: `const KEY = "AKIA…"; // g1t:allow-secret`.
17pub const ALLOW_MARKER: &str = "g1t:allow-secret";
18
19/// What a secret is.
20#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
21pub enum SecretKind {
22 AwsAccessKey,
23 AwsSecretKey,
24 GithubToken,
25 GitlabToken,
26 StripeLiveKey,
27 SlackToken,
28 SlackWebhook,
29 GoogleApiKey,
30 PrivateKey,
31 AnthropicKey,
32 OpenaiKey,
33 ServiceJwt,
34 NpmToken,
35 G1tToken,
36 SendgridKey,
37}
38
39impl SecretKind {
40 pub const ALL: [SecretKind; 15] = [
41 SecretKind::AwsAccessKey,
42 SecretKind::AwsSecretKey,
43 SecretKind::GithubToken,
44 SecretKind::GitlabToken,
45 SecretKind::StripeLiveKey,
46 SecretKind::SlackToken,
47 SecretKind::SlackWebhook,
48 SecretKind::GoogleApiKey,
49 SecretKind::PrivateKey,
50 SecretKind::AnthropicKey,
51 SecretKind::OpenaiKey,
52 SecretKind::ServiceJwt,
53 SecretKind::NpmToken,
54 SecretKind::G1tToken,
55 SecretKind::SendgridKey,
56 ];
57
58 /// Stored and sent between services.
59 pub fn id(self) -> &'static str {
60 match self {
61 SecretKind::AwsAccessKey => "aws_access_key",
62 SecretKind::AwsSecretKey => "aws_secret_key",
63 SecretKind::GithubToken => "github_token",
64 SecretKind::GitlabToken => "gitlab_token",
65 SecretKind::StripeLiveKey => "stripe_live_key",
66 SecretKind::SlackToken => "slack_token",
67 SecretKind::SlackWebhook => "slack_webhook",
68 SecretKind::GoogleApiKey => "google_api_key",
69 SecretKind::PrivateKey => "private_key",
70 SecretKind::AnthropicKey => "anthropic_key",
71 SecretKind::OpenaiKey => "openai_key",
72 SecretKind::ServiceJwt => "service_jwt",
73 SecretKind::NpmToken => "npm_token",
74 SecretKind::G1tToken => "g1t_token",
75 SecretKind::SendgridKey => "sendgrid_key",
76 }
77 }
78
79 /// For a sentence: "contains an AWS access key".
80 pub fn label(self) -> &'static str {
81 match self {
82 SecretKind::AwsAccessKey => "an AWS access key",
83 SecretKind::AwsSecretKey => "an AWS secret access key",
84 SecretKind::GithubToken => "a GitHub token",
85 SecretKind::GitlabToken => "a GitLab token",
86 SecretKind::StripeLiveKey => "a Stripe live key",
87 SecretKind::SlackToken => "a Slack token",
88 SecretKind::SlackWebhook => "a Slack webhook address",
89 SecretKind::GoogleApiKey => "a Google API key",
90 SecretKind::PrivateKey => "a private key",
91 SecretKind::AnthropicKey => "an Anthropic API key",
92 SecretKind::OpenaiKey => "an OpenAI API key",
93 SecretKind::ServiceJwt => "a service-role JWT",
94 SecretKind::NpmToken => "an npm token",
95 SecretKind::G1tToken => "a g1t token",
96 SecretKind::SendgridKey => "a SendGrid key",
97 }
98 }
99
100 pub fn parse(id: &str) -> Option<SecretKind> {
101 SecretKind::ALL.into_iter().find(|kind| kind.id() == id)
102 }
103}
104
105/// A secret found on one line.
106#[derive(Clone, Debug, PartialEq, Eq)]
107pub struct Hit {
108 pub kind: SecretKind,
109 /// From 1.
110 pub line: u32,
111 /// The secret itself. Never stored or shown: see [`Hit::fingerprint`]
112 /// and [`Hit::preview`].
113 pub value: String,
114}
115
116impl Hit {
117 /// Names this secret without holding it: the same secret found again,
118 /// in another commit or a rewritten one, has the same fingerprint.
119 pub fn fingerprint(&self) -> String {
120 fingerprint(self.kind, &self.value)
121 }
122
123 /// Enough of the secret for its owner to recognise it, and no more.
124 pub fn preview(&self) -> String {
125 preview(self.kind, &self.value)
126 }
127
128 /// Why this looks like a value made for tests or documentation rather
129 /// than a real credential, or `None`. See [`test_value`].
130 pub fn test_value(&self) -> Option<&'static str> {
131 test_value(self.kind, &self.value)
132 }
133}
134
135/// Keys their issuers publish in documentation, so that examples never use
136/// a real one. Each is split in two here, so that this file holds no whole
137/// key for a scanner to find.
138const DOCUMENTED_EXAMPLES: &[&str] = &[
139 // AWS's documentation: two access keys and their secret keys.
140 concat!("AKIA", "IOSFODNN7EXAMPLE"),
141 concat!("wJalrXUtnFEMI/K7MDENG/", "bPxRfiCYEXAMPLEKEY"),
142 concat!("AKIA", "I44QH8DHBEXAMPLE"),
143 concat!("je7MtGbClwBF/2Zp9Utk/", "h3yCo8nvbEXAMPLEKEY"),
144];
145
146/// Words that say a value is not real.
147const TEST_WORDS: &[&str] = &[
148 "example",
149 "sample",
150 "dummy",
151 "fake",
152 "placeholder",
153 "changeme",
154 "notreal",
155 "redacted",
156 "xxxxx",
157];
158
159/// Why a secret the rules found looks like a value made for tests or
160/// documentation, or `None` when it looks real. Judged from the value
161/// alone, never from where it is: a key in `tests/` is as real as one in
162/// `src/` if it was ever issued.
163///
164/// A likely test value is still a finding, listed apart from the others;
165/// it never stops a push and is never counted as critical.
166pub fn test_value(kind: SecretKind, value: &str) -> Option<&'static str> {
167 if DOCUMENTED_EXAMPLES.iter().any(|example| value.contains(example)) {
168 return Some("a key its issuer publishes as an example");
169 }
170 let lower = value.to_ascii_lowercase();
171 if TEST_WORDS.iter().any(|word| lower.contains(word)) {
172 return Some("it says it is an example");
173 }
174 // A private key is judged by its words only: its body is base64 of
175 // structured bytes, which can look patterned without being made up.
176 if kind == SecretKind::PrivateKey {
177 return None;
178 }
179 let chars: Vec<char> = body_of(kind, value).chars().map(|c| c.to_ascii_lowercase()).collect();
180 if longest(&chars, |a, b| b as u32 == a as u32 + 1) >= 6 {
181 return Some("it counts up, like abcdef or 123456");
182 }
183 if longest(&chars, |a, b| a == b) >= 5 {
184 return Some("one character over and over");
185 }
186 if repeated_piece(&chars) {
187 return Some("a short piece repeated");
188 }
189 if chars.len() >= 20 && entropy(&chars) < 3.0 {
190 return Some("too little randomness for a real key");
191 }
192 None
193}
194
195/// The part of a value its issuer generated: what follows the prefix its
196/// rule starts with, for the kinds that have one.
197fn body_of(kind: SecretKind, value: &str) -> &str {
198 RULES
199 .iter()
200 .filter(|rule| rule.kind == kind)
201 .flat_map(|rule| rule.prefixes.iter())
202 .filter(|prefix| value.starts_with(**prefix))
203 .map(|prefix| &value[prefix.len()..])
204 .min_by_key(|body| body.len())
205 .unwrap_or(value)
206}
207
208/// The longest run of characters each following the one before it by `next`.
209fn longest(chars: &[char], next: impl Fn(char, char) -> bool) -> usize {
210 let mut best = usize::from(!chars.is_empty());
211 let mut run = 1;
212 for pair in chars.windows(2) {
213 run = if next(pair[0], pair[1]) { run + 1 } else { 1 };
214 best = best.max(run);
215 }
216 best
217}
218
219/// Whether the value is one piece of two to eight characters, three times
220/// or more.
221fn repeated_piece(chars: &[char]) -> bool {
222 (2..=8).any(|size| chars.len() >= size * 3 && chars.iter().enumerate().skip(size).all(|(at, c)| *c == chars[at % size]))
223}
224
225/// Shannon entropy, in bits per character.
226fn entropy(chars: &[char]) -> f64 {
227 let mut counts = std::collections::HashMap::new();
228 for c in chars {
229 *counts.entry(*c).or_insert(0u32) += 1;
230 }
231 let total = chars.len() as f64;
232 counts
233 .values()
234 .map(|count| {
235 let p = f64::from(*count) / total;
236 -p * p.log2()
237 })
238 .sum()
239}
240
241pub fn fingerprint(kind: SecretKind, value: &str) -> String {
242 let digest = Sha256::digest(format!("{}:{value}", kind.id()).as_bytes());
243 digest[..16].iter().map(|byte| format!("{byte:02x}")).collect()
244}
245
246pub fn preview(kind: SecretKind, value: &str) -> String {
247 if kind == SecretKind::PrivateKey {
248 return value.lines().next().unwrap_or("-----BEGIN PRIVATE KEY-----").to_owned();
249 }
250 let chars: Vec<char> = value.chars().collect();
251 let shown = (chars.len() / 4).clamp(3, 8);
252 format!("{}…", chars[..shown.min(chars.len())].iter().collect::<String>())
253}
254
255#[derive(Clone, Copy)]
256enum Alphabet {
257 /// A–Z and 0–9.
258 UpperDigit,
259 /// Letters and digits.
260 Alnum,
261 /// Letters, digits and `_`.
262 Word,
263 /// Letters, digits, `_` and `-`.
264 Token,
265 /// Lowercase hex.
266 Hex,
267 /// Letters, digits, `_`, `-` and `.`.
268 Dotted,
269 /// Letters, digits and `/`.
270 Path,
271}
272
273impl Alphabet {
274 fn has(self, c: char) -> bool {
275 match self {
276 Alphabet::UpperDigit => c.is_ascii_uppercase() || c.is_ascii_digit(),
277 Alphabet::Alnum => c.is_ascii_alphanumeric(),
278 Alphabet::Word => c.is_ascii_alphanumeric() || c == '_',
279 Alphabet::Token => c.is_ascii_alphanumeric() || c == '_' || c == '-',
280 Alphabet::Hex => c.is_ascii_digit() || ('a'..='f').contains(&c),
281 Alphabet::Dotted => c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'),
282 Alphabet::Path => c.is_ascii_alphanumeric() || c == '/',
283 }
284 }
285}
286
287/// A format: what it starts with, which characters follow, and how many.
288struct Rule {
289 kind: SecretKind,
290 prefixes: &'static [&'static str],
291 body: Alphabet,
292 min: usize,
293 max: usize,
294 /// Anything else the body has to be.
295 check: Option<fn(&str) -> bool>,
296}
297
298const RULES: &[Rule] = &[
299 Rule { kind: SecretKind::AwsAccessKey, prefixes: &["AKIA", "ASIA", "ABIA", "ACCA"], body: Alphabet::UpperDigit, min: 16, max: 16, check: None },
300 Rule { kind: SecretKind::GithubToken, prefixes: &["ghp_", "gho_", "ghu_", "ghs_", "ghr_"], body: Alphabet::Alnum, min: 36, max: 255, check: None },
301 Rule { kind: SecretKind::GithubToken, prefixes: &["github_pat_"], body: Alphabet::Word, min: 50, max: 255, check: None },
302 Rule { kind: SecretKind::GitlabToken, prefixes: &["glpat-", "gloas-", "glrt-", "glptt-", "gldt-"], body: Alphabet::Token, min: 20, max: 64, check: None },
303 Rule { kind: SecretKind::StripeLiveKey, prefixes: &["sk_live_", "rk_live_"], body: Alphabet::Alnum, min: 20, max: 255, check: None },
304 Rule { kind: SecretKind::SlackToken, prefixes: &["xoxb-", "xoxp-", "xoxa-", "xoxr-", "xoxs-", "xoxe-"], body: Alphabet::Token, min: 20, max: 255, check: Some(has_digit) },
305 Rule { kind: SecretKind::SlackWebhook, prefixes: &["https://hooks.slack.com/services/"], body: Alphabet::Path, min: 30, max: 120, check: Some(slack_webhook) },
306 Rule { kind: SecretKind::GoogleApiKey, prefixes: &["AIza"], body: Alphabet::Token, min: 35, max: 35, check: None },
307 Rule { kind: SecretKind::AnthropicKey, prefixes: &["sk-ant-"], body: Alphabet::Token, min: 40, max: 255, check: None },
308 Rule { kind: SecretKind::OpenaiKey, prefixes: &["sk-proj-", "sk-svcacct-", "sk-admin-"], body: Alphabet::Token, min: 40, max: 255, check: None },
309 // The keys OpenAI issued before project keys carry "T3BlbkFJ" in the middle.
310 Rule { kind: SecretKind::OpenaiKey, prefixes: &["sk-"], body: Alphabet::Alnum, min: 40, max: 60, check: Some(openai_legacy) },
311 Rule { kind: SecretKind::NpmToken, prefixes: &["npm_"], body: Alphabet::Alnum, min: 36, max: 36, check: None },
312 Rule { kind: SecretKind::G1tToken, prefixes: &["g1t_"], body: Alphabet::Hex, min: 40, max: 40, check: None },
313 Rule { kind: SecretKind::SendgridKey, prefixes: &["SG."], body: Alphabet::Dotted, min: 66, max: 66, check: Some(sendgrid) },
314];
315
316fn has_digit(body: &str) -> bool {
317 body.chars().any(|c| c.is_ascii_digit())
318}
319
320fn slack_webhook(body: &str) -> bool {
321 let parts: Vec<&str> = body.split('/').collect();
322 parts.len() == 3 && parts[0].starts_with('T') && parts[1].starts_with('B') && parts[2].len() >= 20
323}
324
325fn openai_legacy(body: &str) -> bool {
326 body.contains("T3BlbkFJ")
327}
328
329fn sendgrid(body: &str) -> bool {
330 let parts: Vec<&str> = body.split('.').collect();
331 parts.len() == 2 && parts[0].len() == 22 && parts[1].len() == 43
332}
333
334/// A run of one character over and over, or a handful of them, is a
335/// placeholder in documentation: `ghp_xxxxxxxx…`, `AKIA0000…`.
336fn placeholder(body: &str) -> bool {
337 let mut seen = std::collections::HashSet::new();
338 for c in body.chars() {
339 seen.insert(c.to_ascii_lowercase());
340 }
341 seen.len() < 6
342}
343
344fn boundary_before(line: &str, at: usize) -> bool {
345 line[..at]
346 .chars()
347 .next_back()
348 .is_none_or(|c| !c.is_ascii_alphanumeric())
349}
350
351/// The secrets of the formats in [`RULES`] on one line.
352fn by_rules(line: &str, found: &mut Vec<(SecretKind, String)>) {
353 for rule in RULES {
354 for prefix in rule.prefixes {
355 let mut from = 0;
356 while let Some(offset) = line[from..].find(prefix) {
357 let at = from + offset;
358 from = at + prefix.len();
359 if !boundary_before(line, at) {
360 continue;
361 }
362 let body: String = line[from..].chars().take_while(|c| rule.body.has(*c)).collect();
363 let length = body.chars().count();
364 if length < rule.min || length > rule.max || placeholder(&body) {
365 continue;
366 }
367 if rule.check.is_some_and(|check| !check(&body)) {
368 continue;
369 }
370 found.push((rule.kind, format!("{prefix}{body}")));
371 }
372 }
373 }
374}
375
376/// `aws_secret_access_key = "…"`: forty characters of base64 are only an
377/// AWS secret when the line says so.
378fn aws_secret(line: &str, found: &mut Vec<(SecretKind, String)>) {
379 let lower = line.to_ascii_lowercase();
380 if !lower.contains("aws") || !lower.contains("secret") {
381 return;
382 }
383 for (at, _) in line.match_indices(['=', ':']) {
384 let value: String = line[at + 1..]
385 .trim_start_matches([' ', '"', '\'', '\t'])
386 .chars()
387 .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '+'))
388 .collect();
389 let mixed = value.chars().any(|c| c.is_ascii_uppercase())
390 && value.chars().any(|c| c.is_ascii_lowercase())
391 && value.chars().any(|c| c.is_ascii_digit());
392 if value.len() == 40 && mixed && !placeholder(&value) {
393 found.push((SecretKind::AwsSecretKey, value));
394 }
395 }
396}
397
398fn base64url_decode(input: &str) -> Option<Vec<u8>> {
399 let mut bytes = Vec::with_capacity(input.len() * 3 / 4);
400 let (mut buffer, mut bits) = (0u32, 0);
401 for byte in input.bytes().filter(|byte| *byte != b'=') {
402 let value = match byte {
403 b'A'..=b'Z' => byte - b'A',
404 b'a'..=b'z' => byte - b'a' + 26,
405 b'0'..=b'9' => byte - b'0' + 52,
406 b'-' | b'+' => 62,
407 b'_' | b'/' => 63,
408 _ => return None,
409 };
410 buffer = (buffer << 6) | u32::from(value);
411 bits += 6;
412 if bits >= 8 {
413 bits -= 8;
414 bytes.push((buffer >> bits) as u8);
415 }
416 }
417 Some(bytes)
418}
419
420/// A JWT whose claims make it a service's key rather than a user's
421/// session: one that bypasses row-level security, signed to last.
422fn service_jwt(line: &str, found: &mut Vec<(SecretKind, String)>) {
423 let mut from = 0;
424 while let Some(offset) = line[from..].find("eyJ") {
425 let at = from + offset;
426 from = at + 3;
427 if !boundary_before(line, at) {
428 continue;
429 }
430 let token: String = line[at..]
431 .chars()
432 .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'))
433 .collect();
434 let parts: Vec<&str> = token.split('.').collect();
435 if parts.len() != 3 || parts[0].len() < 10 || parts[1].len() < 10 || parts[2].len() < 20 {
436 continue;
437 }
438 let Some(claims) = base64url_decode(parts[1]) else {
439 continue;
440 };
441 let claims = String::from_utf8_lossy(&claims);
442 if claims.contains("\"service_role\"") {
443 from = at + token.len();
444 found.push((SecretKind::ServiceJwt, token));
445 }
446 }
447}
448
449const PEM_BODY: usize = 40;
450
451fn looks_like_pem_body(text: &str) -> bool {
452 let text = text.trim().trim_start_matches(['"', '\'']);
453 text.starts_with("Proc-Type:")
454 || text
455 .chars()
456 .take_while(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '='))
457 .count()
458 >= PEM_BODY
459}
460
461/// The header of a PEM private key, when a key follows it: on the next
462/// line, or after an escaped newline on the same one, as a key pasted into
463/// a string does. A header alone is code that handles keys, not a key.
464fn private_key(line: &str, next: Option<&str>, found: &mut Vec<(SecretKind, String)>) {
465 let Some(start) = line.find("-----BEGIN ") else {
466 return;
467 };
468 let rest = &line[start..];
469 let Some(end) = rest[11..].find("-----") else {
470 return;
471 };
472 let header = &rest[..11 + end + 5];
473 if !header.contains("PRIVATE KEY") {
474 return;
475 }
476 let after = rest[header.len()..].trim_start_matches("\\n").trim_start_matches("\\r\\n");
477 let body = if looks_like_pem_body(after) {
478 Some(after)
479 } else {
480 next.filter(|next| looks_like_pem_body(next))
481 };
482 if let Some(body) = body {
483 let body: String = body
484 .trim()
485 .trim_start_matches(['"', '\''])
486 .chars()
487 .take_while(|c| !c.is_whitespace() && *c != '\\' && *c != '"')
488 .collect();
489 found.push((SecretKind::PrivateKey, format!("{header}\n{body}")));
490 }
491}
492
493/// The secrets on one line. `next` is the line after it, which a private
494/// key's header needs to tell a key from code that mentions one.
495pub fn scan_line(line: &str, next: Option<&str>) -> Vec<(SecretKind, String)> {
496 let mut found = Vec::new();
497 if line.contains(ALLOW_MARKER) {
498 return found;
499 }
500 by_rules(line, &mut found);
501 aws_secret(line, &mut found);
502 service_jwt(line, &mut found);
503 private_key(line, next, &mut found);
504 // Two rules can find one secret (`sk-` and `sk-proj-`); keep the first.
505 let mut seen = std::collections::HashSet::new();
506 found.retain(|(_, value)| seen.insert(value.clone()));
507 found
508}
509
510/// Every secret in `text`, on the lines `wanted` accepts (numbered from 1).
511pub fn scan_lines(text: &str, wanted: impl Fn(u32) -> bool) -> Vec<Hit> {
512 let lines: Vec<&str> = text.lines().collect();
513 let mut hits = Vec::new();
514 for (index, line) in lines.iter().enumerate() {
515 let number = index as u32 + 1;
516 if !wanted(number) {
517 continue;
518 }
519 for (kind, value) in scan_line(line, lines.get(index + 1).copied()) {
520 hits.push(Hit { kind, line: number, value });
521 }
522 }
523 hits
524}
525
526/// Every secret in `text`.
527pub fn scan_text(text: &str) -> Vec<Hit> {
528 scan_lines(text, |_| true)
529}
530
531/// Paths whose contents are never secrets of their own: lockfiles and
532/// vendored or generated code, where a match is someone else's fixture.
533pub fn skipped_path(path: &str) -> bool {
534 let name = path.rsplit('/').next().unwrap_or(path);
535 matches!(
536 name,
537 "package-lock.json" | "pnpm-lock.yaml" | "yarn.lock" | "Cargo.lock" | "go.sum" | "poetry.lock"
538 ) || path.split('/').any(|part| part == "node_modules" || part == "vendor")
539 || name.ends_with(".min.js")
540 || name.ends_with(".map")
541}
542
543#[cfg(test)]
544mod tests {
545 use super::*;
546
547 /// Key-shaped values are put together at run time, so that no whole
548 /// key sits in this file for any scanner, this one included, to flag.
549 fn join(a: &str, b: &str) -> String {
550 format!("{a}{b}")
551 }
552
553 fn kinds(line: &str) -> Vec<SecretKind> {
554 scan_line(line, None).into_iter().map(|(kind, _)| kind).collect()
555 }
556
557 #[test]
558 fn real_formats_are_found() {
559 let cases = [
560 (join("aws_access_key_id = AK", "IAZ7Q4N2XWLM3KDTRV"), SecretKind::AwsAccessKey),
561 (join("token: gh", "p_Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3b"), SecretKind::GithubToken),
562 (join("GITLAB=glp", "at-x7Rk2PqLm9VwT4bN8cZs"), SecretKind::GitlabToken),
563 (join("STRIPE_KEY=sk_l", "ive_51HabcdEFGhijKLMnop7QRSTuv"), SecretKind::StripeLiveKey),
564 (join("SLACK=xo", "xb-2048-1029384756-Zq8wN3vR7tY2uI5oP1aS"), SecretKind::SlackToken),
565 (join("url = https://hooks.slack.com/serv", "ices/T024BE7LD/B01ABCDEFGH/Zq8wN3vR7tY2uI5oP1aS6dF4"), SecretKind::SlackWebhook),
566 (join("key: AI", "zaSyD-9tSrke72PouQMnMX-a7eZSW0jkFMBWY"), SecretKind::GoogleApiKey),
567 (join("ANTHROPIC_API_KEY=sk-an", "t-api03-Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3bVn8wQ2"), SecretKind::AnthropicKey),
568 (join("OPENAI_API_KEY=sk-pr", "oj-Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3bVn8wQ2xx"), SecretKind::OpenaiKey),
569 (join("key = \"sk-Zq8wN3vR7tY2uI5oP1a", "T3BlbkFJS6dF4gH9jK0lXmC3b\""), SecretKind::OpenaiKey),
570 (join("//registry.npmjs.org/:_authToken=np", "m_Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3b"), SecretKind::NpmToken),
571 (join("G1T_TOKEN=g1", "t_3f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a"), SecretKind::G1tToken),
572 (join("SENDGRID=SG", ".Zq8wN3vR7tY2uI5oP1aS6x.dF4gH9jK0lXmC3bVn8wQ2xZq8wN3vR7tY2uI5oP1aS6"), SecretKind::SendgridKey),
573 (join("aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCY", "Q2x9Lm3Kd7"), SecretKind::AwsSecretKey),
574 ];
575 for (line, kind) in cases {
576 assert_eq!(kinds(&line), [kind], "{line}");
577 }
578 }
579
580 #[test]
581 fn fakes_are_found_too_and_the_marker_allows_them() {
582 let example = join("AWS_KEY=AK", "IAIOSFODNN7EXAMPLE");
583 assert_eq!(kinds(&example), [SecretKind::AwsAccessKey]);
584 assert!(kinds(&format!("{example} # g1t:allow-secret")).is_empty());
585 assert!(kinds(&format!("{example} // {ALLOW_MARKER}")).is_empty());
586 }
587
588 fn test_value_of(line: &str) -> Option<&'static str> {
589 let hits = scan_text(line);
590 assert_eq!(hits.len(), 1, "{line}");
591 hits[0].test_value()
592 }
593
594 #[test]
595 fn test_values_are_told_from_real_ones() {
596 // Documented examples, words, counting, repeats, and too little randomness.
597 for (line, why) in [
598 (join("AWS: AK", "IAIOSFODNN7EXAMPLE"), "publishes as an example"),
599 (join("aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCY", "EXAMPLEKEY"), "publishes as an example"),
600 (join("use gh", "p_abcdefghijklmnopqrstuvwxyz0123456789 to clone"), "counts up"),
601 (join("STRIPE_KEY=sk_l", "ive_51HabcdefghijklmnopQRSTUV"), "counts up"),
602 (join("SLACK=xo", "xb-2048-1000000-Zq8wN3vR7tY2uI5oP1aS"), "over and over"),
603 (join("token: gh", "p_q8Zw3Rq8Zw3Rq8Zw3Rq8Zw3Rq8Zw3Rq8Zw3R"), "piece repeated"),
604 (join("key: sk-an", "t-api03-FAKEFAKEZq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3b"), "says it is an example"),
605 (join("G1T_TOKEN=g1", "t_aa1aa2aa3aa4aa5aa6aa1aa2aa3aa4aa5aa6aa1a"), "too little randomness"),
606 ] {
607 let found = test_value_of(&line).unwrap_or_else(|| panic!("{line} looks real"));
608 assert!(found.contains(why), "{line}: {found}");
609 }
610 // Keys that look issued are not.
611 for line in [
612 join("aws_access_key_id = AK", "IAZ7Q4N2XWLM3KDTRV"),
613 join("token: gh", "p_Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3b"),
614 join("G1T_TOKEN=g1", "t_3f9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a"),
615 join("aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCY", "Q2x9Lm3Kd7"),
616 join("ANTHROPIC_API_KEY=sk-an", "t-api03-Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3bVn8wQ2"),
617 ] {
618 assert_eq!(test_value_of(&line), None, "{line}");
619 }
620 // A file's path is never the reason: the same value is judged the same.
621 let key = join("AK", "IAZ7Q4N2XWLM3KDTRV");
622 assert_eq!(test_value(SecretKind::AwsAccessKey, &key), None);
623 }
624
625 #[test]
626 fn ordinary_code_is_left_alone() {
627 for line in [
628 "Commit 9f2c4e1a7b3d5f60812a4c6e8b0d2f4a6c8e0b13 introduced the flaky retry.",
629 "const prefix = \"ghp_\";",
630 "STRIPE_KEY=sk_test_51HabcdEFGhijKLMnop7QRSTuv",
631 "GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
632 "if pem.starts_with(\"-----BEGIN PRIVATE KEY-----\") {",
633 "\"integrity\": \"sha512-Zq8wN3vR7tY2uI5oP1aS6dF4gH9jK0lXmC3bVn8wQ2xZq8wN3vR7tY2uI5oP1aS6dF==\"",
634 "password = hunter2hunter2",
635 "let task = ASIAN_MARKETS;",
636 "import { AIzaClient } from './maps';",
637 "-----BEGIN PUBLIC KEY-----",
638 "secret = process.env.AWS_SECRET_ACCESS_KEY",
639 "https://hooks.slack.com/services/T000/B000/XXXXXXXXXXXXXXXXXXXXXXXX",
640 ] {
641 assert!(kinds(line).is_empty(), "{line}");
642 }
643 }
644
645 #[test]
646 fn a_private_key_needs_its_body() {
647 let header = join("-----BEGIN RSA PRIVA", "TE KEY-----");
648 let body = "MIIEowIBAAKCAQEAu1SU1LfVLPHCozMxH2Mo4lgOEePzNm0tRgeLezV6ffAt0gun";
649 assert_eq!(kinds_with_next(&header, Some(body)), [SecretKind::PrivateKey]);
650 assert!(kinds_with_next(&header, Some("...")).is_empty());
651 let escaped = format!("key: \"{header}\\n{body}\\n\"");
652 assert_eq!(kinds(&escaped), [SecretKind::PrivateKey]);
653 let hits = scan_text(&format!("x\n{header}\n{body}\n-----END RSA PRIVATE KEY-----\n"));
654 assert_eq!(hits.len(), 1);
655 assert_eq!(hits[0].line, 2);
656 assert!(hits[0].preview().contains("BEGIN RSA"));
657 }
658
659 fn kinds_with_next(line: &str, next: Option<&str>) -> Vec<SecretKind> {
660 scan_line(line, next).into_iter().map(|(kind, _)| kind).collect()
661 }
662
663 #[test]
664 fn a_service_role_jwt_is_found_and_a_user_jwt_is_not() {
665 // {"alg":"HS256","typ":"JWT"} . {"role":"service_role","iss":"supabase"}
666 let service = join(
667 "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaXNzIjoic3VwYWJhc2UifQ",
668 ".dGhpc2lzbm90YXJlYWxzaWduYXR1cmVidXRsb25nZW5vdWdo",
669 );
670 assert_eq!(kinds(&format!("SUPABASE_SERVICE_KEY={service}")), [SecretKind::ServiceJwt]);
671 // {"role":"anon"}
672 let anon = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiJ9.dGhpc2lzbm90YXJlYWxzaWduYXR1cmVidXRsb25nZW5vdWdo";
673 assert!(kinds(anon).is_empty());
674 }
675
676 #[test]
677 fn fingerprints_name_the_secret_not_where_it_is() {
678 let line = join("AK", "IAZ7Q4N2XWLM3KDTRV");
679 let a = scan_lines(&format!("a\n{line}\n"), |_| true);
680 let b = scan_lines(&format!("{line}\n"), |_| true);
681 assert_eq!(a[0].line, 2);
682 assert_eq!(b[0].line, 1);
683 assert_eq!(a[0].fingerprint(), b[0].fingerprint());
684 assert_eq!(a[0].fingerprint().len(), 32);
685 assert!(!a[0].preview().contains(&a[0].value));
686 assert!(a[0].preview().starts_with("AKIA"));
687 }
688
689 #[test]
690 fn only_the_wanted_lines_are_scanned() {
691 let key = join("AK", "IAZ7Q4N2XWLM3KDTRV");
692 let text = format!("{key}\nplain\n{key}\n");
693 let hits = scan_lines(&text, |line| line == 3);
694 assert_eq!(hits.len(), 1);
695 assert_eq!(hits[0].line, 3);
696 }
697
698 #[test]
699 fn kinds_round_trip() {
700 for kind in SecretKind::ALL {
701 assert_eq!(SecretKind::parse(kind.id()), Some(kind));
702 assert!(!kind.label().is_empty());
703 }
704 }
705
706 #[test]
707 fn lockfiles_and_vendored_code_are_skipped() {
708 assert!(skipped_path("web/package-lock.json"));
709 assert!(skipped_path("vendor/github.com/x/y.go"));
710 assert!(skipped_path("node_modules/a/index.js"));
711 assert!(!skipped_path("src/config.ts"));
712 }
713}