Skip to content
Sign in

flagon-io/g1t

public
0

g1t/crates/scan/src/custom.rs

352 lines15,328 bytesCodeBlame
1//! Custom patterns: secret formats a workspace or repository defines for
2//! itself, as regular expressions, found alongside the built-in formats in
3//! [`crate::secrets`] by push protection and history scans.
4//!
5//! A pattern is untrusted input that runs on every push, so it is compiled
6//! with the `regex` crate, whose engines run in time linear in the text
7//! (no backtracking, no look-around, no back-references), and within size
8//! limits: [`MAX_PATTERN_CHARS`] of source, [`COMPILED_SIZE_LIMIT`] bytes
9//! compiled. A pattern that matches the empty string is refused, as is one
10//! that matches every test string meant to stay unmatched.
11
12use regex::{Regex, RegexBuilder};
13use serde::{Deserialize, Serialize};
14use sha2::{Digest, Sha256};
15
16use crate::secrets::ALLOW_MARKER;
17
18/// The longest pattern, and the longest before or after context, in
19/// characters.
20pub const MAX_PATTERN_CHARS: usize = 1_000;
21/// What one compiled pattern may take, in bytes, for its program and for
22/// its lazy DFA's cache each.
23pub const COMPILED_SIZE_LIMIT: usize = 1 << 20;
24/// How deeply a pattern may nest groups and repetitions.
25pub const MAX_NEST: u32 = 50;
26/// The most patterns one repository is scanned with: its own and its
27/// workspace's together.
28pub const MAX_PATTERNS: usize = 100;
29/// Test strings a pattern keeps, and how long each may be.
30pub const MAX_TEST_STRINGS: usize = 20;
31pub const MAX_TEST_STRING_CHARS: usize = 2_000;
32/// The longest secret a pattern can find: a longer match is cut here for
33/// its fingerprint and preview, never stored whole.
34pub const MAX_SECRET_CHARS: usize = 1_000;
35/// Lines longer than this are skipped (minified code, data).
36pub const MAX_LINE_CHARS: usize = 4_000;
37
38/// What has to come before a secret when the pattern says nothing: the
39/// start of the line or a character that is not a letter or digit.
40pub const DEFAULT_BEFORE: &str = r"\A|[^0-9A-Za-z]";
41/// And after it: the end of the line or the same.
42pub const DEFAULT_AFTER: &str = r"\z|[^0-9A-Za-z]";
43
44/// A pattern as it is stored and sent between services.
45#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct PatternSpec {
48 /// `pat_…`.
49 pub id: String,
50 /// What people call it: "Acme internal API key".
51 pub name: String,
52 /// The secret's format.
53 pub pattern: String,
54 /// What must come right before it; [`DEFAULT_BEFORE`] when absent.
55 #[serde(default, skip_serializing_if = "Option::is_none")]
56 pub before: Option<String>,
57 /// What must come right after it; [`DEFAULT_AFTER`] when absent.
58 #[serde(default, skip_serializing_if = "Option::is_none")]
59 pub after: Option<String>,
60}
61
62/// A pattern ready to run.
63#[derive(Clone, Debug)]
64pub struct Compiled {
65 pub id: String,
66 pub name: String,
67 regex: Regex,
68}
69
70/// A secret a custom pattern found on one line.
71#[derive(Clone, Debug, PartialEq, Eq)]
72pub struct CustomHit {
73 pub pattern_id: String,
74 pub pattern_name: String,
75 /// From 1.
76 pub line: u32,
77 /// The secret itself: never stored or shown, as with built-in hits.
78 pub value: String,
79}
80
81/// The kind custom findings are stored under; the pattern says which.
82pub const KIND: &str = "custom_pattern";
83
84impl CustomHit {
85 pub fn fingerprint(&self) -> String {
86 fingerprint(&self.pattern_id, &self.value)
87 }
88
89 /// The first quarter of the secret, three to eight characters.
90 pub fn preview(&self) -> String {
91 let chars: Vec<char> = self.value.chars().collect();
92 let shown = (chars.len() / 4).clamp(3, 8).min(chars.len());
93 format!("{}…", chars[..shown].iter().collect::<String>())
94 }
95
96 /// "Acme internal API key", as a sentence names it.
97 pub fn label(&self) -> String {
98 label(&self.pattern_name)
99 }
100}
101
102/// How a sentence names a custom pattern's finding.
103pub fn label(name: &str) -> String {
104 format!("a match for the custom pattern \"{name}\"")
105}
106
107/// Names a secret a custom pattern found without holding it.
108pub fn fingerprint(pattern_id: &str, value: &str) -> String {
109 let digest = Sha256::digest(format!("custom:{pattern_id}:{value}").as_bytes());
110 digest[..16].iter().map(|byte| format!("{byte:02x}")).collect()
111}
112
113fn checked(what: &str, source: &str) -> Result<(), String> {
114 if source.trim().is_empty() {
115 return Err(format!("The {what} is empty."));
116 }
117 if source.chars().count() > MAX_PATTERN_CHARS {
118 return Err(format!("The {what} is longer than {MAX_PATTERN_CHARS} characters."));
119 }
120 Ok(())
121}
122
123fn build(source: &str) -> Result<Regex, String> {
124 RegexBuilder::new(source)
125 .size_limit(COMPILED_SIZE_LIMIT)
126 .dfa_size_limit(COMPILED_SIZE_LIMIT)
127 .nest_limit(MAX_NEST)
128 .build()
129 .map_err(|error| match error {
130 regex::Error::CompiledTooBig(_) => "The pattern is too complex: simplify its repetitions.".to_owned(),
131 regex::Error::Syntax(text) => {
132 // The crate's message is several lines with a caret under
133 // the problem; its last line says what is wrong.
134 let last = text.lines().rev().find(|line| line.starts_with("error:")).unwrap_or(&text);
135 format!("The pattern is not a valid regular expression: {}", last.trim_start_matches("error:").trim())
136 }
137 other => format!("The pattern could not be compiled: {other}"),
138 })
139}
140
141/// Compiles a pattern, or says what is wrong with it.
142pub fn compile(spec: &PatternSpec) -> Result<Compiled, String> {
143 checked("pattern", &spec.pattern)?;
144 if spec.name.trim().is_empty() {
145 return Err("Give the pattern a name.".to_owned());
146 }
147 // Each part is compiled alone first, so an error names the right one.
148 build(&spec.pattern)?;
149 let before = spec.before.as_deref().filter(|text| !text.trim().is_empty());
150 let after = spec.after.as_deref().filter(|text| !text.trim().is_empty());
151 if let Some(before) = before {
152 checked("text before the secret", before)?;
153 build(before).map_err(|error| error.replace("The pattern", "The text before the secret"))?;
154 }
155 if let Some(after) = after {
156 checked("text after the secret", after)?;
157 build(after).map_err(|error| error.replace("The pattern", "The text after the secret"))?;
158 }
159 let pattern = Regex::new(&spec.pattern).map_err(|error| error.to_string())?;
160 if pattern.is_match("") {
161 return Err("The pattern matches an empty string, so it would match everywhere.".to_owned());
162 }
163 let combined = format!(
164 "(?:{})(?P<secret>{})(?:{})",
165 before.unwrap_or(DEFAULT_BEFORE),
166 spec.pattern,
167 after.unwrap_or(DEFAULT_AFTER)
168 );
169 Ok(Compiled { id: spec.id.clone(), name: spec.name.trim().to_owned(), regex: build(&combined)? })
170}
171
172/// Compiles every pattern that compiles; one that no longer does (it was
173/// saved under other limits) is skipped rather than failing a push.
174pub fn compile_all(specs: &[PatternSpec]) -> Vec<Compiled> {
175 specs.iter().take(MAX_PATTERNS).filter_map(|spec| compile(spec).ok()).collect()
176}
177
178impl Compiled {
179 /// The secrets on one line: the `secret` group of each match.
180 pub fn find(&self, line: &str) -> Vec<String> {
181 if line.len() > MAX_LINE_CHARS {
182 return Vec::new();
183 }
184 self.regex
185 .captures_iter(line)
186 .filter_map(|captures| captures.name("secret"))
187 .map(|found| found.as_str().chars().take(MAX_SECRET_CHARS).collect::<String>())
188 .filter(|value| !value.is_empty())
189 .collect()
190 }
191}
192
193/// Every secret the patterns find in `text`, on the lines `wanted` accepts
194/// (numbered from 1). A line with [`ALLOW_MARKER`] is skipped, as it is for
195/// the built-in formats.
196pub fn scan_lines(text: &str, patterns: &[Compiled], wanted: impl Fn(u32) -> bool) -> Vec<CustomHit> {
197 let mut hits = Vec::new();
198 if patterns.is_empty() {
199 return hits;
200 }
201 for (index, line) in text.lines().enumerate() {
202 let number = index as u32 + 1;
203 if !wanted(number) || line.contains(ALLOW_MARKER) {
204 continue;
205 }
206 for pattern in patterns {
207 for value in pattern.find(line) {
208 if hits.iter().any(|hit: &CustomHit| hit.line == number && hit.value == value) {
209 continue;
210 }
211 hits.push(CustomHit { pattern_id: pattern.id.clone(), pattern_name: pattern.name.clone(), line: number, value });
212 }
213 }
214 }
215 hits
216}
217
218/// Where a pattern matched a test string: the match's start and end, in
219/// characters, or `None`.
220pub fn test(compiled: &Compiled, strings: &[String]) -> Vec<Option<(usize, usize)>> {
221 strings
222 .iter()
223 .map(|text| {
224 compiled.regex.captures(text).and_then(|captures| captures.name("secret")).map(|found| {
225 let start = text[..found.start()].chars().count();
226 (start, start + found.as_str().chars().count())
227 })
228 })
229 .collect()
230}
231
232/// Cleans the test strings a person typed: trimmed of blank ones, at most
233/// [`MAX_TEST_STRINGS`] of at most [`MAX_TEST_STRING_CHARS`] each.
234pub fn clean_test_strings(strings: &[String]) -> Result<Vec<String>, String> {
235 let kept: Vec<String> = strings.iter().filter(|text| !text.trim().is_empty()).cloned().collect();
236 if kept.len() > MAX_TEST_STRINGS {
237 return Err(format!("Keep at most {MAX_TEST_STRINGS} test strings."));
238 }
239 if kept.iter().any(|text| text.chars().count() > MAX_TEST_STRING_CHARS) {
240 return Err(format!("A test string is longer than {MAX_TEST_STRING_CHARS} characters."));
241 }
242 Ok(kept)
243}
244
245/// A preview of a match, as a dry run shows it: the line with the secret
246/// masked but for its first characters, cut to 160 characters around it.
247pub fn masked_line(line: &str, value: &str) -> String {
248 let shown: String = value.chars().take((value.chars().count() / 4).clamp(3, 8)).collect();
249 let masked = format!("{shown}{}", "•".repeat(value.chars().count().saturating_sub(shown.chars().count()).min(24)));
250 let replaced = line.replacen(value, &masked, 1);
251 let trimmed = replaced.trim();
252 if trimmed.chars().count() <= 160 {
253 return trimmed.to_owned();
254 }
255 let at = trimmed.find(&masked).map(|at| trimmed[..at].chars().count()).unwrap_or(0);
256 let start = at.saturating_sub(60);
257 let cut: String = trimmed.chars().skip(start).take(160).collect();
258 format!("{}{cut}…", if start > 0 { "…" } else { "" })
259}
260
261#[cfg(test)]
262mod tests {
263 use super::*;
264
265 fn spec(pattern: &str) -> PatternSpec {
266 PatternSpec { id: "pat_1".into(), name: "Acme key".into(), pattern: pattern.into(), before: None, after: None }
267 }
268
269 #[test]
270 fn a_pattern_finds_its_secret_between_boundaries() {
271 let compiled = compile(&spec(r"acme_[a-z0-9]{24}")).unwrap();
272 let text = "a = 1\nKEY=acme_0123456789abcdefghijklmn\nxacme_0123456789abcdefghijklmn\n";
273 let hits = scan_lines(text, std::slice::from_ref(&compiled), |_| true);
274 assert_eq!(hits.len(), 1);
275 assert_eq!((hits[0].line, hits[0].value.as_str()), (2, "acme_0123456789abcdefghijklmn"));
276 assert_eq!(hits[0].preview(), "acme_01…");
277 assert_eq!(hits[0].fingerprint(), fingerprint("pat_1", "acme_0123456789abcdefghijklmn"));
278 assert_ne!(hits[0].fingerprint(), fingerprint("pat_2", "acme_0123456789abcdefghijklmn"));
279 // Only wanted lines, and never a line marked allowed.
280 assert!(scan_lines(text, std::slice::from_ref(&compiled), |line| line != 2).is_empty());
281 let allowed = "KEY=acme_0123456789abcdefghijklmn # g1t:allow-secret";
282 assert!(scan_lines(allowed, &[compiled], |_| true).is_empty());
283 }
284
285 #[test]
286 fn before_and_after_context_narrow_a_match() {
287 let mut with_context = spec(r"[A-Z0-9]{20}");
288 with_context.before = Some(r#"ACME_TOKEN\s*=\s*""#.into());
289 with_context.after = Some("\"".into());
290 let compiled = compile(&with_context).unwrap();
291 assert_eq!(compiled.find(r#"ACME_TOKEN = "ABCDEFGHIJ0123456789""#), ["ABCDEFGHIJ0123456789"]);
292 assert!(compiled.find(r#"OTHER = "ABCDEFGHIJ0123456789""#).is_empty());
293 let results = test(&compiled, &[r#"ACME_TOKEN="ABCDEFGHIJ0123456789""#.into(), "nothing".into()]);
294 assert_eq!(results, [Some((12, 32)), None]);
295 }
296
297 #[test]
298 fn bad_and_dangerous_patterns_are_refused() {
299 assert!(compile(&spec("")).unwrap_err().contains("empty"));
300 assert!(compile(&spec("(unclosed")).unwrap_err().starts_with("The pattern is not a valid regular expression"));
301 assert!(compile(&spec("a*")).unwrap_err().contains("empty string"));
302 // Look-around and back-references, which need backtracking, do
303 // not exist in this engine.
304 assert!(compile(&spec(r"(?=x)abc")).is_err());
305 assert!(compile(&spec(r"(a)\1")).is_err());
306 assert!(compile(&spec(&"a".repeat(MAX_PATTERN_CHARS + 1))).unwrap_err().contains("longer than"));
307 // A pattern whose compiled program would be enormous.
308 assert!(compile(&spec(r"\w{1000}\w{1000}\w{1000}")).unwrap_err().contains("too complex"));
309 let mut unnamed = spec("abc");
310 unnamed.name = " ".into();
311 assert!(compile(&unnamed).is_err());
312 let mut bad_after = spec("abc");
313 bad_after.after = Some("[".into());
314 assert!(compile(&bad_after).unwrap_err().starts_with("The text after the secret"));
315 }
316
317 #[test]
318 fn a_pattern_that_used_to_explode_runs_in_linear_time() {
319 // `(a+)+$` takes exponential time in a backtracking engine on a
320 // run of a's that ends in something else; here it is linear.
321 let compiled = compile(&spec(r"(a+)+b")).unwrap();
322 let line = format!("{}c", "a".repeat(MAX_LINE_CHARS - 1));
323 let started = std::time::Instant::now();
324 assert!(compiled.find(&line).is_empty());
325 assert!(started.elapsed() < std::time::Duration::from_secs(1));
326 // Lines past the limit are skipped outright.
327 assert!(compiled.find(&format!("{}b", "a".repeat(MAX_LINE_CHARS + 1))).is_empty());
328 }
329
330 #[test]
331 fn patterns_that_no_longer_compile_are_skipped() {
332 let specs = vec![spec("acme_[0-9]{8}"), spec("("), spec("x+")];
333 assert_eq!(compile_all(&specs).len(), 2);
334 }
335
336 #[test]
337 fn test_strings_are_limited() {
338 assert_eq!(clean_test_strings(&["a".into(), " ".into()]).unwrap(), ["a"]);
339 assert!(clean_test_strings(&vec!["a".to_owned(); MAX_TEST_STRINGS + 1]).is_err());
340 assert!(clean_test_strings(&["a".repeat(MAX_TEST_STRING_CHARS + 1)]).is_err());
341 }
342
343 #[test]
344 fn a_dry_run_masks_what_it_found() {
345 let line = "const key = 'acme_0123456789abcdefghijklmn';";
346 let masked = masked_line(line, "acme_0123456789abcdefghijklmn");
347 assert!(masked.starts_with("const key = 'acme_01•") && !masked.contains("abcdefghijklmn"));
348 let long = format!("{} acme_0123456789abcdefghijklmn {}", "x".repeat(300), "y".repeat(300));
349 let cut = masked_line(&long, "acme_0123456789abcdefghijklmn");
350 assert!(cut.starts_with('…') && cut.ends_with('…') && cut.contains("acme_01•"));
351 }
352}