| 1 | //! Push protection: which lines a change adds, the secrets on them, and |
| 2 | //! what git is told when a push is refused for them. |
| 3 | |
| 4 | use std::collections::HashSet; |
| 5 | |
| 6 | use similar::{ChangeTag, TextDiff}; |
| 7 | |
| 8 | use crate::custom; |
| 9 | use crate::secrets::{self, ALLOW_MARKER, Hit}; |
| 10 | #[cfg(test)] |
| 11 | use crate::secrets::SecretKind; |
| 12 | |
| 13 | /// Files larger than this are not read for secrets. |
| 14 | pub const MAX_FILE_BYTES: usize = 2 * 1024 * 1024; |
| 15 | |
| 16 | /// The lines of `new` (numbered from 1) that are not in `old`. |
| 17 | pub fn added_lines(old: &str, new: &str) -> HashSet<u32> { |
| 18 | let diff = TextDiff::from_lines(old, new); |
| 19 | diff.iter_all_changes() |
| 20 | .filter(|change| change.tag() == ChangeTag::Insert) |
| 21 | .filter_map(|change| change.new_index().map(|index| index as u32 + 1)) |
| 22 | .collect() |
| 23 | } |
| 24 | |
| 25 | /// Text worth scanning, or `None` for binary, oversized or skipped files. |
| 26 | pub fn text_of<'a>(path: &str, bytes: &'a [u8]) -> Option<&'a str> { |
| 27 | if secrets::skipped_path(path) || bytes.len() > MAX_FILE_BYTES || bytes.contains(&0) { |
| 28 | return None; |
| 29 | } |
| 30 | std::str::from_utf8(bytes).ok() |
| 31 | } |
| 32 | |
| 33 | /// The secrets a change to one file adds: on its new lines only, so a |
| 34 | /// secret already in the repository (found and decided on before) does |
| 35 | /// not block every later push that touches the file. |
| 36 | pub fn scan_change(path: &str, old: Option<&[u8]>, new: &[u8]) -> Vec<Hit> { |
| 37 | let Some(new) = text_of(path, new) else { |
| 38 | return Vec::new(); |
| 39 | }; |
| 40 | match old.and_then(|old| std::str::from_utf8(old).ok()) { |
| 41 | Some(old) => { |
| 42 | let added = added_lines(old, new); |
| 43 | secrets::scan_lines(new, |line| added.contains(&line)) |
| 44 | } |
| 45 | None => secrets::scan_text(new), |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | /// What a change adds that the custom `patterns` find, on its new lines |
| 50 | /// only, as [`scan_change`] does for the built-in formats. |
| 51 | pub fn scan_change_custom(path: &str, old: Option<&[u8]>, new: &[u8], patterns: &[custom::Compiled]) -> Vec<custom::CustomHit> { |
| 52 | if patterns.is_empty() { |
| 53 | return Vec::new(); |
| 54 | } |
| 55 | let Some(new) = text_of(path, new) else { |
| 56 | return Vec::new(); |
| 57 | }; |
| 58 | match old.and_then(|old| std::str::from_utf8(old).ok()) { |
| 59 | Some(old) => { |
| 60 | let added = added_lines(old, new); |
| 61 | custom::scan_lines(new, patterns, |line| added.contains(&line)) |
| 62 | } |
| 63 | None => custom::scan_lines(new, patterns, |_| true), |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | /// A secret that stops a push. |
| 68 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 69 | pub struct Blocked { |
| 70 | /// What it is, for a sentence: "an AWS access key". |
| 71 | pub label: String, |
| 72 | pub path: String, |
| 73 | pub line: u32, |
| 74 | /// The commit that adds it. |
| 75 | pub commit: String, |
| 76 | /// Where it can be allowed, once, by someone who may. |
| 77 | pub allow_url: Option<String>, |
| 78 | } |
| 79 | |
| 80 | fn short(commit: &str) -> &str { |
| 81 | &commit[..commit.len().min(7)] |
| 82 | } |
| 83 | |
| 84 | /// The reason git prints beside each refused ref: one line. |
| 85 | pub fn reason(blocked: &[Blocked]) -> String { |
| 86 | match blocked { |
| 87 | [] => "refused".to_owned(), |
| 88 | [only] => format!("secret found: {}:{} has {}", only.path, only.line, only.label), |
| 89 | [first, rest @ ..] => format!( |
| 90 | "{} secrets found, first {}:{} ({})", |
| 91 | rest.len() + 1, |
| 92 | first.path, |
| 93 | first.line, |
| 94 | first.label |
| 95 | ), |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | /// What git shows the person pushing, a line at a time, as `remote:` |
| 100 | /// lines: every secret, where it is, and both ways forward. |
| 101 | pub fn explain(blocked: &[Blocked]) -> Vec<String> { |
| 102 | let count = blocked.len(); |
| 103 | let mut lines = vec![ |
| 104 | format!( |
| 105 | "g1t found {} in this push, so nothing was pushed.", |
| 106 | if count == 1 { "a secret".to_owned() } else { format!("{count} secrets") } |
| 107 | ), |
| 108 | String::new(), |
| 109 | ]; |
| 110 | let width = blocked |
| 111 | .iter() |
| 112 | .map(|item| item.path.len() + item.line.to_string().len() + 1) |
| 113 | .max() |
| 114 | .unwrap_or(0); |
| 115 | for item in blocked { |
| 116 | let place = format!("{}:{}", item.path, item.line); |
| 117 | lines.push(format!(" {place:<width$} {} (commit {})", item.label, short(&item.commit))); |
| 118 | } |
| 119 | lines.extend([ |
| 120 | String::new(), |
| 121 | "Take the secret out of the commit that adds it (git commit --amend, or".to_owned(), |
| 122 | "git rebase -i for an older commit), rotate it if it was ever real, and".to_owned(), |
| 123 | "push again.".to_owned(), |
| 124 | String::new(), |
| 125 | "If it is not a real secret (a test fixture), or you will rotate it later:".to_owned(), |
| 126 | format!(" - add {ALLOW_MARKER} in a comment on its line, or"), |
| 127 | ]); |
| 128 | let links: Vec<&str> = blocked.iter().filter_map(|item| item.allow_url.as_deref()).collect(); |
| 129 | match links.as_slice() { |
| 130 | [] => lines.push(" - ask a member of the workspace to allow it on the project's Security page.".to_owned()), |
| 131 | [one] => { |
| 132 | lines.push(format!(" - bypass it with a reason at {one}")); |
| 133 | } |
| 134 | many => { |
| 135 | lines.push(" - bypass each with a reason:".to_owned()); |
| 136 | for link in many { |
| 137 | lines.push(format!(" {link}")); |
| 138 | } |
| 139 | } |
| 140 | } |
| 141 | lines.push(" A bypass is recorded with your name and reason (or goes to an owner".to_owned()); |
| 142 | lines.push(" to approve, if your workspace asks), then the same push goes through.".to_owned()); |
| 143 | lines |
| 144 | } |
| 145 | |
| 146 | #[cfg(test)] |
| 147 | mod tests { |
| 148 | use super::*; |
| 149 | |
| 150 | fn key() -> String { |
| 151 | format!("AK{}", "IAZ7Q4N2XWLM3KDTRV") |
| 152 | } |
| 153 | |
| 154 | #[test] |
| 155 | fn only_added_lines_count() { |
| 156 | let old = format!("a\n{}\nb\n", key()); |
| 157 | let new = format!("a\n{}\nb\nc {}\n", key(), key()); |
| 158 | assert_eq!(added_lines(&old, &new), HashSet::from([4])); |
| 159 | let hits = scan_change("src/app.ts", Some(old.as_bytes()), new.as_bytes()); |
| 160 | assert_eq!(hits.len(), 1); |
| 161 | assert_eq!(hits[0].line, 4); |
| 162 | // A new file is scanned whole. |
| 163 | assert_eq!(scan_change("src/app.ts", None, new.as_bytes()).len(), 2); |
| 164 | // Binary files and lockfiles are not. |
| 165 | assert!(scan_change("bin/tool", None, &[0, 1, 2]).is_empty()); |
| 166 | assert!(scan_change("package-lock.json", None, new.as_bytes()).is_empty()); |
| 167 | } |
| 168 | |
| 169 | #[test] |
| 170 | fn custom_patterns_find_only_added_lines_too() { |
| 171 | let pattern = custom::compile(&custom::PatternSpec { |
| 172 | id: "pat_1".into(), |
| 173 | name: "Acme key".into(), |
| 174 | pattern: "acme_[0-9]{8}".into(), |
| 175 | before: None, |
| 176 | after: None, |
| 177 | }) |
| 178 | .unwrap(); |
| 179 | let old = "a = acme_12345678 |
| 180 | "; |
| 181 | let new = "a = acme_12345678 |
| 182 | b = acme_87654321 |
| 183 | "; |
| 184 | let hits = scan_change_custom("src/app.ts", Some(old.as_bytes()), new.as_bytes(), std::slice::from_ref(&pattern)); |
| 185 | assert_eq!(hits.len(), 1); |
| 186 | assert_eq!((hits[0].line, hits[0].value.as_str()), (2, "acme_87654321")); |
| 187 | assert_eq!(scan_change_custom("src/app.ts", None, new.as_bytes(), std::slice::from_ref(&pattern)).len(), 2); |
| 188 | assert!(scan_change_custom("yarn.lock", None, new.as_bytes(), &[pattern]).is_empty()); |
| 189 | } |
| 190 | |
| 191 | #[test] |
| 192 | fn the_refusal_names_the_file_line_kind_and_ways_forward() { |
| 193 | let blocked = vec![Blocked { |
| 194 | label: SecretKind::AwsAccessKey.label().into(), |
| 195 | path: "config/prod.env".into(), |
| 196 | line: 3, |
| 197 | commit: "4807077b296e6edbf410d55e72749d3e1170c291".into(), |
| 198 | allow_url: Some("https://g1t.sh/acme/rocket/security?finding=sec_1".into()), |
| 199 | }]; |
| 200 | assert_eq!(reason(&blocked), "secret found: config/prod.env:3 has an AWS access key"); |
| 201 | let text = explain(&blocked).join("\n"); |
| 202 | assert!(text.starts_with("g1t found a secret in this push, so nothing was pushed.")); |
| 203 | assert!(text.contains("config/prod.env:3 an AWS access key (commit 4807077)")); |
| 204 | assert!(text.contains("g1t:allow-secret")); |
| 205 | assert!(text.contains("bypass it with a reason at https://g1t.sh/acme/rocket/security?finding=sec_1")); |
| 206 | assert!(text.contains("recorded with your name and reason")); |
| 207 | } |
| 208 | |
| 209 | #[test] |
| 210 | fn several_secrets_are_listed_and_counted() { |
| 211 | let item = |path: &str, line| Blocked { |
| 212 | label: SecretKind::GithubToken.label().into(), |
| 213 | path: path.into(), |
| 214 | line, |
| 215 | commit: "c71546fcd893".into(), |
| 216 | allow_url: None, |
| 217 | }; |
| 218 | let blocked = vec![item("a.env", 1), item("src/deep/b.ts", 12)]; |
| 219 | assert_eq!(reason(&blocked), "2 secrets found, first a.env:1 (a GitHub token)"); |
| 220 | let text = explain(&blocked); |
| 221 | assert!(text[0].contains("2 secrets")); |
| 222 | assert!(text.iter().any(|line| line == " a.env:1 a GitHub token (commit c71546f)")); |
| 223 | assert!(text.iter().any(|line| line.contains("Security page"))); |
| 224 | } |
| 225 | } |