| 1 | //! Hiding credentials in text people and agents write about their work: a |
| 2 | //! session's prompts, tool calls and messages, its title and what it |
| 3 | //! reported. Unlike [`crate::secrets`], which must be sure before it stops |
| 4 | //! a push, this errs the other way: a word too many hidden costs nothing, |
| 5 | //! a key shown on a public page costs the key. |
| 6 | //! |
| 7 | //! [`redact`] replaces what looks like a credential with [`REDACTED`]: |
| 8 | //! every format [`crate::secrets`] knows, prefixed tokens of any length, |
| 9 | //! `NAME_TOKEN=value` and `password: value`, credential flags, bearer |
| 10 | //! headers, a URL's user and password and its signed query parameters, |
| 11 | //! JWTs, PEM private keys, and long random-looking strings. [`soften`] |
| 12 | //! takes swearing out of a title, so that how someone spoke to their own |
| 13 | //! agent is not what their team reads. |
| 14 | |
| 15 | use std::sync::LazyLock; |
| 16 | |
| 17 | use regex::{Captures, Regex}; |
| 18 | |
| 19 | /// What a credential is replaced with. |
| 20 | pub const REDACTED: &str = "[redacted]"; |
| 21 | |
| 22 | fn re(pattern: &str) -> Regex { |
| 23 | Regex::new(pattern).expect("a redaction pattern") |
| 24 | } |
| 25 | |
| 26 | /// A key or token by its issuer's prefix: `ghp_…`, `sk-ant-…`, `g1t_…`. |
| 27 | static PREFIXED: LazyLock<Regex> = LazyLock::new(|| { |
| 28 | re(r"\b(g1t|ghp|gho|ghu|ghs|ghr|github_pat|glpat|gloas|glrt|glptt|gldt|xox[abeprs]|sk-ant|sk-proj|sk-svcacct|sk-admin|sk|rk|whsec|npm|hf|pypi|dop_v1|shpat|shpss|sntrys|lin_api|figd)([-_])([A-Za-z0-9_-]{12,})") |
| 29 | }); |
| 30 | |
| 31 | /// `AKIA…`, `AIza…`, `SG.….…`: formats whose prefix needs no separator. |
| 32 | static BARE: LazyLock<Regex> = |
| 33 | LazyLock::new(|| re(r"\b(?:(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}|AIza[A-Za-z0-9_-]{35}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})\b")); |
| 34 | |
| 35 | /// `Authorization: Bearer …`, `Proxy-Authorization: Basic …`. |
| 36 | static AUTH_HEADER: LazyLock<Regex> = |
| 37 | LazyLock::new(|| re(r"(?i)\b((?:proxy-)?authorization\s*[:=]\s*(?:(?:bearer|basic|token|bot|digest)\s+)?)[^\s\x22'`,;]+")); |
| 38 | |
| 39 | /// `Bearer …` anywhere else. |
| 40 | static BEARER: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}")); |
| 41 | |
| 42 | /// `G1T_TOKEN=…`, `$env:API_KEY='…'`, `"password": "…"`, `client_secret: …`. |
| 43 | static ASSIGNED: LazyLock<Regex> = LazyLock::new(|| { |
| 44 | re(r#"(?i)([A-Za-z0-9_.-]*(?:token|secret|passw(?:or)?d|pwd|api[_-]?key|apikey|private[_-]?key|access[_-]?key|auth[_-]?key|signing[_-]?key|session[_-]?key|credentials?|webhook[_-]?url|dsn|connection[_-]?string)["']?\s*(?:=|:)\s*)("[^"\n]*"|'[^'\n]*'|`[^`\n]*`|[^\s,;&)}\]]+)"#) |
| 45 | }); |
| 46 | |
| 47 | /// `--token abc`, `--password=abc`, `--api-key abc`. |
| 48 | static FLAG: LazyLock<Regex> = LazyLock::new(|| { |
| 49 | re(r"(?i)(--(?:token|password|passwd|secret|api-key|apikey|access-token|auth-token|client-secret|private-key)(?:=|\s+))(\x22[^\x22]*\x22|'[^']*'|\S+)") |
| 50 | }); |
| 51 | |
| 52 | /// `curl -u name:password`. |
| 53 | static USER_FLAG: LazyLock<Regex> = LazyLock::new(|| re(r"((?:^|\s)(?:-u|--user)\s+[^\s:]+:)\S+")); |
| 54 | |
| 55 | /// `https://name:password@host`. |
| 56 | static USERINFO: LazyLock<Regex> = LazyLock::new(|| re(r"(?i)\b([a-z][a-z0-9+.-]*://)[^\s/:@]+:[^\s/@]+@")); |
| 57 | |
| 58 | /// `?token=…`, `&X-Amz-Signature=…`. |
| 59 | static QUERY: LazyLock<Regex> = LazyLock::new(|| { |
| 60 | re(r"(?i)([?&](?:access_token|refresh_token|id_token|token|key|api_key|apikey|secret|client_secret|sig|signature|password|code|x-amz-signature|x-amz-credential|x-amz-security-token)=)[^&\s#\x22'`]+") |
| 61 | }); |
| 62 | |
| 63 | /// A JSON Web Token. |
| 64 | static JWT: LazyLock<Regex> = LazyLock::new(|| re(r"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")); |
| 65 | |
| 66 | /// A PEM private key, to its end or to the end of the text. |
| 67 | static PEM: LazyLock<Regex> = |
| 68 | LazyLock::new(|| re(r"(?s)-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----.*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|\z)")); |
| 69 | |
| 70 | /// A long run that may be random: judged by [`random_looking`]. |
| 71 | static LONG: LazyLock<Regex> = LazyLock::new(|| re(r"[A-Za-z0-9+/_=-]{32,}")); |
| 72 | |
| 73 | /// A value that names a secret rather than holding one: `$TOKEN`, |
| 74 | /// `${{ secrets.X }}`, `process.env.X`, or one already hidden. |
| 75 | fn reference(value: &str) -> bool { |
| 76 | let value = value.trim_matches(['"', '\'', '`']); |
| 77 | value.is_empty() |
| 78 | || value.starts_with('$') |
| 79 | || value.starts_with('%') |
| 80 | || value.starts_with('<') |
| 81 | || value.contains("secrets.") |
| 82 | || value.contains("env.") |
| 83 | || value.contains("[redacted") |
| 84 | || value == "…" |
| 85 | } |
| 86 | |
| 87 | fn shannon(text: &str) -> f64 { |
| 88 | let mut counts = std::collections::HashMap::new(); |
| 89 | for c in text.chars() { |
| 90 | *counts.entry(c).or_insert(0usize) += 1; |
| 91 | } |
| 92 | let length = text.chars().count() as f64; |
| 93 | counts |
| 94 | .values() |
| 95 | .map(|count| { |
| 96 | let p = *count as f64 / length; |
| 97 | -p * p.log2() |
| 98 | }) |
| 99 | .sum() |
| 100 | } |
| 101 | |
| 102 | /// Upper case, lower case and digits mixed with the spread of a random |
| 103 | /// string: a key with no prefix. A commit (lowercase hex), a path, a |
| 104 | /// CamelCase name or a word run together does not look like this. |
| 105 | fn random_looking(run: &str) -> bool { |
| 106 | let digits = run.chars().filter(char::is_ascii_digit).count(); |
| 107 | run.chars().any(|c| c.is_ascii_uppercase()) |
| 108 | && run.chars().any(|c| c.is_ascii_lowercase()) |
| 109 | && digits >= 3 |
| 110 | && !run.contains("//") |
| 111 | && shannon(run) >= 4.3 |
| 112 | } |
| 113 | |
| 114 | fn has_letter_and_digit(body: &str) -> bool { |
| 115 | body.chars().any(|c| c.is_ascii_alphabetic()) && body.chars().any(|c| c.is_ascii_digit()) |
| 116 | } |
| 117 | |
| 118 | /// `text` with what looks like a credential replaced by [`REDACTED`]. |
| 119 | pub fn redact(text: &str) -> String { |
| 120 | if text.is_empty() { |
| 121 | return String::new(); |
| 122 | } |
| 123 | // Exact formats first, whatever surrounds them. |
| 124 | let mut out = text.to_owned(); |
| 125 | let mut values: Vec<String> = crate::secrets::scan_text(text).into_iter().map(|hit| hit.value).collect(); |
| 126 | values.sort_by_key(|value| std::cmp::Reverse(value.len())); |
| 127 | for value in values { |
| 128 | // A private key's value is its header and the start of its body. |
| 129 | let value = value.split('\n').next_back().unwrap_or(&value).to_owned(); |
| 130 | if value.len() >= 8 { |
| 131 | out = out.replace(&value, REDACTED); |
| 132 | } |
| 133 | } |
| 134 | let out = PEM.replace_all(&out, REDACTED); |
| 135 | let out = JWT.replace_all(&out, REDACTED); |
| 136 | let out = PREFIXED.replace_all(&out, |c: &Captures| { |
| 137 | if has_letter_and_digit(&c[3]) { format!("{}{}{REDACTED}", &c[1], &c[2]) } else { c[0].to_owned() } |
| 138 | }); |
| 139 | let out = BARE.replace_all(&out, REDACTED); |
| 140 | let out = AUTH_HEADER.replace_all(&out, |c: &Captures| { |
| 141 | let value = &c[0][c[1].len()..]; |
| 142 | if reference(value) { c[0].to_owned() } else { format!("{}{REDACTED}", &c[1]) } |
| 143 | }); |
| 144 | let out = BEARER.replace_all(&out, |c: &Captures| { |
| 145 | let value = &c[0][c[1].len()..]; |
| 146 | if reference(value) { c[0].to_owned() } else { format!("{}{REDACTED}", &c[1]) } |
| 147 | }); |
| 148 | let out = USERINFO.replace_all(&out, format!("${{1}}{REDACTED}@").as_str()); |
| 149 | let out = QUERY.replace_all(&out, format!("${{1}}{REDACTED}").as_str()); |
| 150 | let out = FLAG.replace_all(&out, |c: &Captures| { |
| 151 | if reference(&c[2]) { c[0].to_owned() } else { format!("{}{REDACTED}", &c[1]) } |
| 152 | }); |
| 153 | let out = USER_FLAG.replace_all(&out, format!("${{1}}{REDACTED}").as_str()); |
| 154 | let out = ASSIGNED.replace_all(&out, |c: &Captures| { |
| 155 | let value = &c[2]; |
| 156 | // `token: string` in code is a type, not a token. |
| 157 | let typed = matches!(value, "string" | "String" | "str" | "&str" | "number" | "bool" | "boolean" | "null" | "None" | "undefined" | "true" | "false" | "Option<String>"); |
| 158 | if reference(value) || typed { c[0].to_owned() } else { format!("{}{REDACTED}", &c[1]) } |
| 159 | }); |
| 160 | let out = LONG.replace_all(&out, |c: &Captures| { |
| 161 | if random_looking(&c[0]) { REDACTED.to_owned() } else { c[0].to_owned() } |
| 162 | }); |
| 163 | out.into_owned() |
| 164 | } |
| 165 | |
| 166 | /// Words that say how someone felt, not what they asked for. |
| 167 | const SWEARING: &[&str] = &[ |
| 168 | "fuck", "fucking", "fucked", "fucker", "fuckin", "motherfucker", "motherfucking", "shit", "shitty", "bullshit", "horseshit", |
| 169 | "damn", "damned", "dammit", "damnit", "goddamn", "goddamned", "goddammit", "wtf", "wth", "ffs", "stfu", "omfg", "asshole", |
| 170 | "bitch", "bitching", "crap", "crappy", "idiot", "idiotic", "moron", "moronic", "dumbass", "retard", "retarded", "bastard", |
| 171 | "pissed", "jfc", "fml", |
| 172 | ]; |
| 173 | |
| 174 | fn swearing(word: &str) -> bool { |
| 175 | let bare: String = word.chars().filter(|c| c.is_alphanumeric()).collect::<String>().to_lowercase(); |
| 176 | !bare.is_empty() && (SWEARING.contains(&bare.as_str()) || bare.starts_with("fuck") || bare.starts_with("shit")) |
| 177 | } |
| 178 | |
| 179 | /// A title as the team reads it: credentials hidden and swearing left out. |
| 180 | /// Empty when nothing else was said. |
| 181 | pub fn soften(title: &str) -> String { |
| 182 | let redacted = redact(title); |
| 183 | let mut words: Vec<&str> = Vec::new(); |
| 184 | for word in redacted.split_whitespace() { |
| 185 | if swearing(word) { |
| 186 | // Its punctuation stays with the sentence: "this, fucking thing" reads "this, thing". |
| 187 | continue; |
| 188 | } |
| 189 | words.push(word); |
| 190 | } |
| 191 | let text = words.join(" "); |
| 192 | let text = text.trim_matches(|c: char| c.is_whitespace() || matches!(c, ',' | ';' | ':' | '-')); |
| 193 | let mut chars = text.chars(); |
| 194 | match chars.next() { |
| 195 | // A title that lost its first word still starts as a sentence. |
| 196 | Some(first) if words.len() < redacted.split_whitespace().count() => first.to_uppercase().chain(chars).collect(), |
| 197 | Some(_) => text.to_owned(), |
| 198 | None => String::new(), |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | #[cfg(test)] |
| 203 | mod tests { |
| 204 | use super::*; |
| 205 | |
| 206 | fn hidden(text: &str, secret: &str) { |
| 207 | let out = redact(text); |
| 208 | assert!(!out.contains(secret), "{secret} shows in {out}"); |
| 209 | assert!(out.contains(REDACTED), "nothing hidden in {out}"); |
| 210 | } |
| 211 | |
| 212 | #[test] |
| 213 | fn tokens_by_their_prefix() { |
| 214 | hidden("source ~/.bashrc; curl -H 'x' g1t_0123456789abcdef0123456789abcdef01234567", "0123456789abcdef0123"); |
| 215 | hidden("use ghp_A1b2C3d4E5f6G7h8I9j0 please", "A1b2C3d4E5f6G7h8I9j0"); |
| 216 | hidden("key sk-ant-api03-abcDEF123456ghiJKL", "abcDEF123456ghiJKL"); |
| 217 | hidden("whsec_1234567890abcdefXYZ", "1234567890abcdefXYZ"); |
| 218 | hidden("AKIAIOSFODNN7EXAMPLE was it", "IOSFODNN7EXAMPLE"); |
| 219 | // Names that only share a prefix are left alone. |
| 220 | assert_eq!(redact("npm_config_user_agent is set"), "npm_config_user_agent is set"); |
| 221 | assert_eq!(redact("sk-learn-classifier"), "sk-learn-classifier"); |
| 222 | } |
| 223 | |
| 224 | #[test] |
| 225 | fn assignments_flags_and_headers() { |
| 226 | hidden("G1T_TOKEN=abc123def456 g1t whoami", "abc123def456"); |
| 227 | hidden("$env:G1T_TOKEN='abc123def456'; g1t whoami", "abc123def456"); |
| 228 | hidden(r#"{"password": "hunter2!"}"#, "hunter2"); |
| 229 | hidden("client_secret: s3cr3tvalue", "s3cr3tvalue"); |
| 230 | hidden("DATABASE_PASSWORD=\"correct horse\"", "correct horse"); |
| 231 | hidden("g1t auth login --token abc123", "abc123"); |
| 232 | hidden("x --password=pa55", "pa55"); |
| 233 | hidden("curl -u admin:pa55word https://x", "pa55word"); |
| 234 | hidden(r#"curl -H "Authorization: Bearer abc.def-123" https://api"#, "abc.def-123"); |
| 235 | hidden("Authorization: Basic dXNlcjpwYXNz", "dXNlcjpwYXNz"); |
| 236 | hidden("send it with bearer abcdefgh12345678", "abcdefgh12345678"); |
| 237 | hidden("X-Api-Key: 1234abcd5678", "1234abcd5678"); |
| 238 | } |
| 239 | |
| 240 | #[test] |
| 241 | fn references_and_types_are_not_secrets() { |
| 242 | let line = "TOKEN=$G1T_TOKEN; token: ${{ secrets.DEPLOY }}; password: process.env.PW; fn f(token: String)"; |
| 243 | assert_eq!(redact(line), line); |
| 244 | assert_eq!(redact(r#"-H "Authorization: Bearer …""#), r#"-H "Authorization: Bearer …""#); |
| 245 | } |
| 246 | |
| 247 | #[test] |
| 248 | fn urls_jwts_and_keys() { |
| 249 | hidden("git clone https://x-access-token:abc123@g1t.sh/a/b", "abc123"); |
| 250 | hidden("postgres://app:pa55@db:5432/app", "pa55"); |
| 251 | hidden("open https://x.com/cb?code=abc&state=1&token=t0k3n", "t0k3n"); |
| 252 | hidden("GET /file?X-Amz-Signature=deadbeef1234&x=1", "deadbeef1234"); |
| 253 | hidden("jwt eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U", "dozjgNryP4J3"); |
| 254 | let pem = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQ\n-----END OPENSSH PRIVATE KEY-----\nafter"; // g1t:allow-secret |
| 255 | let out = redact(pem); |
| 256 | assert!(!out.contains("b3BlbnNzaC1"), "{out}"); |
| 257 | assert!(out.ends_with("after")); |
| 258 | hidden("-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA0Z3VS5JJcds3xfn", "MIIEpAIBAAKCAQEA"); // g1t:allow-secret |
| 259 | } |
| 260 | |
| 261 | #[test] |
| 262 | fn random_strings_go_and_ordinary_ones_stay() { |
| 263 | hidden("the key is Xk9fQ2mZr7Lp4Wv8Ty3Nb6Hc1Jd5Ge0S", "Xk9fQ2mZr7Lp4Wv8Ty3Nb6Hc1Jd5Ge0S"); |
| 264 | for ordinary in [ |
| 265 | "git show 58def4040e93a1dcda5f388f55360f3cd46cd97e", |
| 266 | "services/work/src/coordination/sessions.rs", |
| 267 | "CoordinationOp::StartAgentSession and ListWorkSessionsArgs", |
| 268 | "MSYS_NO_PATHCONV=1 git show origin/40-g1t_upkeep:.g1t/upkeep.yml", |
| 269 | "Fix the merge queue: it re-tests every pull request after a push", |
| 270 | ] { |
| 271 | assert_eq!(redact(ordinary), ordinary); |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | #[test] |
| 276 | fn swearing_is_left_out_of_titles() { |
| 277 | assert_eq!(soften("Fix this fucking bug"), "Fix this bug"); |
| 278 | assert_eq!(soften("wtf, why is the PR page slow"), "Why is the PR page slow"); |
| 279 | assert_eq!(soften("Make the shitty button work"), "Make the button work"); |
| 280 | assert_eq!(soften("fuck"), ""); |
| 281 | assert_eq!(soften("Shell scripts and hello world"), "Shell scripts and hello world"); |
| 282 | assert_eq!(soften("Rotate G1T_TOKEN=abc123def456 now"), format!("Rotate G1T_TOKEN={REDACTED} now")); |
| 283 | } |
| 284 | } |