| 1 | //! What rules about commits look at, read from a pack: each commit's |
| 2 | //! message, addresses, parents and signature, and the files it adds, |
| 3 | //! changes or deletes with their sizes. A push's pack is read before it is |
| 4 | //! stored; a pull request's commits are fetched as a pack from its source |
| 5 | //! (`inspect_commits`), so both are read by the same code. |
| 6 | |
| 7 | use std::cell::Cell; |
| 8 | use std::collections::{HashMap, HashSet, VecDeque}; |
| 9 | |
| 10 | use futures_util::future::{join_all, try_join_all}; |
| 11 | use g1t_contracts::rules::{CommitFacts, FileChange, Signature}; |
| 12 | use g1t_scan::pack::{ObjectKind, Pack}; |
| 13 | use worker::Result; |
| 14 | |
| 15 | use crate::secret_scan::Objects; |
| 16 | use crate::store::GitRepo; |
| 17 | |
| 18 | /// Who registered each signing key (fingerprint to user id), and who |
| 19 | /// verified each address (to user id and username). |
| 20 | pub type Owners = (HashMap<String, String>, HashMap<String, (String, String)>); |
| 21 | |
| 22 | /// The most commits read for one ref. |
| 23 | pub const MAX_COMMITS: usize = 300; |
| 24 | /// The most files listed for one commit; more is not complete. |
| 25 | pub const MAX_FILES: usize = 1000; |
| 26 | /// The most of a message kept. |
| 27 | const MAX_MESSAGE: usize = 4096; |
| 28 | |
| 29 | /// A raw commit's headers and message. |
| 30 | #[derive(Debug, Default, PartialEq, Eq)] |
| 31 | pub struct CommitText { |
| 32 | pub tree: String, |
| 33 | pub parents: Vec<String>, |
| 34 | pub author_email: Option<String>, |
| 35 | pub committer_email: Option<String>, |
| 36 | pub message: String, |
| 37 | } |
| 38 | |
| 39 | fn email(value: &str) -> Option<String> { |
| 40 | let start = value.rfind('<')?; |
| 41 | let end = start + value[start..].find('>')?; |
| 42 | Some(value[start + 1..end].trim().to_owned()) |
| 43 | } |
| 44 | |
| 45 | /// Reads a raw commit object. |
| 46 | pub fn read_commit(data: &[u8]) -> CommitText { |
| 47 | let text = String::from_utf8_lossy(data); |
| 48 | let (headers, message) = text.split_once("\n\n").unwrap_or((&text, "")); |
| 49 | let mut commit = CommitText::default(); |
| 50 | for line in headers.split('\n') { |
| 51 | if let Some(tree) = line.strip_prefix("tree ") { |
| 52 | commit.tree = tree.trim().to_owned(); |
| 53 | } else if let Some(parent) = line.strip_prefix("parent ") { |
| 54 | commit.parents.push(parent.trim().to_owned()); |
| 55 | } else if let Some(author) = line.strip_prefix("author ") { |
| 56 | commit.author_email = email(author); |
| 57 | } else if let Some(committer) = line.strip_prefix("committer ") { |
| 58 | commit.committer_email = email(committer); |
| 59 | } |
| 60 | } |
| 61 | let mut end = message.len().min(MAX_MESSAGE); |
| 62 | while !message.is_char_boundary(end) { |
| 63 | end -= 1; |
| 64 | } |
| 65 | commit.message = message[..end].to_owned(); |
| 66 | commit |
| 67 | } |
| 68 | |
| 69 | /// The commits of the pack reachable from `tip` without leaving it, |
| 70 | /// newest first: what a push adds to a ref. `None` past `limit`. |
| 71 | pub fn added(pack: &Pack, tip: &str, limit: usize) -> Option<Vec<String>> { |
| 72 | let mut seen = HashSet::new(); |
| 73 | let mut queue = VecDeque::from([tip.to_owned()]); |
| 74 | let mut out = Vec::new(); |
| 75 | while let Some(id) = queue.pop_front() { |
| 76 | if !seen.insert(id.clone()) { |
| 77 | continue; |
| 78 | } |
| 79 | let Some((ObjectKind::Commit, data)) = pack.get(&id) else { |
| 80 | continue; |
| 81 | }; |
| 82 | out.push(id); |
| 83 | if out.len() > limit { |
| 84 | return None; |
| 85 | } |
| 86 | queue.extend(read_commit(data).parents); |
| 87 | } |
| 88 | Some(out) |
| 89 | } |
| 90 | |
| 91 | /// The commits of `ids` as default merge messages read them: message, |
| 92 | /// addresses and parents, from the pack alone. No files are listed and no |
| 93 | /// signature is decided, so nothing is read from the repository. |
| 94 | pub fn messages(pack: &Pack, ids: &[String]) -> Vec<CommitFacts> { |
| 95 | ids.iter() |
| 96 | .filter_map(|id| { |
| 97 | let Some((ObjectKind::Commit, data)) = pack.get(id) else { |
| 98 | return None; |
| 99 | }; |
| 100 | let commit = read_commit(data); |
| 101 | Some(CommitFacts { |
| 102 | sha: id.clone(), |
| 103 | message: commit.message, |
| 104 | author_email: commit.author_email, |
| 105 | committer_email: commit.committer_email, |
| 106 | parents: commit.parents.len() as u32, |
| 107 | signature: Signature::default(), |
| 108 | files: Vec::new(), |
| 109 | files_complete: false, |
| 110 | }) |
| 111 | }) |
| 112 | .collect() |
| 113 | } |
| 114 | |
| 115 | /// The files that differ between two trees, deletions included, with the |
| 116 | /// size of each new blob the pack holds. Whether the list is complete. |
| 117 | /// Each level of the trees is read at once. |
| 118 | async fn changed<R: GitRepo>(objects: &Objects<'_, R>, old_root: Option<String>, new_root: Option<String>) -> Result<(Vec<FileChange>, bool)> { |
| 119 | let mut files = Vec::new(); |
| 120 | let mut level: Vec<(String, Option<String>, Option<String>)> = vec![(String::new(), old_root, new_root)]; |
| 121 | while !level.is_empty() { |
| 122 | let read = try_join_all(level.iter().map(|(_, old, new)| async move { |
| 123 | let tree = async |id: &Option<String>| match id { |
| 124 | Some(id) => objects.tree(id).await, |
| 125 | None => Ok(Vec::new()), |
| 126 | }; |
| 127 | futures_util::future::try_join(tree(old), tree(new)).await |
| 128 | })) |
| 129 | .await?; |
| 130 | let mut next = Vec::new(); |
| 131 | for ((prefix, _, _), (old_items, new_items)) in level.into_iter().zip(read) { |
| 132 | for item in &new_items { |
| 133 | let before = old_items.iter().find(|entry| entry.name == item.name); |
| 134 | if before.is_some_and(|before| before.id == item.id && before.mode == item.mode) { |
| 135 | continue; |
| 136 | } |
| 137 | let path = format!("{prefix}{}", item.name); |
| 138 | if item.is_tree() { |
| 139 | next.push((format!("{path}/"), before.filter(|b| b.is_tree()).map(|b| b.id.clone()), Some(item.id.clone()))); |
| 140 | // A file replaced by a directory is deleted. |
| 141 | if before.is_some_and(|b| !b.is_tree()) { |
| 142 | files.push(FileChange { path: path.clone(), size: None, deleted: true }); |
| 143 | } |
| 144 | } else { |
| 145 | let size = match objects.pack.get(&item.id) { |
| 146 | Some((ObjectKind::Blob, data)) => Some(data.len() as u64), |
| 147 | _ => None, |
| 148 | }; |
| 149 | files.push(FileChange { path, size, deleted: false }); |
| 150 | if let Some(before) = before.filter(|b| b.is_tree()) { |
| 151 | next.push((format!("{prefix}{}/", item.name), Some(before.id.clone()), None)); |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | for item in &old_items { |
| 156 | if new_items.iter().any(|entry| entry.name == item.name) { |
| 157 | continue; |
| 158 | } |
| 159 | let path = format!("{prefix}{}", item.name); |
| 160 | if item.is_tree() { |
| 161 | next.push((format!("{path}/"), Some(item.id.clone()), None)); |
| 162 | } else { |
| 163 | files.push(FileChange { path, size: None, deleted: true }); |
| 164 | } |
| 165 | } |
| 166 | if files.len() > MAX_FILES { |
| 167 | files.truncate(MAX_FILES); |
| 168 | return Ok((files, false)); |
| 169 | } |
| 170 | } |
| 171 | level = next; |
| 172 | } |
| 173 | Ok((files, true)) |
| 174 | } |
| 175 | |
| 176 | /// One commit of the pack, read as rules look at it. `signature` is what |
| 177 | /// was made of its signature, when one was asked for. |
| 178 | pub async fn facts<R: GitRepo>(objects: &Objects<'_, R>, id: &str, signature: Option<Signature>) -> Result<Option<CommitFacts>> { |
| 179 | let Some((ObjectKind::Commit, data)) = objects.pack.get(id) else { |
| 180 | return Ok(None); |
| 181 | }; |
| 182 | let commit = read_commit(data); |
| 183 | let old_tree = match commit.parents.first() { |
| 184 | Some(parent) => objects.commit_tree(parent).await?, |
| 185 | None => None, |
| 186 | }; |
| 187 | let (files, files_complete) = changed(objects, old_tree, Some(commit.tree.clone())).await?; |
| 188 | Ok(Some(CommitFacts { |
| 189 | sha: id.to_owned(), |
| 190 | message: commit.message, |
| 191 | author_email: commit.author_email, |
| 192 | committer_email: commit.committer_email, |
| 193 | parents: commit.parents.len() as u32, |
| 194 | signature: signature.unwrap_or_default(), |
| 195 | files, |
| 196 | files_complete, |
| 197 | })) |
| 198 | } |
| 199 | |
| 200 | /// Whether `old` is in the history of `new`: a fast-forward. Walks the |
| 201 | /// pack's commits, then the repository's history from where it leaves it, |
| 202 | /// reading the histories from each place it leaves at once. |
| 203 | pub async fn contains<R: GitRepo>(pack: &Pack, repo: &R, new: &str, old: &str, depth: u32) -> Result<bool> { |
| 204 | if new == old { |
| 205 | return Ok(true); |
| 206 | } |
| 207 | let mut seen = HashSet::new(); |
| 208 | let mut queue = VecDeque::from([new.to_owned()]); |
| 209 | let mut boundary = Vec::new(); |
| 210 | while let Some(id) = queue.pop_front() { |
| 211 | if id == old { |
| 212 | return Ok(true); |
| 213 | } |
| 214 | if !seen.insert(id.clone()) || seen.len() > 5000 { |
| 215 | continue; |
| 216 | } |
| 217 | match pack.get(&id) { |
| 218 | Some((ObjectKind::Commit, data)) => queue.extend(read_commit(data).parents), |
| 219 | _ => boundary.push(id), |
| 220 | } |
| 221 | } |
| 222 | let histories = join_all(boundary.iter().take(20).map(|start| repo.log(start, depth))).await; |
| 223 | if found_in(&histories, old) { |
| 224 | return Ok(true); |
| 225 | } |
| 226 | // Merges: the history is first-parent only, so look along the second |
| 227 | // parents it names too. |
| 228 | let seconds: Vec<&String> = histories |
| 229 | .iter() |
| 230 | .flatten() |
| 231 | .flat_map(|history| history.iter().filter(|commit| commit.parents.len() > 1).take(10)) |
| 232 | .flat_map(|commit| commit.parents.iter().skip(1)) |
| 233 | .collect(); |
| 234 | if seconds.iter().any(|parent| *parent == old) { |
| 235 | return Ok(true); |
| 236 | } |
| 237 | let further = join_all(seconds.iter().map(|parent| repo.log(parent, depth))).await; |
| 238 | if found_in(&further, old) { |
| 239 | return Ok(true); |
| 240 | } |
| 241 | // Not found: a history that could not be read may have held it. |
| 242 | for history in histories.into_iter().chain(further) { |
| 243 | history?; |
| 244 | } |
| 245 | Ok(false) |
| 246 | } |
| 247 | |
| 248 | /// Whether any history read holds `old`. |
| 249 | fn found_in(histories: &[Result<Vec<g1t_contracts::repos::Commit>>], old: &str) -> bool { |
| 250 | histories.iter().flatten().any(|history| history.iter().any(|commit| commit.hash == old)) |
| 251 | } |
| 252 | |
| 253 | /// The signature fingerprints and committer addresses of commits, for |
| 254 | /// looking up who owns them. |
| 255 | pub fn signing_facts(pack: &Pack, ids: &[String]) -> (Vec<String>, Vec<String>) { |
| 256 | let mut fingerprints = Vec::new(); |
| 257 | let mut emails = Vec::new(); |
| 258 | for id in ids { |
| 259 | let Some((ObjectKind::Commit, data)) = pack.get(id) else { continue }; |
| 260 | if let Some(fingerprint) = crate::signatures::fingerprint(data) |
| 261 | && !fingerprints.contains(&fingerprint) |
| 262 | { |
| 263 | fingerprints.push(fingerprint); |
| 264 | if let Some(email) = read_commit(data).committer_email.map(|email| email.to_lowercase()) |
| 265 | && !emails.contains(&email) |
| 266 | { |
| 267 | emails.push(email); |
| 268 | } |
| 269 | } |
| 270 | } |
| 271 | (fingerprints, emails) |
| 272 | } |
| 273 | |
| 274 | /// Every commit of `ids` read, with its signature decided against who |
| 275 | /// owns the keys and addresses (`owners`, when signatures matter). The |
| 276 | /// commits are read at once. |
| 277 | pub async fn read_all<R: GitRepo>( |
| 278 | pack: &Pack, |
| 279 | repo: &R, |
| 280 | ids: &[String], |
| 281 | owners: Option<&Owners>, |
| 282 | ) -> Result<Vec<CommitFacts>> { |
| 283 | let objects = Objects { pack, repo, reads: Cell::new(0) }; |
| 284 | let objects = &objects; |
| 285 | let read = try_join_all(ids.iter().map(|id| async move { |
| 286 | let signature = owners.and_then(|(keys, emails)| { |
| 287 | let (_, data) = pack.get(id)?; |
| 288 | let committer = read_commit(data).committer_email; |
| 289 | Some(crate::signatures::decide(data, committer.as_deref(), keys, emails)) |
| 290 | }); |
| 291 | facts(objects, id, signature).await |
| 292 | })) |
| 293 | .await?; |
| 294 | Ok(read.into_iter().flatten().collect()) |
| 295 | } |
| 296 | |
| 297 | #[cfg(test)] |
| 298 | mod tests { |
| 299 | use super::*; |
| 300 | |
| 301 | #[test] |
| 302 | fn a_commit_reads_its_parents_addresses_and_message() { |
| 303 | let raw = b"tree aaaa\nparent bbbb\nparent cccc\nauthor Ada Lovelace <ada@acme.com> 1 +0000\ncommitter G <noreply@g1t.sh> 1 +0000\ngpgsig -----BEGIN SSH SIGNATURE-----\n abc\n -----END SSH SIGNATURE-----\n\nfeat: rules\n\nWith a body.\n"; |
| 304 | let commit = read_commit(raw); |
| 305 | assert_eq!(commit.tree, "aaaa"); |
| 306 | assert_eq!(commit.parents, vec!["bbbb", "cccc"]); |
| 307 | assert_eq!(commit.author_email.as_deref(), Some("ada@acme.com")); |
| 308 | assert_eq!(commit.committer_email.as_deref(), Some("noreply@g1t.sh")); |
| 309 | assert_eq!(commit.message, "feat: rules\n\nWith a body.\n"); |
| 310 | } |
| 311 | |
| 312 | #[test] |
| 313 | fn a_long_message_is_cut_on_a_character() { |
| 314 | let message = "é".repeat(5000); |
| 315 | let raw = format!("tree a\n\n{message}"); |
| 316 | assert!(read_commit(raw.as_bytes()).message.len() <= MAX_MESSAGE); |
| 317 | } |
| 318 | |
| 319 | #[test] |
| 320 | fn the_commits_a_push_adds_are_those_its_pack_holds() { |
| 321 | use g1t_scan::pack::write_pack; |
| 322 | let first = b"tree t\nauthor A <a@x> 1 +0000\ncommitter A <a@x> 1 +0000\n\none\n".to_vec(); |
| 323 | let first_id = g1t_scan::pack::object_id(ObjectKind::Commit, &first); |
| 324 | let second = format!("tree t\nparent {first_id}\nparent {}\nauthor A <a@x> 1 +0000\ncommitter A <a@x> 1 +0000\n\ntwo\n", "f".repeat(40)).into_bytes(); |
| 325 | let second_id = g1t_scan::pack::object_id(ObjectKind::Commit, &second); |
| 326 | let pack = Pack::parse(&write_pack(&[(ObjectKind::Commit, first), (ObjectKind::Commit, second)])).unwrap(); |
| 327 | assert_eq!(added(&pack, &second_id, 10), Some(vec![second_id.clone(), first_id.clone()])); |
| 328 | assert_eq!(added(&pack, &second_id, 1), None, "past the limit"); |
| 329 | assert_eq!(added(&pack, &"0".repeat(40), 10), Some(Vec::new()), "a tip the pack does not hold adds nothing"); |
| 330 | } |
| 331 | |
| 332 | #[test] |
| 333 | fn messages_are_read_from_the_pack_alone() { |
| 334 | use g1t_scan::pack::write_pack; |
| 335 | let first = b"tree t |
| 336 | author Ada <ada@x> 1 +0000 |
| 337 | committer A <a@x> 1 +0000 |
| 338 | |
| 339 | feat: one |
| 340 | ".to_vec(); |
| 341 | let first_id = g1t_scan::pack::object_id(ObjectKind::Commit, &first); |
| 342 | let merge = format!("tree t |
| 343 | parent {first_id} |
| 344 | parent {} |
| 345 | author B <b@x> 1 +0000 |
| 346 | committer B <b@x> 1 +0000 |
| 347 | |
| 348 | Merge main |
| 349 | ", "f".repeat(40)).into_bytes(); |
| 350 | let merge_id = g1t_scan::pack::object_id(ObjectKind::Commit, &merge); |
| 351 | let pack = Pack::parse(&write_pack(&[(ObjectKind::Commit, first), (ObjectKind::Commit, merge)])).unwrap(); |
| 352 | let ids = added(&pack, &merge_id, 10).unwrap(); |
| 353 | let read = messages(&pack, &ids); |
| 354 | assert_eq!(read.iter().map(|commit| commit.message.as_str()).collect::<Vec<_>>(), ["Merge main |
| 355 | ", "feat: one |
| 356 | "]); |
| 357 | assert_eq!(read[0].parents, 2); |
| 358 | assert_eq!(read[1].author_email.as_deref(), Some("ada@x")); |
| 359 | assert!(read.iter().all(|commit| commit.files.is_empty() && commit.signature == Signature::Unsigned)); |
| 360 | // An id the pack does not hold is left out. |
| 361 | assert_eq!(messages(&pack, &["0".repeat(40)]).len(), 0); |
| 362 | } |
| 363 | |
| 364 | /// Histories by where they start; a start missing from it fails. |
| 365 | struct Histories(HashMap<String, Vec<g1t_contracts::repos::Commit>>); |
| 366 | |
| 367 | impl GitRepo for Histories { |
| 368 | async fn access(&self, _scope: crate::store::Scope) -> Result<g1t_contracts::repos::GitAccess> { |
| 369 | unimplemented!() |
| 370 | } |
| 371 | async fn branches(&self) -> Result<Vec<g1t_contracts::repos::Branch>> { |
| 372 | Ok(Vec::new()) |
| 373 | } |
| 374 | async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<g1t_contracts::repos::Commit>> { |
| 375 | self.0.get(git_ref).cloned().ok_or_else(|| worker::Error::RustError(format!("no history from {git_ref}"))) |
| 376 | } |
| 377 | async fn parents(&self, _commit_hash: &str) -> Result<Option<Vec<String>>> { |
| 378 | Ok(None) |
| 379 | } |
| 380 | async fn read_tree(&self, _tree_hash: &str) -> Result<Option<Vec<g1t_contracts::repos::TreeEntry>>> { |
| 381 | Ok(None) |
| 382 | } |
| 383 | async fn read_blob(&self, _blob_hash: &str) -> Result<Option<Vec<u8>>> { |
| 384 | Ok(None) |
| 385 | } |
| 386 | async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> { |
| 387 | Ok(None) |
| 388 | } |
| 389 | async fn fork(&self, _target_key: &str, _default_branch_only: bool) -> Result<()> { |
| 390 | Ok(()) |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | fn run<F: std::future::Future>(future: F) -> F::Output { |
| 395 | let waker = std::task::Waker::noop(); |
| 396 | match std::pin::pin!(future).as_mut().poll(&mut std::task::Context::from_waker(waker)) { |
| 397 | std::task::Poll::Ready(output) => output, |
| 398 | std::task::Poll::Pending => panic!("the fake store never waits"), |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | fn at(hash: &str, parents: &[&str]) -> g1t_contracts::repos::Commit { |
| 403 | g1t_contracts::repos::Commit { |
| 404 | hash: hash.to_owned(), |
| 405 | tree_hash: String::new(), |
| 406 | message: String::new(), |
| 407 | author: g1t_contracts::repos::Signature { name: "A".into(), email: "a@x".into() }, |
| 408 | parents: parents.iter().map(|parent| (*parent).to_owned()).collect(), |
| 409 | authored_at: String::new(), |
| 410 | } |
| 411 | } |
| 412 | |
| 413 | #[test] |
| 414 | fn a_fast_forward_is_found_along_the_history_and_its_merges() { |
| 415 | use g1t_scan::pack::write_pack; |
| 416 | // The push brings one commit on top of `base`; `base` merged `side`, |
| 417 | // whose history holds `old`. |
| 418 | let (base, side, old) = ("b".repeat(40), "5".repeat(40), "0".repeat(40)); |
| 419 | let tip = format!("tree t |
| 420 | parent {base} |
| 421 | author A <a@x> 1 +0000 |
| 422 | committer A <a@x> 1 +0000 |
| 423 | |
| 424 | tip |
| 425 | ").into_bytes(); |
| 426 | let tip_id = g1t_scan::pack::object_id(ObjectKind::Commit, &tip); |
| 427 | let pack = Pack::parse(&write_pack(&[(ObjectKind::Commit, tip)])).unwrap(); |
| 428 | let mut histories = HashMap::new(); |
| 429 | histories.insert(base.clone(), vec![at(&base, &["1".repeat(40).as_str(), side.as_str()])]); |
| 430 | histories.insert(side.clone(), vec![at(&side, &[]), at(&old, &[])]); |
| 431 | let repo = Histories(histories); |
| 432 | assert!(run(contains(&pack, &repo, &tip_id, &old, 100)).unwrap()); |
| 433 | assert!(run(contains(&pack, &repo, &tip_id, &side, 100)).unwrap(), "a second parent itself"); |
| 434 | assert!(!run(contains(&pack, &repo, &tip_id, &"9".repeat(40), 100)).unwrap()); |
| 435 | // A history that cannot be read does not hide one found elsewhere, |
| 436 | // and is an error only when nothing was found. |
| 437 | let mut histories = repo.0; |
| 438 | histories.remove(&side); |
| 439 | let repo = Histories(histories); |
| 440 | assert!(run(contains(&pack, &repo, &tip_id, &side, 100)).unwrap()); |
| 441 | assert!(run(contains(&pack, &repo, &tip_id, &old, 100)).is_err()); |
| 442 | } |
| 443 | } |