| 1 | //! Listing a repository's files for services that index it, such as |
| 2 | //! search: every file on a branch, the files two commits differ in, and |
| 3 | //! the text of many blobs at once. |
| 4 | //! |
| 5 | //! Trees are read a level at a time, all of a level at once, and identical |
| 6 | //! subtrees are skipped by hash, so a push costs what it changed rather |
| 7 | //! than what the repository holds. Directories the caller names (vendored |
| 8 | //! code, build output) are never read at all. |
| 9 | |
| 10 | use std::collections::{BTreeMap, BTreeSet}; |
| 11 | |
| 12 | use futures_util::future::{try_join, try_join_all}; |
| 13 | use g1t_contracts::repos::{BlobText, EntryKind, FileEntry, FileList, MAX_LISTED_FILES, MAX_READ_BLOBS, TreeEntry}; |
| 14 | use worker::Result; |
| 15 | |
| 16 | use crate::store::GitRepo; |
| 17 | |
| 18 | /// Whether an entry is a file whose text can be read: a blob, executable |
| 19 | /// or not. Symlinks and submodules are not. |
| 20 | fn is_file(entry: &TreeEntry) -> bool { |
| 21 | matches!(entry.kind, EntryKind::Blob | EntryKind::Exec) |
| 22 | } |
| 23 | |
| 24 | async fn entries<R: GitRepo>(repo: &R, tree: Option<&str>) -> Result<BTreeMap<String, TreeEntry>> { |
| 25 | let Some(tree) = tree else { |
| 26 | return Ok(BTreeMap::new()); |
| 27 | }; |
| 28 | Ok(repo |
| 29 | .read_tree(tree) |
| 30 | .await? |
| 31 | .unwrap_or_default() |
| 32 | .into_iter() |
| 33 | .map(|entry| (entry.name.clone(), entry)) |
| 34 | .collect()) |
| 35 | } |
| 36 | |
| 37 | /// The files that differ between two trees, a level at a time, with |
| 38 | /// `hash` null for a file the newer tree no longer has. With no `old`, |
| 39 | /// every file of `new`. Directories named in `skip` are not entered. |
| 40 | pub async fn changed<R: GitRepo>( |
| 41 | repo: &R, |
| 42 | old: Option<&str>, |
| 43 | new: &str, |
| 44 | skip: &[String], |
| 45 | limit: u32, |
| 46 | ) -> Result<(Vec<FileEntry>, bool)> { |
| 47 | changed_with(repo, old, new, skip, limit, false).await |
| 48 | } |
| 49 | |
| 50 | /// [`changed`], with symlinks among the files when `links` is set. |
| 51 | pub async fn changed_with<R: GitRepo>( |
| 52 | repo: &R, |
| 53 | old: Option<&str>, |
| 54 | new: &str, |
| 55 | skip: &[String], |
| 56 | limit: u32, |
| 57 | links: bool, |
| 58 | ) -> Result<(Vec<FileEntry>, bool)> { |
| 59 | let listed = |entry: &TreeEntry| is_file(entry) || (links && entry.kind == EntryKind::Symlink); |
| 60 | let limit = limit.clamp(1, MAX_LISTED_FILES) as usize; |
| 61 | let skipped = |name: &str| skip.iter().any(|dir| dir.eq_ignore_ascii_case(name)); |
| 62 | let mut files = Vec::new(); |
| 63 | let mut level = vec![(String::new(), old.map(str::to_owned), Some(new.to_owned()))]; |
| 64 | while !level.is_empty() { |
| 65 | let read = try_join_all(level.iter().map(|(_, old_tree, new_tree)| async move { |
| 66 | try_join(entries(repo, old_tree.as_deref()), entries(repo, new_tree.as_deref())).await |
| 67 | })) |
| 68 | .await?; |
| 69 | let mut next = Vec::new(); |
| 70 | for ((prefix, _, _), (before, after)) in level.iter().zip(read) { |
| 71 | let names: BTreeSet<&String> = before.keys().chain(after.keys()).collect(); |
| 72 | for name in names { |
| 73 | let (was, now) = (before.get(name), after.get(name)); |
| 74 | if was.map(|entry| (&entry.hash, entry.kind)) == now.map(|entry| (&entry.hash, entry.kind)) { |
| 75 | continue; |
| 76 | } |
| 77 | let path = format!("{prefix}{name}"); |
| 78 | let dir = |entry: Option<&TreeEntry>| { |
| 79 | entry.filter(|entry| entry.kind == EntryKind::Tree).map(|entry| entry.hash.clone()) |
| 80 | }; |
| 81 | let (old_dir, new_dir) = (dir(was), dir(now)); |
| 82 | if (old_dir.is_some() || new_dir.is_some()) && !skipped(name) { |
| 83 | next.push((format!("{path}/"), old_dir, new_dir)); |
| 84 | } |
| 85 | let (old_file, new_file) = (was.filter(|e| listed(e)), now.filter(|e| listed(e))); |
| 86 | if old_file.is_none() && new_file.is_none() { |
| 87 | continue; |
| 88 | } |
| 89 | if files.len() >= limit { |
| 90 | return Ok((files, true)); |
| 91 | } |
| 92 | files.push(FileEntry { |
| 93 | path, |
| 94 | hash: new_file.map(|entry| entry.hash.clone()), |
| 95 | kind: new_file.map(|entry| entry.kind), |
| 96 | }); |
| 97 | } |
| 98 | } |
| 99 | level = next; |
| 100 | } |
| 101 | Ok((files, false)) |
| 102 | } |
| 103 | |
| 104 | /// The commit a branch or commit names, and its tree. |
| 105 | pub async fn resolve<R: GitRepo>(repo: &R, git_ref: &str) -> Result<Option<(String, String)>> { |
| 106 | Ok(repo |
| 107 | .log(git_ref, 1) |
| 108 | .await? |
| 109 | .into_iter() |
| 110 | .next() |
| 111 | .map(|commit| (commit.hash, commit.tree_hash))) |
| 112 | } |
| 113 | |
| 114 | /// Every file on `git_ref`, or what changed from `base` to it; symlinks |
| 115 | /// too with `links`. |
| 116 | pub async fn list<R: GitRepo>( |
| 117 | repo: &R, |
| 118 | base: Option<&str>, |
| 119 | head: &str, |
| 120 | skip: &[String], |
| 121 | limit: u32, |
| 122 | links: bool, |
| 123 | ) -> Result<FileList> { |
| 124 | let Some(head_commit) = repo.log(head, 1).await?.into_iter().next() else { |
| 125 | return Ok(FileList::default()); |
| 126 | }; |
| 127 | let (commit, tree, date) = (head_commit.hash, head_commit.tree_hash, head_commit.authored_at); |
| 128 | let base_tree = match base { |
| 129 | Some(base) => resolve(repo, base).await?.map(|(_, tree)| tree), |
| 130 | None => None, |
| 131 | }; |
| 132 | let (files, truncated) = changed_with(repo, base_tree.as_deref(), &tree, skip, limit, links).await?; |
| 133 | Ok(FileList { |
| 134 | commit: Some(commit), |
| 135 | date: Some(date), |
| 136 | files, |
| 137 | truncated, |
| 138 | }) |
| 139 | } |
| 140 | |
| 141 | /// The text of each blob asked for, in order: none for one that is |
| 142 | /// missing, larger than `max_bytes`, or binary. |
| 143 | pub async fn read<R: GitRepo>(repo: &R, hashes: &[String], max_bytes: u32) -> Result<Vec<BlobText>> { |
| 144 | let hashes: Vec<&String> = hashes.iter().take(MAX_READ_BLOBS).collect(); |
| 145 | let mut out = Vec::with_capacity(hashes.len()); |
| 146 | // A few at a time: each read is a round trip, and a batch of large |
| 147 | // blobs held at once would not fit in memory. |
| 148 | for group in hashes.chunks(8) { |
| 149 | let read = try_join_all(group.iter().map(|hash| repo.read_blob(hash))).await?; |
| 150 | for (hash, bytes) in group.iter().zip(read) { |
| 151 | let size = bytes.as_ref().map_or(0, |bytes| bytes.len() as u64); |
| 152 | let text = bytes |
| 153 | .filter(|bytes| bytes.len() <= max_bytes as usize && !bytes.contains(&0)) |
| 154 | .and_then(|bytes| String::from_utf8(bytes).ok()); |
| 155 | out.push(BlobText { |
| 156 | hash: (*hash).clone(), |
| 157 | size, |
| 158 | text, |
| 159 | }); |
| 160 | } |
| 161 | } |
| 162 | Ok(out) |
| 163 | } |
| 164 | |
| 165 | #[cfg(test)] |
| 166 | mod tests { |
| 167 | use std::collections::HashMap; |
| 168 | use std::future::Future; |
| 169 | use std::pin::pin; |
| 170 | use std::task::{Context, Poll, Waker}; |
| 171 | |
| 172 | use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature}; |
| 173 | |
| 174 | use super::*; |
| 175 | use crate::store::Scope; |
| 176 | |
| 177 | fn run<F: Future>(future: F) -> F::Output { |
| 178 | match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) { |
| 179 | Poll::Ready(output) => output, |
| 180 | Poll::Pending => panic!("the fake store never waits"), |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | #[derive(Default)] |
| 185 | struct Fake { |
| 186 | trees: HashMap<String, Vec<TreeEntry>>, |
| 187 | blobs: HashMap<String, Vec<u8>>, |
| 188 | commits: HashMap<String, Commit>, |
| 189 | } |
| 190 | |
| 191 | impl GitRepo for Fake { |
| 192 | async fn access(&self, _scope: Scope) -> Result<GitAccess> { |
| 193 | unimplemented!() |
| 194 | } |
| 195 | async fn branches(&self) -> Result<Vec<Branch>> { |
| 196 | Ok(Vec::new()) |
| 197 | } |
| 198 | async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> { |
| 199 | Ok(self.commits.get(git_ref).cloned().into_iter().collect()) |
| 200 | } |
| 201 | async fn parents(&self, _commit_hash: &str) -> Result<Option<Vec<String>>> { |
| 202 | Ok(None) |
| 203 | } |
| 204 | async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> { |
| 205 | Ok(self.trees.get(tree_hash).cloned()) |
| 206 | } |
| 207 | async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> { |
| 208 | Ok(self.blobs.get(blob_hash).cloned()) |
| 209 | } |
| 210 | async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> { |
| 211 | Ok(None) |
| 212 | } |
| 213 | async fn fork(&self, _target_key: &str) -> Result<()> { |
| 214 | Ok(()) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | fn entry(name: &str, hash: &str, kind: EntryKind) -> TreeEntry { |
| 219 | TreeEntry { |
| 220 | name: name.into(), |
| 221 | hash: hash.into(), |
| 222 | kind, |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | fn commit(hash: &str, tree: &str) -> Commit { |
| 227 | Commit { |
| 228 | hash: hash.into(), |
| 229 | tree_hash: tree.into(), |
| 230 | message: String::new(), |
| 231 | author: Signature { name: String::new(), email: String::new() }, |
| 232 | parents: Vec::new(), |
| 233 | authored_at: String::new(), |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | /// Two commits: c1 has README, src/a.rs, node_modules/x.js; c2 changes |
| 238 | /// src/a.rs, deletes README, adds src/b.rs and a symlink. |
| 239 | fn repo() -> Fake { |
| 240 | let mut fake = Fake::default(); |
| 241 | fake.trees.insert("src1".into(), vec![entry("a.rs", "a1", EntryKind::Blob)]); |
| 242 | fake.trees.insert("nm".into(), vec![entry("x.js", "x1", EntryKind::Blob)]); |
| 243 | fake.trees.insert( |
| 244 | "root1".into(), |
| 245 | vec![ |
| 246 | entry("README.md", "r1", EntryKind::Blob), |
| 247 | entry("src", "src1", EntryKind::Tree), |
| 248 | entry("node_modules", "nm", EntryKind::Tree), |
| 249 | ], |
| 250 | ); |
| 251 | fake.trees.insert( |
| 252 | "src2".into(), |
| 253 | vec![entry("a.rs", "a2", EntryKind::Blob), entry("b.rs", "b1", EntryKind::Exec)], |
| 254 | ); |
| 255 | fake.trees.insert( |
| 256 | "root2".into(), |
| 257 | vec![ |
| 258 | entry("src", "src2", EntryKind::Tree), |
| 259 | entry("node_modules", "nm", EntryKind::Tree), |
| 260 | entry("link", "l1", EntryKind::Symlink), |
| 261 | ], |
| 262 | ); |
| 263 | fake.commits.insert("c1".into(), commit("c1", "root1")); |
| 264 | fake.commits.insert("c2".into(), commit("c2", "root2")); |
| 265 | fake.commits.insert("main".into(), commit("c2", "root2")); |
| 266 | fake |
| 267 | } |
| 268 | |
| 269 | fn paths(list: &FileList) -> Vec<(String, Option<String>)> { |
| 270 | list.files.iter().map(|f| (f.path.clone(), f.hash.clone())).collect() |
| 271 | } |
| 272 | |
| 273 | #[test] |
| 274 | fn lists_every_file_but_skipped_directories() { |
| 275 | let fake = repo(); |
| 276 | let listed = run(list(&fake, None, "c1", &["node_modules".into()], 100, false)).unwrap(); |
| 277 | assert_eq!(listed.commit.as_deref(), Some("c1")); |
| 278 | assert_eq!( |
| 279 | paths(&listed), |
| 280 | vec![("README.md".into(), Some("r1".into())), ("src/a.rs".into(), Some("a1".into()))] |
| 281 | ); |
| 282 | } |
| 283 | |
| 284 | #[test] |
| 285 | fn lists_only_what_changed() { |
| 286 | let fake = repo(); |
| 287 | let listed = run(list(&fake, Some("c1"), "c2", &[], 100, false)).unwrap(); |
| 288 | assert_eq!( |
| 289 | paths(&listed), |
| 290 | vec![ |
| 291 | ("README.md".into(), None), |
| 292 | ("src/a.rs".into(), Some("a2".into())), |
| 293 | ("src/b.rs".into(), Some("b1".into())), |
| 294 | ] |
| 295 | ); |
| 296 | assert!(!listed.truncated); |
| 297 | } |
| 298 | |
| 299 | #[test] |
| 300 | fn an_archive_listing_has_symlinks_and_each_files_kind() { |
| 301 | let fake = repo(); |
| 302 | let listed = run(list(&fake, None, "c2", &[], 100, true)).unwrap(); |
| 303 | let kinds: Vec<(&str, Option<EntryKind>)> = listed.files.iter().map(|f| (f.path.as_str(), f.kind)).collect(); |
| 304 | assert_eq!( |
| 305 | kinds, |
| 306 | vec![ |
| 307 | ("link", Some(EntryKind::Symlink)), |
| 308 | ("node_modules/x.js", Some(EntryKind::Blob)), |
| 309 | ("src/a.rs", Some(EntryKind::Blob)), |
| 310 | ("src/b.rs", Some(EntryKind::Exec)), |
| 311 | ] |
| 312 | ); |
| 313 | assert_eq!(listed.date.as_deref(), Some("")); |
| 314 | // Without links, as search and the sidebar read it. |
| 315 | let plain = run(list(&fake, None, "c2", &[], 100, false)).unwrap(); |
| 316 | assert!(plain.files.iter().all(|f| f.path != "link")); |
| 317 | } |
| 318 | |
| 319 | #[test] |
| 320 | fn says_when_the_list_was_cut_short() { |
| 321 | let fake = repo(); |
| 322 | let listed = run(list(&fake, Some("c1"), "c2", &[], 2, false)).unwrap(); |
| 323 | assert_eq!(listed.files.len(), 2); |
| 324 | assert!(listed.truncated); |
| 325 | } |
| 326 | |
| 327 | #[test] |
| 328 | fn reads_text_but_not_binaries_or_large_blobs() { |
| 329 | let mut fake = repo(); |
| 330 | fake.blobs.insert("t".into(), b"fn main() {}\n".to_vec()); |
| 331 | fake.blobs.insert("bin".into(), vec![0, 1, 2]); |
| 332 | fake.blobs.insert("big".into(), vec![b'a'; 64]); |
| 333 | let read = run(read(&fake, &["t".into(), "bin".into(), "big".into(), "gone".into()], 32)).unwrap(); |
| 334 | let texts: Vec<Option<&str>> = read.iter().map(|b| b.text.as_deref()).collect(); |
| 335 | assert_eq!(texts, vec![Some("fn main() {}\n"), None, None, None]); |
| 336 | assert_eq!(read[2].size, 64); |
| 337 | } |
| 338 | } |