Sign in

pulls/pr_01m47d24b0e6n91zwymwxg0vpx

public

pr_01m47d24b0e6n91zwymwxg0vpx/services/repos/src/listing.rs

304 lines11,185 bytesCodeBlame
1//! Listing a repository's files for services that index it, such as
2//! search: every file on a branch, the files two commits differ in, and
3//! the text of many blobs at once.
4//!
5//! Trees are read a level at a time, all of a level at once, and identical
6//! subtrees are skipped by hash, so a push costs what it changed rather
7//! than what the repository holds. Directories the caller names (vendored
8//! code, build output) are never read at all.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use futures_util::future::{try_join, try_join_all};
13use g1t_contracts::repos::{BlobText, EntryKind, FileEntry, FileList, MAX_LISTED_FILES, MAX_READ_BLOBS, TreeEntry};
14use worker::Result;
15
16use crate::store::GitRepo;
17
18/// Whether an entry is a file whose text can be read: a blob, executable
19/// or not. Symlinks and submodules are not.
20fn is_file(entry: &TreeEntry) -> bool {
21 matches!(entry.kind, EntryKind::Blob | EntryKind::Exec)
22}
23
24async fn entries<R: GitRepo>(repo: &R, tree: Option<&str>) -> Result<BTreeMap<String, TreeEntry>> {
25 let Some(tree) = tree else {
26 return Ok(BTreeMap::new());
27 };
28 Ok(repo
29 .read_tree(tree)
30 .await?
31 .unwrap_or_default()
32 .into_iter()
33 .map(|entry| (entry.name.clone(), entry))
34 .collect())
35}
36
37/// The files that differ between two trees, a level at a time, with
38/// `hash` null for a file the newer tree no longer has. With no `old`,
39/// every file of `new`. Directories named in `skip` are not entered.
40pub async fn changed<R: GitRepo>(
41 repo: &R,
42 old: Option<&str>,
43 new: &str,
44 skip: &[String],
45 limit: u32,
46) -> Result<(Vec<FileEntry>, bool)> {
47 let limit = limit.clamp(1, MAX_LISTED_FILES) as usize;
48 let skipped = |name: &str| skip.iter().any(|dir| dir.eq_ignore_ascii_case(name));
49 let mut files = Vec::new();
50 let mut level = vec![(String::new(), old.map(str::to_owned), Some(new.to_owned()))];
51 while !level.is_empty() {
52 let read = try_join_all(level.iter().map(|(_, old_tree, new_tree)| async move {
53 try_join(entries(repo, old_tree.as_deref()), entries(repo, new_tree.as_deref())).await
54 }))
55 .await?;
56 let mut next = Vec::new();
57 for ((prefix, _, _), (before, after)) in level.iter().zip(read) {
58 let names: BTreeSet<&String> = before.keys().chain(after.keys()).collect();
59 for name in names {
60 let (was, now) = (before.get(name), after.get(name));
61 if was.map(|entry| (&entry.hash, entry.kind)) == now.map(|entry| (&entry.hash, entry.kind)) {
62 continue;
63 }
64 let path = format!("{prefix}{name}");
65 let dir = |entry: Option<&TreeEntry>| {
66 entry.filter(|entry| entry.kind == EntryKind::Tree).map(|entry| entry.hash.clone())
67 };
68 let (old_dir, new_dir) = (dir(was), dir(now));
69 if (old_dir.is_some() || new_dir.is_some()) && !skipped(name) {
70 next.push((format!("{path}/"), old_dir, new_dir));
71 }
72 let (old_file, new_file) = (was.filter(|e| is_file(e)), now.filter(|e| is_file(e)));
73 if old_file.is_none() && new_file.is_none() {
74 continue;
75 }
76 if files.len() >= limit {
77 return Ok((files, true));
78 }
79 files.push(FileEntry {
80 path,
81 hash: new_file.map(|entry| entry.hash.clone()),
82 });
83 }
84 }
85 level = next;
86 }
87 Ok((files, false))
88}
89
90/// The commit a branch or commit names, and its tree.
91pub async fn resolve<R: GitRepo>(repo: &R, git_ref: &str) -> Result<Option<(String, String)>> {
92 Ok(repo
93 .log(git_ref, 1)
94 .await?
95 .into_iter()
96 .next()
97 .map(|commit| (commit.hash, commit.tree_hash)))
98}
99
100/// Every file on `git_ref`, or what changed from `base` to it.
101pub async fn list<R: GitRepo>(
102 repo: &R,
103 base: Option<&str>,
104 head: &str,
105 skip: &[String],
106 limit: u32,
107) -> Result<FileList> {
108 let Some((commit, tree)) = resolve(repo, head).await? else {
109 return Ok(FileList::default());
110 };
111 let base_tree = match base {
112 Some(base) => match resolve(repo, base).await? {
113 Some((_, tree)) => Some(tree),
114 // A base that cannot be read is treated as nothing: every file.
115 None => None,
116 },
117 None => None,
118 };
119 let (files, truncated) = changed(repo, base_tree.as_deref(), &tree, skip, limit).await?;
120 Ok(FileList {
121 commit: Some(commit),
122 files,
123 truncated,
124 })
125}
126
127/// The text of each blob asked for, in order: none for one that is
128/// missing, larger than `max_bytes`, or binary.
129pub async fn read<R: GitRepo>(repo: &R, hashes: &[String], max_bytes: u32) -> Result<Vec<BlobText>> {
130 let hashes: Vec<&String> = hashes.iter().take(MAX_READ_BLOBS).collect();
131 let mut out = Vec::with_capacity(hashes.len());
132 // A few at a time: each read is a round trip, and a batch of large
133 // blobs held at once would not fit in memory.
134 for group in hashes.chunks(8) {
135 let read = try_join_all(group.iter().map(|hash| repo.read_blob(hash))).await?;
136 for (hash, bytes) in group.iter().zip(read) {
137 let size = bytes.as_ref().map_or(0, |bytes| bytes.len() as u64);
138 let text = bytes
139 .filter(|bytes| bytes.len() <= max_bytes as usize && !bytes.contains(&0))
140 .and_then(|bytes| String::from_utf8(bytes).ok());
141 out.push(BlobText {
142 hash: (*hash).clone(),
143 size,
144 text,
145 });
146 }
147 }
148 Ok(out)
149}
150
151#[cfg(test)]
152mod tests {
153 use std::collections::HashMap;
154 use std::future::Future;
155 use std::pin::pin;
156 use std::task::{Context, Poll, Waker};
157
158 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature};
159
160 use super::*;
161 use crate::store::Scope;
162
163 fn run<F: Future>(future: F) -> F::Output {
164 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
165 Poll::Ready(output) => output,
166 Poll::Pending => panic!("the fake store never waits"),
167 }
168 }
169
170 #[derive(Default)]
171 struct Fake {
172 trees: HashMap<String, Vec<TreeEntry>>,
173 blobs: HashMap<String, Vec<u8>>,
174 commits: HashMap<String, Commit>,
175 }
176
177 impl GitRepo for Fake {
178 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
179 unimplemented!()
180 }
181 async fn branches(&self) -> Result<Vec<Branch>> {
182 Ok(Vec::new())
183 }
184 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
185 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
186 }
187 async fn parents(&self, _commit_hash: &str) -> Result<Option<Vec<String>>> {
188 Ok(None)
189 }
190 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
191 Ok(self.trees.get(tree_hash).cloned())
192 }
193 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
194 Ok(self.blobs.get(blob_hash).cloned())
195 }
196 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
197 Ok(None)
198 }
199 async fn fork(&self, _target_key: &str) -> Result<()> {
200 Ok(())
201 }
202 }
203
204 fn entry(name: &str, hash: &str, kind: EntryKind) -> TreeEntry {
205 TreeEntry {
206 name: name.into(),
207 hash: hash.into(),
208 kind,
209 }
210 }
211
212 fn commit(hash: &str, tree: &str) -> Commit {
213 Commit {
214 hash: hash.into(),
215 tree_hash: tree.into(),
216 message: String::new(),
217 author: Signature { name: String::new(), email: String::new() },
218 parents: Vec::new(),
219 authored_at: String::new(),
220 }
221 }
222
223 /// Two commits: c1 has README, src/a.rs, node_modules/x.js; c2 changes
224 /// src/a.rs, deletes README, adds src/b.rs and a symlink.
225 fn repo() -> Fake {
226 let mut fake = Fake::default();
227 fake.trees.insert("src1".into(), vec![entry("a.rs", "a1", EntryKind::Blob)]);
228 fake.trees.insert("nm".into(), vec![entry("x.js", "x1", EntryKind::Blob)]);
229 fake.trees.insert(
230 "root1".into(),
231 vec![
232 entry("README.md", "r1", EntryKind::Blob),
233 entry("src", "src1", EntryKind::Tree),
234 entry("node_modules", "nm", EntryKind::Tree),
235 ],
236 );
237 fake.trees.insert(
238 "src2".into(),
239 vec![entry("a.rs", "a2", EntryKind::Blob), entry("b.rs", "b1", EntryKind::Exec)],
240 );
241 fake.trees.insert(
242 "root2".into(),
243 vec![
244 entry("src", "src2", EntryKind::Tree),
245 entry("node_modules", "nm", EntryKind::Tree),
246 entry("link", "l1", EntryKind::Symlink),
247 ],
248 );
249 fake.commits.insert("c1".into(), commit("c1", "root1"));
250 fake.commits.insert("c2".into(), commit("c2", "root2"));
251 fake.commits.insert("main".into(), commit("c2", "root2"));
252 fake
253 }
254
255 fn paths(list: &FileList) -> Vec<(String, Option<String>)> {
256 list.files.iter().map(|f| (f.path.clone(), f.hash.clone())).collect()
257 }
258
259 #[test]
260 fn lists_every_file_but_skipped_directories() {
261 let fake = repo();
262 let listed = run(list(&fake, None, "c1", &["node_modules".into()], 100)).unwrap();
263 assert_eq!(listed.commit.as_deref(), Some("c1"));
264 assert_eq!(
265 paths(&listed),
266 vec![("README.md".into(), Some("r1".into())), ("src/a.rs".into(), Some("a1".into()))]
267 );
268 }
269
270 #[test]
271 fn lists_only_what_changed() {
272 let fake = repo();
273 let listed = run(list(&fake, Some("c1"), "c2", &[], 100)).unwrap();
274 assert_eq!(
275 paths(&listed),
276 vec![
277 ("README.md".into(), None),
278 ("src/a.rs".into(), Some("a2".into())),
279 ("src/b.rs".into(), Some("b1".into())),
280 ]
281 );
282 assert!(!listed.truncated);
283 }
284
285 #[test]
286 fn says_when_the_list_was_cut_short() {
287 let fake = repo();
288 let listed = run(list(&fake, Some("c1"), "c2", &[], 2)).unwrap();
289 assert_eq!(listed.files.len(), 2);
290 assert!(listed.truncated);
291 }
292
293 #[test]
294 fn reads_text_but_not_binaries_or_large_blobs() {
295 let mut fake = repo();
296 fake.blobs.insert("t".into(), b"fn main() {}\n".to_vec());
297 fake.blobs.insert("bin".into(), vec![0, 1, 2]);
298 fake.blobs.insert("big".into(), vec![b'a'; 64]);
299 let read = run(read(&fake, &["t".into(), "bin".into(), "big".into(), "gone".into()], 32)).unwrap();
300 let texts: Vec<Option<&str>> = read.iter().map(|b| b.text.as_deref()).collect();
301 assert_eq!(texts, vec![Some("fn main() {}\n"), None, None, None]);
302 assert_eq!(read[2].size, 64);
303 }
304}