Sign in

flagon-io/g1t

public

g1t/services/repos/src/listing.rs

300 lines11,043 bytesCodeBlame
1//! Listing a repository's files for services that index it, such as
2//! search: every file on a branch, the files two commits differ in, and
3//! the text of many blobs at once.
4//!
5//! Trees are read a level at a time, all of a level at once, and identical
6//! subtrees are skipped by hash, so a push costs what it changed rather
7//! than what the repository holds. Directories the caller names (vendored
8//! code, build output) are never read at all.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use futures_util::future::{try_join, try_join_all};
13use g1t_contracts::repos::{BlobText, EntryKind, FileEntry, FileList, MAX_LISTED_FILES, MAX_READ_BLOBS, TreeEntry};
14use worker::Result;
15
16use crate::store::GitRepo;
17
18/// Whether an entry is a file whose text can be read: a blob, executable
19/// or not. Symlinks and submodules are not.
20fn is_file(entry: &TreeEntry) -> bool {
21 matches!(entry.kind, EntryKind::Blob | EntryKind::Exec)
22}
23
24async fn entries<R: GitRepo>(repo: &R, tree: Option<&str>) -> Result<BTreeMap<String, TreeEntry>> {
25 let Some(tree) = tree else {
26 return Ok(BTreeMap::new());
27 };
28 Ok(repo
29 .read_tree(tree)
30 .await?
31 .unwrap_or_default()
32 .into_iter()
33 .map(|entry| (entry.name.clone(), entry))
34 .collect())
35}
36
37/// The files that differ between two trees, a level at a time, with
38/// `hash` null for a file the newer tree no longer has. With no `old`,
39/// every file of `new`. Directories named in `skip` are not entered.
40pub async fn changed<R: GitRepo>(
41 repo: &R,
42 old: Option<&str>,
43 new: &str,
44 skip: &[String],
45 limit: u32,
46) -> Result<(Vec<FileEntry>, bool)> {
47 let limit = limit.clamp(1, MAX_LISTED_FILES) as usize;
48 let skipped = |name: &str| skip.iter().any(|dir| dir.eq_ignore_ascii_case(name));
49 let mut files = Vec::new();
50 let mut level = vec![(String::new(), old.map(str::to_owned), Some(new.to_owned()))];
51 while !level.is_empty() {
52 let read = try_join_all(level.iter().map(|(_, old_tree, new_tree)| async move {
53 try_join(entries(repo, old_tree.as_deref()), entries(repo, new_tree.as_deref())).await
54 }))
55 .await?;
56 let mut next = Vec::new();
57 for ((prefix, _, _), (before, after)) in level.iter().zip(read) {
58 let names: BTreeSet<&String> = before.keys().chain(after.keys()).collect();
59 for name in names {
60 let (was, now) = (before.get(name), after.get(name));
61 if was.map(|entry| (&entry.hash, entry.kind)) == now.map(|entry| (&entry.hash, entry.kind)) {
62 continue;
63 }
64 let path = format!("{prefix}{name}");
65 let dir = |entry: Option<&TreeEntry>| {
66 entry.filter(|entry| entry.kind == EntryKind::Tree).map(|entry| entry.hash.clone())
67 };
68 let (old_dir, new_dir) = (dir(was), dir(now));
69 if (old_dir.is_some() || new_dir.is_some()) && !skipped(name) {
70 next.push((format!("{path}/"), old_dir, new_dir));
71 }
72 let (old_file, new_file) = (was.filter(|e| is_file(e)), now.filter(|e| is_file(e)));
73 if old_file.is_none() && new_file.is_none() {
74 continue;
75 }
76 if files.len() >= limit {
77 return Ok((files, true));
78 }
79 files.push(FileEntry {
80 path,
81 hash: new_file.map(|entry| entry.hash.clone()),
82 });
83 }
84 }
85 level = next;
86 }
87 Ok((files, false))
88}
89
90/// The commit a branch or commit names, and its tree.
91pub async fn resolve<R: GitRepo>(repo: &R, git_ref: &str) -> Result<Option<(String, String)>> {
92 Ok(repo
93 .log(git_ref, 1)
94 .await?
95 .into_iter()
96 .next()
97 .map(|commit| (commit.hash, commit.tree_hash)))
98}
99
100/// Every file on `git_ref`, or what changed from `base` to it.
101pub async fn list<R: GitRepo>(
102 repo: &R,
103 base: Option<&str>,
104 head: &str,
105 skip: &[String],
106 limit: u32,
107) -> Result<FileList> {
108 let Some((commit, tree)) = resolve(repo, head).await? else {
109 return Ok(FileList::default());
110 };
111 let base_tree = match base {
112 Some(base) => resolve(repo, base).await?.map(|(_, tree)| tree),
113 None => None,
114 };
115 let (files, truncated) = changed(repo, base_tree.as_deref(), &tree, skip, limit).await?;
116 Ok(FileList {
117 commit: Some(commit),
118 files,
119 truncated,
120 })
121}
122
123/// The text of each blob asked for, in order: none for one that is
124/// missing, larger than `max_bytes`, or binary.
125pub async fn read<R: GitRepo>(repo: &R, hashes: &[String], max_bytes: u32) -> Result<Vec<BlobText>> {
126 let hashes: Vec<&String> = hashes.iter().take(MAX_READ_BLOBS).collect();
127 let mut out = Vec::with_capacity(hashes.len());
128 // A few at a time: each read is a round trip, and a batch of large
129 // blobs held at once would not fit in memory.
130 for group in hashes.chunks(8) {
131 let read = try_join_all(group.iter().map(|hash| repo.read_blob(hash))).await?;
132 for (hash, bytes) in group.iter().zip(read) {
133 let size = bytes.as_ref().map_or(0, |bytes| bytes.len() as u64);
134 let text = bytes
135 .filter(|bytes| bytes.len() <= max_bytes as usize && !bytes.contains(&0))
136 .and_then(|bytes| String::from_utf8(bytes).ok());
137 out.push(BlobText {
138 hash: (*hash).clone(),
139 size,
140 text,
141 });
142 }
143 }
144 Ok(out)
145}
146
147#[cfg(test)]
148mod tests {
149 use std::collections::HashMap;
150 use std::future::Future;
151 use std::pin::pin;
152 use std::task::{Context, Poll, Waker};
153
154 use g1t_contracts::repos::{Branch, Commit, GitAccess, Signature};
155
156 use super::*;
157 use crate::store::Scope;
158
159 fn run<F: Future>(future: F) -> F::Output {
160 match pin!(future).as_mut().poll(&mut Context::from_waker(Waker::noop())) {
161 Poll::Ready(output) => output,
162 Poll::Pending => panic!("the fake store never waits"),
163 }
164 }
165
166 #[derive(Default)]
167 struct Fake {
168 trees: HashMap<String, Vec<TreeEntry>>,
169 blobs: HashMap<String, Vec<u8>>,
170 commits: HashMap<String, Commit>,
171 }
172
173 impl GitRepo for Fake {
174 async fn access(&self, _scope: Scope) -> Result<GitAccess> {
175 unimplemented!()
176 }
177 async fn branches(&self) -> Result<Vec<Branch>> {
178 Ok(Vec::new())
179 }
180 async fn log(&self, git_ref: &str, _limit: u32) -> Result<Vec<Commit>> {
181 Ok(self.commits.get(git_ref).cloned().into_iter().collect())
182 }
183 async fn parents(&self, _commit_hash: &str) -> Result<Option<Vec<String>>> {
184 Ok(None)
185 }
186 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
187 Ok(self.trees.get(tree_hash).cloned())
188 }
189 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
190 Ok(self.blobs.get(blob_hash).cloned())
191 }
192 async fn read_file(&self, _git_ref: &str, _path: &str) -> Result<Option<Vec<u8>>> {
193 Ok(None)
194 }
195 async fn fork(&self, _target_key: &str) -> Result<()> {
196 Ok(())
197 }
198 }
199
200 fn entry(name: &str, hash: &str, kind: EntryKind) -> TreeEntry {
201 TreeEntry {
202 name: name.into(),
203 hash: hash.into(),
204 kind,
205 }
206 }
207
208 fn commit(hash: &str, tree: &str) -> Commit {
209 Commit {
210 hash: hash.into(),
211 tree_hash: tree.into(),
212 message: String::new(),
213 author: Signature { name: String::new(), email: String::new() },
214 parents: Vec::new(),
215 authored_at: String::new(),
216 }
217 }
218
219 /// Two commits: c1 has README, src/a.rs, node_modules/x.js; c2 changes
220 /// src/a.rs, deletes README, adds src/b.rs and a symlink.
221 fn repo() -> Fake {
222 let mut fake = Fake::default();
223 fake.trees.insert("src1".into(), vec![entry("a.rs", "a1", EntryKind::Blob)]);
224 fake.trees.insert("nm".into(), vec![entry("x.js", "x1", EntryKind::Blob)]);
225 fake.trees.insert(
226 "root1".into(),
227 vec![
228 entry("README.md", "r1", EntryKind::Blob),
229 entry("src", "src1", EntryKind::Tree),
230 entry("node_modules", "nm", EntryKind::Tree),
231 ],
232 );
233 fake.trees.insert(
234 "src2".into(),
235 vec![entry("a.rs", "a2", EntryKind::Blob), entry("b.rs", "b1", EntryKind::Exec)],
236 );
237 fake.trees.insert(
238 "root2".into(),
239 vec![
240 entry("src", "src2", EntryKind::Tree),
241 entry("node_modules", "nm", EntryKind::Tree),
242 entry("link", "l1", EntryKind::Symlink),
243 ],
244 );
245 fake.commits.insert("c1".into(), commit("c1", "root1"));
246 fake.commits.insert("c2".into(), commit("c2", "root2"));
247 fake.commits.insert("main".into(), commit("c2", "root2"));
248 fake
249 }
250
251 fn paths(list: &FileList) -> Vec<(String, Option<String>)> {
252 list.files.iter().map(|f| (f.path.clone(), f.hash.clone())).collect()
253 }
254
255 #[test]
256 fn lists_every_file_but_skipped_directories() {
257 let fake = repo();
258 let listed = run(list(&fake, None, "c1", &["node_modules".into()], 100)).unwrap();
259 assert_eq!(listed.commit.as_deref(), Some("c1"));
260 assert_eq!(
261 paths(&listed),
262 vec![("README.md".into(), Some("r1".into())), ("src/a.rs".into(), Some("a1".into()))]
263 );
264 }
265
266 #[test]
267 fn lists_only_what_changed() {
268 let fake = repo();
269 let listed = run(list(&fake, Some("c1"), "c2", &[], 100)).unwrap();
270 assert_eq!(
271 paths(&listed),
272 vec![
273 ("README.md".into(), None),
274 ("src/a.rs".into(), Some("a2".into())),
275 ("src/b.rs".into(), Some("b1".into())),
276 ]
277 );
278 assert!(!listed.truncated);
279 }
280
281 #[test]
282 fn says_when_the_list_was_cut_short() {
283 let fake = repo();
284 let listed = run(list(&fake, Some("c1"), "c2", &[], 2)).unwrap();
285 assert_eq!(listed.files.len(), 2);
286 assert!(listed.truncated);
287 }
288
289 #[test]
290 fn reads_text_but_not_binaries_or_large_blobs() {
291 let mut fake = repo();
292 fake.blobs.insert("t".into(), b"fn main() {}\n".to_vec());
293 fake.blobs.insert("bin".into(), vec![0, 1, 2]);
294 fake.blobs.insert("big".into(), vec![b'a'; 64]);
295 let read = run(read(&fake, &["t".into(), "bin".into(), "big".into(), "gone".into()], 32)).unwrap();
296 let texts: Vec<Option<&str>> = read.iter().map(|b| b.text.as_deref()).collect();
297 assert_eq!(texts, vec![Some("fn main() {}\n"), None, None, None]);
298 assert_eq!(read[2].size, 64);
299 }
300}