Sign in

flagon-io/g1t

public

g1t/services/search/src/read.rs

493 lines19,612 bytesCodeBlame
1//! Answering searches, suggestions as someone types, and Explore.
2
3use std::collections::{HashMap, HashSet};
4
5use g1t_contracts::repos::{ReadableArgs, Repo};
6use g1t_contracts::search::*;
7use g1t_contracts::{Outcome, Viewer};
8use serde::Deserialize;
9use worker::Result;
10
11use crate::Search;
12use crate::query::{ItemKind, Query};
13use crate::rules::normalize_language;
14use crate::snippet;
15use crate::sql::{self, CodePlan, Page, Param, Sql};
16use crate::store;
17use crate::visibility::{Indexed, Reader, check};
18
19/// Results a page can reach: no further than the counts go.
20const MAX_PAGE: u32 = 50;
21/// Results of each kind in a suggestion.
22const SUGGEST_EACH: u32 = 4;
23/// Repositories on a page of Explore.
24const EXPLORE_PAGE: u32 = 24;
25/// Pieces of code read for each file on a page, at most.
26const PIECES_PER_FILE: u32 = 4;
27
28/// A path's segments made safe to put in an address.
29pub fn encode_path(path: &str) -> String {
30 path.split('/')
31 .map(|segment| {
32 let mut out = String::with_capacity(segment.len());
33 for byte in segment.bytes() {
34 match byte {
35 b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b'@' | b'+' | b'=' | b',' => {
36 out.push(byte as char)
37 }
38 _ => out.push_str(&format!("%{byte:02X}")),
39 }
40 }
41 out
42 })
43 .collect::<Vec<_>>()
44 .join("/")
45}
46
47fn split_labels(labels: &str) -> Vec<String> {
48 labels.split('|').filter(|label| !label.is_empty()).map(str::to_owned).collect()
49}
50
51fn split_topics(topics: &str) -> Vec<String> {
52 topics.split_whitespace().map(str::to_owned).collect()
53}
54
55#[derive(Deserialize)]
56struct RepoRow {
57 repo_id: String,
58 namespace: String,
59 name: String,
60 description: Option<String>,
61 topics: String,
62 language: Option<String>,
63 private: u8,
64 pushed_at: Option<String>,
65 created_at: String,
66 #[serde(default)]
67 readme: Option<String>,
68}
69
70#[derive(Deserialize)]
71struct FileHit {
72 fid: i64,
73}
74
75#[derive(Deserialize)]
76struct FileRow {
77 fid: i64,
78 path: String,
79 language: Option<String>,
80 repo_id: String,
81 namespace: String,
82 name: String,
83 default_branch: String,
84 private: u8,
85}
86
87#[derive(Deserialize)]
88struct PieceRow {
89 fid: i64,
90 start_line: u32,
91 content: String,
92}
93
94#[derive(Deserialize)]
95struct ItemRow {
96 kind: String,
97 number: u32,
98 title: String,
99 state: String,
100 status: String,
101 author: String,
102 #[serde(default)]
103 requested_by: Option<String>,
104 labels: String,
105 updated_at: String,
106 repo_id: String,
107 namespace: String,
108 name: String,
109 private: u8,
110 body: Option<String>,
111}
112
113#[derive(Deserialize)]
114struct PersonRow {
115 kind: String,
116 slug: String,
117 name: Option<String>,
118 bio: Option<String>,
119 avatar: Option<String>,
120}
121
122/// A result, and the repository whose visibility decides whether it is shown.
123struct Found {
124 hit: Hit,
125 repo: Option<Indexed>,
126}
127
128fn repo_hit(row: RepoRow, terms: &[String]) -> Found {
129 let full = format!("{}/{}", row.namespace, row.name);
130 let mut hit = Hit::new(HitKind::Repository, full.clone(), format!("/{}/{}", row.namespace, row.name));
131 hit.repo = Some(full);
132 hit.private = row.private != 0;
133 hit.snippet = match (&row.description, &row.readme) {
134 (Some(description), _) if !description.trim().is_empty() => snippet::prose(description, terms),
135 (_, Some(readme)) if snippet::contains_any(readme, terms) => snippet::prose(readme, terms),
136 _ => Vec::new(),
137 };
138 hit.description = row.description;
139 hit.language = row.language;
140 hit.topics = split_topics(&row.topics);
141 hit.updated_at = row.pushed_at.or(Some(row.created_at));
142 Found {
143 hit,
144 repo: Some(Indexed {
145 repo_id: row.repo_id,
146 namespace: row.namespace,
147 private: row.private != 0,
148 }),
149 }
150}
151
152fn item_hit(row: ItemRow, terms: &[String]) -> Found {
153 let pull = row.kind == "pull";
154 let full = format!("{}/{}", row.namespace, row.name);
155 let url = format!("/{full}/{}/{}", if pull { "pull" } else { "issues" }, row.number);
156 let mut hit = Hit::new(if pull { HitKind::Pull } else { HitKind::Issue }, row.title, url);
157 hit.repo = Some(full);
158 hit.private = row.private != 0;
159 hit.snippet = snippet::prose(row.body.as_deref().unwrap_or_default(), terms);
160 hit.number = Some(row.number);
161 hit.state = Some(if pull { row.status } else { row.state });
162 hit.author = Some(row.author);
163 hit.requested_by = row.requested_by;
164 hit.labels = split_labels(&row.labels);
165 hit.updated_at = Some(row.updated_at);
166 Found {
167 hit,
168 repo: Some(Indexed {
169 repo_id: row.repo_id,
170 namespace: row.namespace,
171 private: row.private != 0,
172 }),
173 }
174}
175
176fn person_hit(row: PersonRow, terms: &[String]) -> Found {
177 let workspace = row.kind == "workspace";
178 let url = if workspace { format!("/{}", row.slug) } else { format!("/u/{}", row.slug) };
179 let title = row.name.clone().filter(|name| !name.trim().is_empty()).unwrap_or_else(|| row.slug.clone());
180 let mut hit = Hit::new(if workspace { HitKind::Workspace } else { HitKind::User }, title, url);
181 hit.slug = Some(row.slug);
182 hit.snippet = row.bio.as_deref().map(|bio| snippet::prose(bio, terms)).unwrap_or_default();
183 hit.description = row.bio;
184 hit.avatar = row.avatar;
185 Found { hit, repo: None }
186}
187
188impl Search {
189 /// The second visibility check: of these repositories, the ones the
190 /// repos service says this viewer may read now, and of those, the ones
191 /// that are archived. Corrects the index where it was behind.
192 async fn readable(&self, viewer: &Viewer, indexed: &[Indexed]) -> (HashSet<String>, HashSet<String>) {
193 if indexed.is_empty() {
194 return (HashSet::new(), HashSet::new());
195 }
196 let ids: Vec<String> = indexed.iter().map(|row| row.repo_id.clone()).collect::<HashSet<_>>().into_iter().collect();
197 let readable: Result<Vec<Repo>> =
198 g1t_kit::call(&self.repos, "readable", &ReadableArgs { ids, viewer: viewer.clone() }).await;
199 if let Err(error) = &readable {
200 worker::console_error!("search: could not check visibility: {error}");
201 }
202 let verdict = check(&Reader::of(viewer), indexed, readable.as_deref().ok());
203 let archived: HashSet<String> = readable
204 .as_deref()
205 .unwrap_or_default()
206 .iter()
207 .filter(|repo| repo.archived())
208 .map(|repo| repo.id.clone())
209 .collect();
210 if !verdict.corrections.is_empty() {
211 let mut statements = Vec::new();
212 for correction in &verdict.corrections {
213 let (text, params) = match &correction.path {
214 Some((namespace, name)) => (
215 "UPDATE repos SET private = ?, namespace = ?, name = ? WHERE repo_id = ?",
216 vec![
217 Param::Int(i64::from(correction.private)),
218 Param::Text(namespace.clone()),
219 Param::Text(name.clone()),
220 Param::Text(correction.repo_id.clone()),
221 ],
222 ),
223 None => (
224 "UPDATE repos SET private = ? WHERE repo_id = ?",
225 vec![Param::Int(i64::from(correction.private)), Param::Text(correction.repo_id.clone())],
226 ),
227 };
228 if let Ok(statement) = store::prepare(&self.db, text, params) {
229 statements.push(statement);
230 }
231 }
232 if let Err(error) = store::run_all(&self.db, statements).await {
233 worker::console_error!("search: could not correct visibility: {error}");
234 }
235 }
236 (verdict.keep, archived)
237 }
238
239 /// Results the viewer may see, in order.
240 async fn admit(&self, viewer: &Viewer, found: Vec<Found>) -> Vec<Hit> {
241 let indexed: Vec<Indexed> = found.iter().filter_map(|f| f.repo.clone()).collect();
242 let (keep, _) = self.readable(viewer, &indexed).await;
243 found
244 .into_iter()
245 .filter(|f| f.repo.as_ref().is_none_or(|repo| keep.contains(&repo.repo_id)))
246 .map(|f| f.hit)
247 .collect()
248 }
249
250 async fn count_of(&self, kind: SearchType, query: &Query, reader: &Reader) -> Result<u32> {
251 if !query.applies_to(kind) {
252 return Ok(0);
253 }
254 let sql = match kind {
255 SearchType::Repositories => Some(sql::repos_count(query, reader)),
256 SearchType::Code => sql::code_plan(query).ok().map(|plan| sql::code_count(&plan, query, reader)),
257 SearchType::Issues => Some(sql::items_count(query, reader, ItemKind::Issue)),
258 SearchType::Pulls => Some(sql::items_count(query, reader, ItemKind::Pull)),
259 SearchType::People => sql::people_select(query).map(|select| sql::count(&select)),
260 };
261 match sql {
262 Some(sql) => Ok(store::count(&self.db, &sql).await?.min(COUNT_CAP)),
263 None => Ok(0),
264 }
265 }
266
267 async fn code_hits(&self, plan: &CodePlan, query: &Query, reader: &Reader, page: Page, terms: &[String]) -> Result<Vec<Found>> {
268 let fids: Vec<i64> = store::all::<FileHit>(&self.db, &sql::code_page(plan, query, reader, page))
269 .await?
270 .into_iter()
271 .map(|row| row.fid)
272 .collect();
273 if fids.is_empty() {
274 return Ok(Vec::new());
275 }
276 let files = store::all::<FileRow>(&self.db, &sql::code_files(&fids)).await?;
277 let mut pieces: HashMap<i64, Vec<PieceRow>> = HashMap::new();
278 for piece in store::all::<PieceRow>(&self.db, &sql::code_pieces(plan, &fids, page.limit * PIECES_PER_FILE)).await? {
279 pieces.entry(piece.fid).or_default().push(piece);
280 }
281 // Files that matched by name have no matching piece: their first.
282 let missing: Vec<i64> = fids.iter().copied().filter(|fid| !pieces.contains_key(fid)).collect();
283 if !missing.is_empty() {
284 for piece in store::all::<PieceRow>(&self.db, &sql::first_pieces(&missing)).await? {
285 pieces.entry(piece.fid).or_default().push(piece);
286 }
287 }
288 let by_id: HashMap<i64, FileRow> = files.into_iter().map(|file| (file.fid, file)).collect();
289 let mut found = Vec::new();
290 for fid in fids {
291 let Some(file) = by_id.get(&fid) else { continue };
292 let parts: Vec<(u32, &str)> = pieces
293 .get(&fid)
294 .map(|list| list.iter().take(PIECES_PER_FILE as usize).map(|p| (p.start_line, p.content.as_str())).collect())
295 .unwrap_or_default();
296 let lines = snippet::code(&parts, terms);
297 let line = snippet::first_match(&lines);
298 let full = format!("{}/{}", file.namespace, file.name);
299 let url = format!(
300 "/{full}/blob/{}/{}{}",
301 encode_path(&file.default_branch),
302 encode_path(&file.path),
303 line.map(|n| format!("#L{n}")).unwrap_or_default()
304 );
305 let mut hit = Hit::new(HitKind::Code, file.path.clone(), url);
306 hit.repo = Some(full);
307 hit.private = file.private != 0;
308 hit.path = Some(file.path.clone());
309 hit.language = file.language.clone();
310 hit.git_ref = Some(file.default_branch.clone());
311 hit.lines = lines;
312 found.push(Found {
313 hit,
314 repo: Some(Indexed {
315 repo_id: file.repo_id.clone(),
316 namespace: file.namespace.clone(),
317 private: file.private != 0,
318 }),
319 });
320 }
321 Ok(found)
322 }
323
324 async fn page_of(&self, kind: SearchType, query: &Query, reader: &Reader, page: Page, terms: &[String]) -> Result<Vec<Found>> {
325 if !query.applies_to(kind) {
326 return Ok(Vec::new());
327 }
328 Ok(match kind {
329 SearchType::Repositories => store::all::<RepoRow>(&self.db, &sql::repos_page(query, reader, page))
330 .await?
331 .into_iter()
332 .map(|row| repo_hit(row, terms))
333 .collect(),
334 SearchType::Code => match sql::code_plan(query) {
335 Ok(plan) => self.code_hits(&plan, query, reader, page, terms).await?,
336 Err(_) => Vec::new(),
337 },
338 SearchType::Issues | SearchType::Pulls => {
339 let item = if kind == SearchType::Issues { ItemKind::Issue } else { ItemKind::Pull };
340 store::all::<ItemRow>(&self.db, &sql::items_page(query, reader, item, page))
341 .await?
342 .into_iter()
343 .map(|row| item_hit(row, terms))
344 .collect()
345 }
346 SearchType::People => match sql::people_page(query, page) {
347 Some(sql) => store::all::<PersonRow>(&self.db, &sql)
348 .await?
349 .into_iter()
350 .map(|row| person_hit(row, terms))
351 .collect(),
352 None => Vec::new(),
353 },
354 })
355 }
356
357 pub async fn search(&self, a: SearchArgs) -> Result<Outcome<SearchResults>> {
358 let query = Query::parse(&a.query);
359 let reader = Reader::of(&a.viewer);
360 let kind = a.kind.unwrap_or_else(|| query.default_type());
361 let per_page = a.per_page.unwrap_or(DEFAULT_PER_PAGE).clamp(1, MAX_PER_PAGE);
362 let page = a.page.unwrap_or(1).clamp(1, MAX_PAGE);
363 let mut results = SearchResults {
364 query: query.display(),
365 kind,
366 counts: SearchCounts::default(),
367 page,
368 per_page,
369 more: false,
370 hits: Vec::new(),
371 notes: query.notes.clone(),
372 };
373 if query.is_empty() {
374 results.notes.push("Type a word, a \"phrase\" or a qualifier such as repo:owner/name.".into());
375 return Ok(Outcome::Ok(results));
376 }
377 if kind == SearchType::Code
378 && let Err(why) = sql::code_plan(&query)
379 {
380 results.notes.push(why.to_owned());
381 }
382 for each in SearchType::ALL {
383 let counted = match self.count_of(each, &query, &reader).await {
384 Ok(count) => count,
385 Err(error) => {
386 worker::console_error!("search: counting {} failed: {error}", each.as_str());
387 0
388 }
389 };
390 results.counts.set(each, counted);
391 }
392 // One more than a page, to know whether there is another.
393 let window = Page { limit: per_page + 1, offset: (page - 1) * per_page };
394 let terms = query.highlights();
395 let found = match self.page_of(kind, &query, &reader, window, &terms).await {
396 Ok(found) => found,
397 Err(error) => {
398 // Most often a query the index could not read.
399 worker::console_error!("search: {} failed for {:?}: {error}", kind.as_str(), results.query);
400 results.notes.push("Part of that query could not be read. Try fewer symbols, or put them in quotes.".into());
401 Vec::new()
402 }
403 };
404 let before = found.len();
405 let mut hits = self.admit(&a.viewer, found).await;
406 // What the second check dropped is not counted either.
407 let dropped = (before - hits.len()) as u32;
408 results.counts.set(kind, results.counts.get(kind).saturating_sub(dropped));
409 results.more = hits.len() > per_page as usize;
410 hits.truncate(per_page as usize);
411 results.hits = hits;
412 Ok(Outcome::Ok(results))
413 }
414
415 /// A few repositories, issues, pull requests and people, for the
416 /// command palette. Never code: that is a full search away.
417 pub async fn suggest(&self, a: SuggestArgs) -> Result<Vec<Hit>> {
418 let query = Query::parse(&a.query);
419 if query.prose_match().is_none() {
420 return Ok(Vec::new());
421 }
422 let reader = Reader::of(&a.viewer);
423 let page = Page { limit: SUGGEST_EACH, offset: 0 };
424 let terms = query.highlights();
425 let mut found = Vec::new();
426 for kind in [SearchType::Repositories, SearchType::Issues, SearchType::Pulls, SearchType::People] {
427 match self.page_of(kind, &query, &reader, page, &terms).await {
428 Ok(more) => found.extend(more),
429 Err(error) => worker::console_error!("search: suggesting {} failed: {error}", kind.as_str()),
430 }
431 }
432 Ok(self.admit(&a.viewer, found).await)
433 }
434
435 pub async fn explore(&self, a: ExploreArgs) -> Result<Explore> {
436 let newest = a.sort.as_deref() == Some("new");
437 let language = a.language.as_deref().map(str::trim).filter(|l| !l.is_empty()).map(normalize_language);
438 let topic = a.topic.as_deref().map(|t| t.trim().to_lowercase()).filter(|t| !t.is_empty());
439 let page = a.page.unwrap_or(1).clamp(1, MAX_PAGE);
440 let window = Page { limit: EXPLORE_PAGE + 1, offset: (page - 1) * EXPLORE_PAGE };
441 let rows = store::all::<RepoRow>(&self.db, &sql::explore(newest, language.as_deref(), topic.as_deref(), window)).await?;
442 let indexed: Vec<Indexed> = rows
443 .iter()
444 .map(|row| Indexed { repo_id: row.repo_id.clone(), namespace: row.namespace.clone(), private: row.private != 0 })
445 .collect();
446 // Explore is for everyone: checked as if signed out, whoever asks.
447 let (keep, archived) = self.readable(&None, &indexed).await;
448 let mut repos: Vec<ExploreRepo> = rows
449 .into_iter()
450 .filter(|row| keep.contains(&row.repo_id))
451 .map(|row| ExploreRepo {
452 archived: archived.contains(&row.repo_id),
453 namespace: row.namespace,
454 name: row.name,
455 description: row.description,
456 topics: split_topics(&row.topics),
457 language: row.language,
458 created_at: row.created_at,
459 pushed_at: row.pushed_at,
460 })
461 .collect();
462 let more = repos.len() > EXPLORE_PAGE as usize;
463 repos.truncate(EXPLORE_PAGE as usize);
464 let (languages, topics) = if page == 1 {
465 (
466 store::all::<Facet>(&self.db, &Sql { text: sql::LANGUAGE_FACETS.into(), params: vec![] }).await?,
467 store::all::<Facet>(&self.db, &Sql { text: sql::TOPIC_FACETS.into(), params: vec![] }).await?,
468 )
469 } else {
470 (Vec::new(), Vec::new())
471 };
472 Ok(Explore { repos, languages, topics, page, more })
473 }
474}
475
476#[cfg(test)]
477mod tests {
478 use super::*;
479
480 #[test]
481 fn paths_are_safe_in_addresses() {
482 assert_eq!(encode_path("src/main.rs"), "src/main.rs");
483 assert_eq!(encode_path("docs/a b#c?.md"), "docs/a%20b%23c%3F.md");
484 assert_eq!(encode_path("src/ünï.rs"), "src/%C3%BCn%C3%AF.rs");
485 }
486
487 #[test]
488 fn labels_and_topics_are_split() {
489 assert_eq!(split_labels("|bug|good first issue|"), vec!["bug", "good first issue"]);
490 assert!(split_labels("|").is_empty());
491 assert_eq!(split_topics("cli rust"), vec!["cli", "rust"]);
492 }
493}