Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/search/src/query.rs

540 lines21,010 bytesCodeBlame
1//! Reading a search query: words, quoted phrases, `-` to leave a word out,
2//! and qualifiers such as `repo:acme/web` or `is:open`. Pure, so every rule
3//! is tested apart from the index.
4
5use g1t_contracts::search::{MAX_QUERY_CHARS, SearchType};
6
7use crate::rules::normalize_language;
8
9/// A word or phrase to match.
10#[derive(Clone, Debug, PartialEq, Eq)]
11pub struct Term {
12 pub text: String,
13 /// Written in quotes: matched exactly, as one piece.
14 pub phrase: bool,
15 /// Written with a leading `-`: results must not have it.
16 pub negated: bool,
17}
18
19/// Values of one qualifier, kept and left out (`-label:wontfix`).
20#[derive(Clone, Debug, Default, PartialEq, Eq)]
21pub struct Filter {
22 pub include: Vec<String>,
23 pub exclude: Vec<String>,
24}
25
26impl Filter {
27 fn add(&mut self, value: String, negated: bool) {
28 let list = if negated { &mut self.exclude } else { &mut self.include };
29 if !list.contains(&value) {
30 list.push(value);
31 }
32 }
33
34 pub fn is_empty(&self) -> bool {
35 self.include.is_empty() && self.exclude.is_empty()
36 }
37}
38
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum ItemKind {
41 Issue,
42 Pull,
43}
44
45/// `is:open`, `is:closed`, `is:merged`, `is:draft`.
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47pub enum ItemState {
48 Open,
49 Closed,
50 Merged,
51 Draft,
52}
53
54/// `is:public` or `is:private`.
55#[derive(Clone, Copy, Debug, PartialEq, Eq)]
56pub enum Visibility {
57 Public,
58 Private,
59}
60
61#[derive(Clone, Debug, Default, PartialEq, Eq)]
62pub struct Query {
63 pub terms: Vec<Term>,
64 /// `repo:owner/name`, lowercase `owner/name`.
65 pub repos: Filter,
66 /// `org:`, `workspace:`, `owner:` and `user:`: a workspace's slug.
67 pub owners: Filter,
68 pub languages: Filter,
69 /// `path:`, lowercase, as written: a glob when it has a `*`.
70 pub paths: Filter,
71 pub authors: Filter,
72 pub labels: Filter,
73 pub kind: Option<ItemKind>,
74 pub state: Option<ItemState>,
75 pub visibility: Option<Visibility>,
76 /// `type:code` and the like.
77 pub type_hint: Option<SearchType>,
78 /// What the query asked that could not be done, said plainly.
79 pub notes: Vec<String>,
80}
81
82/// Splits a query into tokens: runs of non-space, with quoted runs kept
83/// whole wherever they start (`"a b"`, `label:"good first issue"`).
84fn tokens(text: &str) -> Vec<String> {
85 let mut out = Vec::new();
86 let mut current = String::new();
87 let mut quoted = false;
88 for c in text.chars() {
89 match c {
90 '"' => {
91 quoted = !quoted;
92 current.push(c);
93 }
94 c if c.is_whitespace() && !quoted => {
95 if !current.is_empty() {
96 out.push(std::mem::take(&mut current));
97 }
98 }
99 c => current.push(c),
100 }
101 }
102 if !current.is_empty() {
103 out.push(current);
104 }
105 out
106}
107
108/// A value with its quotes taken off.
109fn unquote(value: &str) -> (String, bool) {
110 let trimmed = value.trim();
111 if trimmed.len() >= 2 && trimmed.starts_with('"') && trimmed.ends_with('"') {
112 return (trimmed[1..trimmed.len() - 1].to_owned(), true);
113 }
114 // An opening quote never closed: the rest of the query.
115 if let Some(rest) = trimmed.strip_prefix('"') {
116 return (rest.trim_end_matches('"').to_owned(), true);
117 }
118 (trimmed.replace('"', ""), false)
119}
120
121impl Query {
122 pub fn parse(text: &str) -> Query {
123 let text: String = text.chars().take(MAX_QUERY_CHARS).collect();
124 let mut query = Query::default();
125 for token in tokens(&text) {
126 let (negated, body) = match token.strip_prefix('-') {
127 Some(rest) if !rest.is_empty() => (true, rest.to_owned()),
128 _ => (false, token.clone()),
129 };
130 if let Some((key, value)) = body.split_once(':')
131 && !key.starts_with('"')
132 && query.qualifier(&key.to_lowercase(), value, negated)
133 {
134 continue;
135 }
136 let (text, phrase) = unquote(&body);
137 if text.trim().is_empty() {
138 continue;
139 }
140 query.terms.push(Term {
141 text: if phrase { text } else { text.trim().to_owned() },
142 phrase,
143 negated,
144 });
145 }
146 query
147 }
148
149 /// Takes `key:value` as a qualifier, if `key` is one. False when it is
150 /// not, so the token is read as words.
151 fn qualifier(&mut self, key: &str, value: &str, negated: bool) -> bool {
152 let (value, _) = unquote(value);
153 let value = value.trim().to_owned();
154 let known = matches!(
155 key,
156 "repo" | "org" | "workspace" | "owner" | "user" | "language" | "lang" | "path" | "is" | "state" | "author"
157 | "label" | "type" | "in"
158 );
159 if !known {
160 return false;
161 }
162 if value.is_empty() {
163 self.notes.push(format!("{key}: needs a value, so it was left out."));
164 return true;
165 }
166 let lower = value.to_lowercase();
167 match key {
168 "repo" => match lower.split_once('/') {
169 Some((owner, name)) if !owner.is_empty() && !name.is_empty() && !name.contains('/') => {
170 self.repos.add(format!("{owner}/{name}"), negated);
171 }
172 _ => self.notes.push(format!("repo:{value} should be written repo:owner/name.")),
173 },
174 "org" | "workspace" | "owner" | "user" => self.owners.add(lower, negated),
175 "language" | "lang" => self.languages.add(normalize_language(&lower), negated),
176 "path" => self.paths.add(lower.trim_start_matches('/').to_owned(), negated),
177 "author" => self.authors.add(lower.trim_start_matches('@').to_owned(), negated),
178 "label" => self.labels.add(lower, negated),
179 "type" => match SearchType::parse(&lower) {
180 Some(kind) => self.type_hint = Some(kind),
181 None => self.notes.push(format!("type:{value} is not a kind of result.")),
182 },
183 // `in:title` and the like narrow nothing here: every field is searched.
184 "in" => {}
185 "is" | "state" => self.is(&lower, &value, negated),
186 _ => unreachable!(),
187 }
188 true
189 }
190
191 fn is(&mut self, lower: &str, value: &str, negated: bool) {
192 if negated {
193 // `-is:open` reads as its opposite where there is one.
194 match lower {
195 "open" => self.state = Some(ItemState::Closed),
196 "closed" => self.state = Some(ItemState::Open),
197 "private" => self.visibility = Some(Visibility::Public),
198 "public" => self.visibility = Some(Visibility::Private),
199 _ => self.notes.push(format!("-is:{value} cannot be left out, so it was ignored.")),
200 }
201 return;
202 }
203 match lower {
204 "issue" | "issues" => self.kind = Some(ItemKind::Issue),
205 "pr" | "pull" | "pulls" | "pull-request" => self.kind = Some(ItemKind::Pull),
206 "open" => self.state = Some(ItemState::Open),
207 "closed" => self.state = Some(ItemState::Closed),
208 "merged" => {
209 self.kind = Some(ItemKind::Pull);
210 self.state = Some(ItemState::Merged);
211 }
212 "draft" => {
213 self.kind = Some(ItemKind::Pull);
214 self.state = Some(ItemState::Draft);
215 }
216 "public" => self.visibility = Some(Visibility::Public),
217 "private" => self.visibility = Some(Visibility::Private),
218 _ => self.notes.push(format!("is:{value} is not something g1t can search for.")),
219 }
220 }
221
222 /// Words and phrases results must have.
223 pub fn positive(&self) -> impl Iterator<Item = &Term> {
224 self.terms.iter().filter(|term| !term.negated)
225 }
226
227 /// What to highlight in a result: every word and phrase it must have.
228 pub fn highlights(&self) -> Vec<String> {
229 self.positive().map(|term| term.text.clone()).filter(|text| !text.is_empty()).collect()
230 }
231
232 /// Whether anything at all was asked.
233 pub fn is_empty(&self) -> bool {
234 self.terms.is_empty()
235 && self.repos.is_empty()
236 && self.owners.is_empty()
237 && self.languages.is_empty()
238 && self.paths.is_empty()
239 && self.authors.is_empty()
240 && self.labels.is_empty()
241 && self.kind.is_none()
242 && self.state.is_none()
243 && self.visibility.is_none()
244 }
245
246 /// The tab a query is about when none is named: `type:` if given, then
247 /// pull requests for `is:pr`, issues for `is:issue` or anything only
248 /// issues have, code for `path:`, else repositories.
249 pub fn default_type(&self) -> SearchType {
250 if let Some(kind) = self.type_hint {
251 return kind;
252 }
253 match self.kind {
254 Some(ItemKind::Pull) => return SearchType::Pulls,
255 Some(ItemKind::Issue) => return SearchType::Issues,
256 None => {}
257 }
258 if !self.paths.is_empty() {
259 return SearchType::Code;
260 }
261 if self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty() {
262 return SearchType::Issues;
263 }
264 SearchType::Repositories
265 }
266
267 /// Whether the query can have results of `kind`: qualifiers only one
268 /// kind has rule the others out (`path:` is only code, `label:` only
269 /// issues and pull requests).
270 pub fn applies_to(&self, kind: SearchType) -> bool {
271 let item_only = self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty();
272 let repo_scoped = !self.repos.is_empty() || !self.owners.is_empty() || self.visibility.is_some();
273 match kind {
274 SearchType::Repositories => self.paths.is_empty() && self.kind.is_none() && !item_only,
275 SearchType::Code => self.kind.is_none() && !item_only,
276 SearchType::Issues => {
277 self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Pull)
278 && !matches!(self.state, Some(ItemState::Merged | ItemState::Draft))
279 }
280 SearchType::Pulls => self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Issue),
281 SearchType::People => {
282 self.paths.is_empty() && self.languages.is_empty() && self.kind.is_none() && !item_only && !repo_scoped
283 }
284 }
285 }
286
287 /// The full-text expression for prose (repositories, issues, people):
288 /// every word, each a prefix as people type it, phrases exact, `-`
289 /// words left out. `None` when there are no words to match.
290 pub fn prose_match(&self) -> Option<String> {
291 let words: Vec<&Term> = self
292 .terms
293 .iter()
294 .filter(|term| term.text.chars().any(char::is_alphanumeric))
295 .collect();
296 let positive: Vec<&&Term> = words.iter().filter(|term| !term.negated).collect();
297 if positive.is_empty() {
298 return None;
299 }
300 let mut parts: Vec<String> = positive
301 .iter()
302 .map(|term| {
303 let quoted = fts_string(&term.text);
304 // A bare word that ends in a letter or digit also matches
305 // longer words it starts: `pars` finds `parser`.
306 let prefix = !term.phrase && term.text.chars().last().is_some_and(char::is_alphanumeric);
307 if prefix { format!("{quoted}*") } else { quoted }
308 })
309 .collect();
310 for term in words.iter().filter(|term| term.negated) {
311 parts.push(format!("NOT {}", fts_string(&term.text)));
312 }
313 Some(parts.join(" "))
314 }
315
316 /// The full-text expression for code, which matches any run of three
317 /// characters or more: each word or phrase as written. Shorter words
318 /// cannot use the index and come back separately, to be matched
319 /// against the text directly.
320 pub fn code_match(&self) -> (Option<String>, Vec<String>) {
321 let mut parts = Vec::new();
322 let mut short = Vec::new();
323 for term in self.positive() {
324 if term.text.chars().count() >= 3 {
325 parts.push(fts_string(&term.text));
326 } else {
327 short.push(term.text.to_lowercase());
328 }
329 }
330 if parts.is_empty() {
331 return (None, short);
332 }
333 for term in self.terms.iter().filter(|term| term.negated && term.text.chars().count() >= 3) {
334 parts.push(format!("NOT {}", fts_string(&term.text)));
335 }
336 (Some(parts.join(" ")), short)
337 }
338
339 /// The query written back out, tidied, as results say it was read.
340 pub fn display(&self) -> String {
341 let mut out: Vec<String> = Vec::new();
342 for term in &self.terms {
343 let text = if term.phrase || term.text.contains(' ') { format!("\"{}\"", term.text) } else { term.text.clone() };
344 out.push(if term.negated { format!("-{text}") } else { text });
345 }
346 let mut qualifier = |key: &str, filter: &Filter| {
347 for value in &filter.include {
348 out.push(format!("{key}:{}", quote_if_needed(value)));
349 }
350 for value in &filter.exclude {
351 out.push(format!("-{key}:{}", quote_if_needed(value)));
352 }
353 };
354 qualifier("repo", &self.repos);
355 qualifier("org", &self.owners);
356 qualifier("language", &self.languages);
357 qualifier("path", &self.paths);
358 qualifier("author", &self.authors);
359 qualifier("label", &self.labels);
360 match self.kind {
361 Some(ItemKind::Issue) => out.push("is:issue".into()),
362 Some(ItemKind::Pull) if !matches!(self.state, Some(ItemState::Merged | ItemState::Draft)) => out.push("is:pr".into()),
363 _ => {}
364 }
365 match self.state {
366 Some(ItemState::Open) => out.push("is:open".into()),
367 Some(ItemState::Closed) => out.push("is:closed".into()),
368 Some(ItemState::Merged) => out.push("is:merged".into()),
369 Some(ItemState::Draft) => out.push("is:draft".into()),
370 None => {}
371 }
372 match self.visibility {
373 Some(Visibility::Public) => out.push("is:public".into()),
374 Some(Visibility::Private) => out.push("is:private".into()),
375 None => {}
376 }
377 out.join(" ")
378 }
379}
380
381fn quote_if_needed(value: &str) -> String {
382 if value.contains(' ') { format!("\"{value}\"") } else { value.to_owned() }
383}
384
385/// A string as the full-text index reads it: in double quotes, with any
386/// double quote in it doubled, so nothing in it is read as syntax.
387pub fn fts_string(text: &str) -> String {
388 format!("\"{}\"", text.replace('"', "\"\""))
389}
390
391/// A `path:` value as a SQL `LIKE` pattern: a glob when it has a `*`
392/// (`*.rs`, `src/*/mod.rs`), else anywhere in the path. `%`, `_` and `\`
393/// match themselves.
394pub fn path_pattern(value: &str) -> String {
395 let mut escaped = String::with_capacity(value.len() + 2);
396 for c in value.chars() {
397 match c {
398 '\\' | '%' | '_' => {
399 escaped.push('\\');
400 escaped.push(c);
401 }
402 '*' => escaped.push('%'),
403 '?' => escaped.push('_'),
404 c => escaped.push(c),
405 }
406 }
407 if value.contains('*') || value.contains('?') {
408 escaped
409 } else {
410 format!("%{escaped}%")
411 }
412}
413
414#[cfg(test)]
415mod tests {
416 use super::*;
417
418 fn words(query: &Query) -> Vec<(&str, bool, bool)> {
419 query.terms.iter().map(|t| (t.text.as_str(), t.phrase, t.negated)).collect()
420 }
421
422 #[test]
423 fn words_phrases_and_left_out_words() {
424 let query = Query::parse(r#"parse "exact phrase" -legacy token"#);
425 assert_eq!(
426 words(&query),
427 vec![("parse", false, false), ("exact phrase", true, false), ("legacy", false, true), ("token", false, false)]
428 );
429 }
430
431 #[test]
432 fn qualifiers_are_read() {
433 let query = Query::parse("fn repo:Acme/Web org:acme language:TS path:src/ is:pr is:open author:@ana label:\"good first issue\"");
434 assert_eq!(words(&query), vec![("fn", false, false)]);
435 assert_eq!(query.repos.include, vec!["acme/web"]);
436 assert_eq!(query.owners.include, vec!["acme"]);
437 assert_eq!(query.languages.include, vec!["typescript"]);
438 assert_eq!(query.paths.include, vec!["src/"]);
439 assert_eq!(query.authors.include, vec!["ana"]);
440 assert_eq!(query.labels.include, vec!["good first issue"]);
441 assert_eq!(query.kind, Some(ItemKind::Pull));
442 assert_eq!(query.state, Some(ItemState::Open));
443 assert!(query.notes.is_empty());
444 }
445
446 #[test]
447 fn workspace_and_org_mean_the_same() {
448 assert_eq!(Query::parse("workspace:acme").owners, Query::parse("org:ACME").owners);
449 }
450
451 #[test]
452 fn qualifiers_can_be_left_out() {
453 let query = Query::parse("-label:wontfix -language:markdown -is:closed");
454 assert_eq!(query.labels.exclude, vec!["wontfix"]);
455 assert_eq!(query.languages.exclude, vec!["markdown"]);
456 assert_eq!(query.state, Some(ItemState::Open));
457 }
458
459 #[test]
460 fn unknown_keys_are_words() {
461 let query = Query::parse("http://example.com std::io");
462 assert_eq!(words(&query), vec![("http://example.com", false, false), ("std::io", false, false)]);
463 }
464
465 #[test]
466 fn mistakes_are_said() {
467 let query = Query::parse("repo:acme is:spaceship label:");
468 assert_eq!(query.notes.len(), 3);
469 assert!(query.repos.is_empty());
470 }
471
472 #[test]
473 fn the_tab_follows_the_qualifiers() {
474 assert_eq!(Query::parse("router").default_type(), SearchType::Repositories);
475 assert_eq!(Query::parse("router path:src").default_type(), SearchType::Code);
476 assert_eq!(Query::parse("crash is:pr").default_type(), SearchType::Pulls);
477 assert_eq!(Query::parse("crash is:open").default_type(), SearchType::Issues);
478 assert_eq!(Query::parse("crash label:bug").default_type(), SearchType::Issues);
479 assert_eq!(Query::parse("crash type:code").default_type(), SearchType::Code);
480 }
481
482 #[test]
483 fn qualifiers_rule_kinds_out() {
484 let code = Query::parse("x path:src");
485 assert!(code.applies_to(SearchType::Code));
486 assert!(!code.applies_to(SearchType::Repositories));
487 assert!(!code.applies_to(SearchType::Issues));
488 let issues = Query::parse("x label:bug");
489 assert!(issues.applies_to(SearchType::Issues) && issues.applies_to(SearchType::Pulls));
490 assert!(!issues.applies_to(SearchType::Code) && !issues.applies_to(SearchType::People));
491 let merged = Query::parse("x is:merged");
492 assert!(merged.applies_to(SearchType::Pulls) && !merged.applies_to(SearchType::Issues));
493 let scoped = Query::parse("x repo:a/b");
494 assert!(!scoped.applies_to(SearchType::People));
495 assert!(Query::parse("ana").applies_to(SearchType::People));
496 }
497
498 #[test]
499 fn prose_matching_is_safe_and_prefixed() {
500 let query = Query::parse(r#"pars "say hi" -old"#);
501 assert_eq!(query.prose_match().unwrap(), r#""pars"* "say hi" NOT "old""#);
502 // A quote inside a word is doubled, never read as syntax.
503 assert_eq!(fts_string(r#"a"b"#), r#""a""b""#);
504 // Punctuation alone matches nothing and is not sent.
505 assert_eq!(Query::parse("++ --").prose_match(), None);
506 // Only words left out: nothing to match on.
507 assert_eq!(Query::parse("-old").prose_match(), None);
508 // Words that look like the index's own syntax are only words.
509 assert_eq!(Query::parse("NEAR OR").prose_match().unwrap(), r#""NEAR"* "OR"*"#);
510 }
511
512 #[test]
513 fn code_matching_needs_three_characters() {
514 let (expression, short) = Query::parse("fn parse_query -old").code_match();
515 assert_eq!(expression.unwrap(), r#""parse_query" NOT "old""#);
516 assert_eq!(short, vec!["fn"]);
517 let (expression, short) = Query::parse("fn io").code_match();
518 assert_eq!(expression, None);
519 assert_eq!(short, vec!["fn", "io"]);
520 }
521
522 #[test]
523 fn paths_become_patterns() {
524 assert_eq!(path_pattern("src/"), "%src/%");
525 assert_eq!(path_pattern("*.rs"), "%.rs");
526 assert_eq!(path_pattern("100%_done"), "%100\\%\\_done%");
527 }
528
529 #[test]
530 fn the_query_is_written_back_tidied() {
531 let query = Query::parse(" Parse REPO:acme/web \"two words\" is:issue ");
532 assert_eq!(query.display(), "Parse \"two words\" repo:acme/web is:issue");
533 }
534
535 #[test]
536 fn long_queries_are_cut() {
537 let query = Query::parse(&"a ".repeat(1000));
538 assert!(query.terms.len() <= MAX_QUERY_CHARS / 2);
539 }
540}