Index work items in search
Search indexes work items per workspace (kind item) and finds them with is:item and type:items; repo: matches any repository an item is linked to and workspace: its workspace. Members of the workspace see all of its items, anyone who reads an item's origin sees those filed there, and items with no origin are for members only, in the query and again on the page. Results link to /{workspace}/-/work/{number}. Events are work_item.* in place of issue.*; work.migrated reads the moved items again by workspace and number, and migration 0003 rekeys the items table (work items by id, pull requests by repository and number), keeps pull requests and drops the old issue rows. A new deployment's backfill lists each workspace's work items.
- G1T-Session:
- ses_01m566c35sewwsq09vknpxzh49
| 1654 | 1654 | dependencies = [ | |
| 1655 | 1655 | "g1t-contracts", | |
| 1656 | 1656 | "g1t-kit", | |
| 1657 | + | "rusqlite", | |
| 1657 | 1658 | "serde", | |
| 1658 | 1659 | "serde_json", | |
| 1659 | 1660 | "worker", |
| 3 | 3 | version = "0.1.0" | |
| 4 | 4 | edition.workspace = true | |
| 5 | 5 | license.workspace = true | |
| 6 | − | description = "Search across all of g1t: repositories, code, issues, pull requests, people and workspaces; and Explore." | |
| 6 | + | description = "Search across all of g1t: repositories, code, work items, pull requests, people and workspaces; and Explore." | |
| 7 | 7 | ||
| 8 | 8 | [lib] | |
| 9 | 9 | crate-type = ["cdylib"] | |
| ⋯ | |||
| 15 | 15 | serde_json.workspace = true | |
| 16 | 16 | worker.workspace = true | |
| 17 | 17 | ||
| 18 | + | [dev-dependencies] | |
| 19 | + | rusqlite = { version = "0.32", features = ["bundled"] } | |
| 20 | + | ||
| 18 | 21 | # wasm-opt at -O1: about the same gzipped size as -O in a tenth of the | |
| 19 | 22 | # time. | |
| 20 | 23 | [package.metadata.wasm-pack.profile.release] | |
| 1 | + | -- Issues became work items of a workspace (the work service's | |
| 2 | + | -- 0049_work_items). Items are kept by key: a work item by its id, a pull | |
| 3 | + | -- request by its repository and number. A work item keeps its workspace, | |
| 4 | + | -- its number there, its origin (where it was filed, if anywhere) and every | |
| 5 | + | -- repository it is linked to. | |
| 6 | + | -- | |
| 7 | + | -- Pull requests are copied as they are. Issues are dropped: they are | |
| 8 | + | -- indexed again as work items, by id, when the work service's move | |
| 9 | + | -- announces them (`work.migrated`), with their workspace and new number. | |
| 10 | + | ||
| 11 | + | DROP TRIGGER items_ai; | |
| 12 | + | DROP TRIGGER items_ad; | |
| 13 | + | DROP TRIGGER items_au; | |
| 14 | + | DROP TABLE items_fts; | |
| 15 | + | ||
| 16 | + | CREATE TABLE items_next ( | |
| 17 | + | iid INTEGER PRIMARY KEY, | |
| 18 | + | -- item or pull. | |
| 19 | + | kind TEXT NOT NULL, | |
| 20 | + | -- A work item's id; a pull request's repository id and number, `rep_…/12`. | |
| 21 | + | key TEXT NOT NULL UNIQUE, | |
| 22 | + | -- A pull request's repository; a work item's origin, if it has one. | |
| 23 | + | repo_id TEXT, | |
| 24 | + | -- A work item's workspace, lowercase. | |
| 25 | + | workspace TEXT, | |
| 26 | + | number INTEGER NOT NULL, | |
| 27 | + | title TEXT NOT NULL, | |
| 28 | + | body TEXT NOT NULL DEFAULT '', | |
| 29 | + | -- open or closed. | |
| 30 | + | state TEXT NOT NULL, | |
| 31 | + | -- A work item: open, completed or not_planned. A pull request: draft, | |
| 32 | + | -- open, merged or closed. | |
| 33 | + | status TEXT NOT NULL, | |
| 34 | + | -- Lowercase username. | |
| 35 | + | author TEXT NOT NULL, | |
| 36 | + | requested_by TEXT, | |
| 37 | + | -- Lowercase, each between bars: |bug|good first issue|. | |
| 38 | + | labels TEXT NOT NULL DEFAULT '|', | |
| 39 | + | -- A work item's linked repositories by id, each between bars: | |
| 40 | + | -- |rep_1|rep_2|. | |
| 41 | + | repos TEXT NOT NULL DEFAULT '|', | |
| 42 | + | created_at TEXT NOT NULL, | |
| 43 | + | updated_at TEXT NOT NULL | |
| 44 | + | ); | |
| 45 | + | ||
| 46 | + | INSERT INTO items_next | |
| 47 | + | (iid, kind, key, repo_id, workspace, number, title, body, state, status, author, requested_by, labels, repos, | |
| 48 | + | created_at, updated_at) | |
| 49 | + | SELECT iid, kind, repo_id || '/' || number, repo_id, NULL, number, title, body, state, status, author, requested_by, | |
| 50 | + | labels, '|', created_at, updated_at | |
| 51 | + | FROM items WHERE kind = 'pull'; | |
| 52 | + | ||
| 53 | + | DROP TABLE items; | |
| 54 | + | ALTER TABLE items_next RENAME TO items; | |
| 55 | + | ||
| 56 | + | CREATE INDEX items_recent ON items (kind, updated_at); | |
| 57 | + | CREATE INDEX items_by_author ON items (author); | |
| 58 | + | CREATE INDEX items_by_repo ON items (repo_id); | |
| 59 | + | CREATE INDEX items_by_workspace ON items (workspace, number); | |
| 60 | + | ||
| 61 | + | CREATE VIRTUAL TABLE items_fts USING fts5( | |
| 62 | + | title, body, | |
| 63 | + | content = 'items', content_rowid = 'iid', | |
| 64 | + | tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3' | |
| 65 | + | ); | |
| 66 | + | INSERT INTO items_fts (items_fts) VALUES ('rebuild'); | |
| 67 | + | CREATE TRIGGER items_ai AFTER INSERT ON items BEGIN | |
| 68 | + | INSERT INTO items_fts (rowid, title, body) VALUES (new.iid, new.title, new.body); | |
| 69 | + | END; | |
| 70 | + | CREATE TRIGGER items_ad AFTER DELETE ON items BEGIN | |
| 71 | + | INSERT INTO items_fts (items_fts, rowid, title, body) VALUES ('delete', old.iid, old.title, old.body); | |
| 72 | + | END; | |
| 73 | + | CREATE TRIGGER items_au AFTER UPDATE OF title, body ON items BEGIN | |
| 74 | + | INSERT INTO items_fts (items_fts, rowid, title, body) VALUES ('delete', old.iid, old.title, old.body); | |
| 75 | + | INSERT INTO items_fts (rowid, title, body) VALUES (new.iid, new.title, new.body); | |
| 76 | + | END; |
| 11 | 11 | ||
| 12 | 12 | use std::collections::{HashMap, HashSet}; | |
| 13 | 13 | ||
| 14 | − | use g1t_contracts::events::{Event, ProjectUpdated}; | |
| 14 | + | use g1t_contracts::events::{Event, ProjectUpdated, WorkMigrated}; | |
| 15 | 15 | use g1t_contracts::identity::{DirectoryArgs, DirectoryEntry, DirectoryPage, Profile, SlugArgs, UsernameArgs, Workspace}; | |
| 16 | 16 | use g1t_contracts::repos::{ | |
| 17 | 17 | AllIdsArgs, BlobText, ChangedFilesArgs, FileList, IdPage, ListFilesArgs, PathByIdArgs, ReadBlobsArgs, RepoPath, TreeArgs, | |
| ⋯ | |||
| 19 | 19 | }; | |
| 20 | 20 | use g1t_contracts::time::rfc3339; | |
| 21 | 21 | use g1t_contracts::work::{ | |
| 22 | − | Issue, IssueDetail, ListIssuesArgs, ListPullsArgs, Pull, PullDetail, State, ViewArgs, | |
| 22 | + | ListPullsArgs, ListWorkItemsArgs, Pull, PullDetail, State, ViewArgs, WorkItem, WorkItemDetail, WorkItemViewArgs, | |
| 23 | 23 | }; | |
| 24 | 24 | use g1t_contracts::{Membership, Outcome, PrincipalKind, User}; | |
| 25 | 25 | use g1t_kit::now_ms; | |
| ⋯ | |||
| 44 | 44 | const READ_GROUP: usize = 20; | |
| 45 | 45 | /// Pieces of a file written in one statement. | |
| 46 | 46 | const PIECES_PER_STATEMENT: usize = 8; | |
| 47 | − | /// The most of an issue's or pull request's body indexed. | |
| 47 | + | /// The most of a work item's or pull request's body indexed. | |
| 48 | 48 | const BODY_CHARS: usize = 32 * 1024; | |
| 49 | 49 | /// The most of a README indexed. | |
| 50 | 50 | const README_CHARS: usize = 4000; | |
| ⋯ | |||
| 52 | 52 | const BACKFILL_PAGE: u32 = 100; | |
| 53 | 53 | /// Accounts or workspaces per directory job. | |
| 54 | 54 | const DIRECTORY_PAGE: u32 = 200; | |
| 55 | + | /// Work items read one at a time in one job, at most. | |
| 56 | + | const ITEMS_PER_JOB: usize = 20; | |
| 57 | + | ||
| 58 | + | /// Stores a work item or pull request by its key, as it is now. | |
| 59 | + | pub const UPSERT_ITEM: &str = "INSERT INTO items (kind, key, repo_id, workspace, number, repos, title, body, state, status, | |
| 60 | + | author, requested_by, labels, created_at, updated_at) | |
| 61 | + | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) | |
| 62 | + | ON CONFLICT (key) DO UPDATE SET repo_id = excluded.repo_id, workspace = excluded.workspace, | |
| 63 | + | number = excluded.number, repos = excluded.repos, title = excluded.title, body = excluded.body, | |
| 64 | + | state = excluded.state, status = excluded.status, author = excluded.author, | |
| 65 | + | requested_by = excluded.requested_by, labels = excluded.labels, updated_at = excluded.updated_at"; | |
| 55 | 66 | ||
| 56 | 67 | /// Where `park` keeps a backfill page in `meta`, by its key's prefix. | |
| 57 | 68 | const PARKED: &str = "parked_job:"; | |
| ⋯ | |||
| 76 | 87 | after: Option<String>, | |
| 77 | 88 | }, | |
| 78 | 89 | /// Compares a repository's default branch with the index, whole, and | |
| 79 | − | /// indexes its issues and pull requests. | |
| 90 | + | /// indexes its pull requests. | |
| 80 | 91 | Repo { repo_id: String }, | |
| 81 | 92 | /// Indexes what a push to the default branch changed. | |
| 82 | 93 | Push { | |
| ⋯ | |||
| 93 | 104 | #[serde(default)] | |
| 94 | 105 | after: Option<String>, | |
| 95 | 106 | }, | |
| 107 | + | /// Indexes a workspace's work items, as many as one list returns. | |
| 108 | + | Workspace { workspace: String }, | |
| 109 | + | /// Indexes these work items of a workspace, one at a time. | |
| 110 | + | Items { workspace: String, numbers: Vec<u32> }, | |
| 96 | 111 | } | |
| 97 | 112 | ||
| 98 | 113 | /// What this service reads of the events it handles. | |
| ⋯ | |||
| 121 | 136 | } | |
| 122 | 137 | ||
| 123 | 138 | #[derive(Deserialize)] | |
| 139 | + | #[serde(rename_all = "camelCase")] | |
| 140 | + | struct ItemEvent { | |
| 141 | + | item_id: String, | |
| 142 | + | workspace: String, | |
| 143 | + | number: u32, | |
| 144 | + | } | |
| 145 | + | ||
| 146 | + | #[derive(Deserialize)] | |
| 124 | 147 | struct UserEvent { | |
| 125 | 148 | username: String, | |
| 126 | 149 | } | |
| ⋯ | |||
| 147 | 170 | state: String, | |
| 148 | 171 | } | |
| 149 | 172 | ||
| 173 | + | /// Where a row of `items` belongs and what it is linked to. | |
| 174 | + | struct Row<'a> { | |
| 175 | + | kind: &'a str, | |
| 176 | + | key: &'a str, | |
| 177 | + | repo_id: Option<&'a str>, | |
| 178 | + | workspace: Option<&'a str>, | |
| 179 | + | number: u32, | |
| 180 | + | repos: &'a str, | |
| 181 | + | } | |
| 182 | + | ||
| 183 | + | /// What a row of `items` says. | |
| 184 | + | struct Text<'a> { | |
| 185 | + | title: &'a str, | |
| 186 | + | body: &'a str, | |
| 187 | + | state: &'a str, | |
| 188 | + | status: &'a str, | |
| 189 | + | /// Who opened it, and for g1t's work, who asked for it. | |
| 190 | + | author: &'a str, | |
| 191 | + | requested_by: Option<&'a str>, | |
| 192 | + | labels: &'a [String], | |
| 193 | + | created_at: &'a str, | |
| 194 | + | updated_at: &'a str, | |
| 195 | + | } | |
| 196 | + | ||
| 150 | 197 | fn now() -> String { | |
| 151 | 198 | rfc3339(now_ms()) | |
| 152 | 199 | } | |
| ⋯ | |||
| 173 | 220 | text.chars().take(max).collect() | |
| 174 | 221 | } | |
| 175 | 222 | ||
| 223 | + | /// Repository ids between bars, as `items.repos` keeps them: `|rep_1|rep_2|`. | |
| 224 | + | fn repos_of(ids: &[&str]) -> String { | |
| 225 | + | let mut out = String::from("|"); | |
| 226 | + | for id in ids { | |
| 227 | + | out.push_str(id); | |
| 228 | + | out.push('|'); | |
| 229 | + | } | |
| 230 | + | out | |
| 231 | + | } | |
| 232 | + | ||
| 233 | + | /// A pull request's key in `items`. | |
| 234 | + | fn pull_key(repo_id: &str, number: u32) -> String { | |
| 235 | + | format!("{repo_id}/{number}") | |
| 236 | + | } | |
| 237 | + | ||
| 238 | + | /// The jobs that index the work items `work.migrated` announces: grouped | |
| 239 | + | /// by workspace, a few to a job. | |
| 240 | + | fn migrated_jobs(migrated: &WorkMigrated) -> Vec<Job> { | |
| 241 | + | let mut by_workspace: Vec<(String, Vec<u32>)> = Vec::new(); | |
| 242 | + | for item in &migrated.items { | |
| 243 | + | let workspace = item.workspace.to_lowercase(); | |
| 244 | + | match by_workspace.iter_mut().find(|(slug, _)| *slug == workspace) { | |
| 245 | + | Some((_, numbers)) => numbers.push(item.number), | |
| 246 | + | None => by_workspace.push((workspace, vec![item.number])), | |
| 247 | + | } | |
| 248 | + | } | |
| 249 | + | let mut jobs = Vec::new(); | |
| 250 | + | for (workspace, numbers) in by_workspace { | |
| 251 | + | for group in numbers.chunks(ITEMS_PER_JOB) { | |
| 252 | + | jobs.push(Job::Items { workspace: workspace.clone(), numbers: group.to_vec() }); | |
| 253 | + | } | |
| 254 | + | } | |
| 255 | + | jobs | |
| 256 | + | } | |
| 257 | + | ||
| 176 | 258 | impl Search { | |
| 177 | 259 | /// The workspace itself, through no one: how this service reads a | |
| 178 | 260 | /// repository, private or not, to index it. | |
| ⋯ | |||
| 272 | 354 | store::prepare(&self.db, "DELETE FROM chunks WHERE fid IN (SELECT fid FROM files WHERE repo_id = ?)", id())?, | |
| 273 | 355 | store::prepare(&self.db, "DELETE FROM files WHERE repo_id = ?", id())?, | |
| 274 | 356 | store::prepare(&self.db, "DELETE FROM pending WHERE repo_id = ?", id())?, | |
| 275 | − | store::prepare(&self.db, "DELETE FROM items WHERE repo_id = ?", id())?, | |
| 357 | + | // Its work items stay: they are their workspace's, now for | |
| 358 | + | // its members only. | |
| 359 | + | store::prepare(&self.db, "DELETE FROM items WHERE kind = 'pull' AND repo_id = ?", id())?, | |
| 276 | 360 | store::prepare(&self.db, "DELETE FROM repo_topics WHERE repo_id = ?", id())?, | |
| 277 | 361 | store::prepare(&self.db, "DELETE FROM repos WHERE repo_id = ?", id())?, | |
| 278 | 362 | ], | |
| ⋯ | |||
| 517 | 601 | Ok(statements) | |
| 518 | 602 | } | |
| 519 | 603 | ||
| 520 | − | // ---- Issues and pull requests ------------------------------------------ | |
| 604 | + | // ---- Work items and pull requests ------------------------------------- | |
| 521 | 605 | ||
| 522 | − | fn issue_row(&self, repo_id: &str, issue: &Issue) -> Result<worker::D1PreparedStatement> { | |
| 523 | − | let status = match issue.state { | |
| 606 | + | fn work_item_row(&self, item: &WorkItem) -> Result<worker::D1PreparedStatement> { | |
| 607 | + | let status = match item.state { | |
| 524 | 608 | State::Open => "open", | |
| 525 | − | State::Closed => issue.reason.map_or("completed", |reason| reason.as_str()), | |
| 609 | + | State::Closed => item.reason.map_or("completed", |reason| reason.as_str()), | |
| 526 | 610 | }; | |
| 611 | + | let linked: Vec<&str> = item.repos.iter().map(|repo| repo.repo_id.as_str()).collect(); | |
| 527 | 612 | self.item_row( | |
| 528 | − | repo_id, | |
| 529 | − | "issue", | |
| 530 | − | issue.number, | |
| 531 | − | &issue.title, | |
| 532 | − | &issue.body, | |
| 533 | − | if issue.state == State::Open { "open" } else { "closed" }, | |
| 534 | − | status, | |
| 535 | − | (&issue.author.username, issue.requested_by.as_ref().map(|user| user.username.as_str())), | |
| 536 | − | &issue.labels, | |
| 537 | − | &issue.created_at, | |
| 538 | − | &issue.updated_at, | |
| 613 | + | Row { | |
| 614 | + | kind: "item", | |
| 615 | + | key: &item.id, | |
| 616 | + | repo_id: item.origin().map(|repo| repo.repo_id.as_str()), | |
| 617 | + | workspace: Some(&item.workspace.to_lowercase()), | |
| 618 | + | number: item.number, | |
| 619 | + | repos: &repos_of(&linked), | |
| 620 | + | }, | |
| 621 | + | Text { | |
| 622 | + | title: &item.title, | |
| 623 | + | body: &item.body, | |
| 624 | + | state: if item.state == State::Open { "open" } else { "closed" }, | |
| 625 | + | status, | |
| 626 | + | author: &item.author.username, | |
| 627 | + | requested_by: item.requested_by.as_ref().map(|user| user.username.as_str()), | |
| 628 | + | labels: &item.labels, | |
| 629 | + | created_at: &item.created_at, | |
| 630 | + | updated_at: &item.updated_at, | |
| 631 | + | }, | |
| 539 | 632 | ) | |
| 540 | 633 | } | |
| 541 | 634 | ||
| 542 | − | fn pull_row(&self, repo_id: &str, pull: &Pull, labels: &[String]) -> Result<worker::D1PreparedStatement> { | |
| 635 | + | fn pull_row(&self, repo_id: &str, pull: &Pull) -> Result<worker::D1PreparedStatement> { | |
| 543 | 636 | self.item_row( | |
| 544 | − | repo_id, | |
| 545 | − | "pull", | |
| 546 | − | pull.number, | |
| 547 | − | &pull.title, | |
| 548 | − | pull.body.as_deref().unwrap_or_default(), | |
| 549 | − | if pull.status.is_active() { "open" } else { "closed" }, | |
| 550 | − | pull.status.as_str(), | |
| 551 | − | (&pull.author.username, pull.requested_by.as_ref().map(|user| user.username.as_str())), | |
| 552 | − | labels, | |
| 553 | − | &pull.created_at, | |
| 554 | − | &pull.updated_at, | |
| 637 | + | Row { | |
| 638 | + | kind: "pull", | |
| 639 | + | key: &pull_key(repo_id, pull.number), | |
| 640 | + | repo_id: Some(repo_id), | |
| 641 | + | workspace: None, | |
| 642 | + | number: pull.number, | |
| 643 | + | repos: "|", | |
| 644 | + | }, | |
| 645 | + | Text { | |
| 646 | + | title: &pull.title, | |
| 647 | + | body: pull.body.as_deref().unwrap_or_default(), | |
| 648 | + | state: if pull.status.is_active() { "open" } else { "closed" }, | |
| 649 | + | status: pull.status.as_str(), | |
| 650 | + | author: &pull.author.username, | |
| 651 | + | requested_by: pull.requested_by.as_ref().map(|user| user.username.as_str()), | |
| 652 | + | labels: &pull.labels, | |
| 653 | + | created_at: &pull.created_at, | |
| 654 | + | updated_at: &pull.updated_at, | |
| 655 | + | }, | |
| 555 | 656 | ) | |
| 556 | 657 | } | |
| 557 | 658 | ||
| 558 | − | #[allow(clippy::too_many_arguments)] | |
| 559 | − | fn item_row( | |
| 560 | − | &self, | |
| 561 | − | repo_id: &str, | |
| 562 | − | kind: &str, | |
| 563 | − | number: u32, | |
| 564 | − | title: &str, | |
| 565 | − | body: &str, | |
| 566 | − | state: &str, | |
| 567 | − | status: &str, | |
| 568 | − | // Who opened it, and for g1t's work, who asked for it. | |
| 569 | − | (author, requested_by): (&str, Option<&str>), | |
| 570 | − | labels: &[String], | |
| 571 | − | created_at: &str, | |
| 572 | − | updated_at: &str, | |
| 573 | − | ) -> Result<worker::D1PreparedStatement> { | |
| 659 | + | fn item_row(&self, row: Row<'_>, text: Text<'_>) -> Result<worker::D1PreparedStatement> { | |
| 574 | 660 | store::prepare( | |
| 575 | 661 | &self.db, | |
| 576 | − | "INSERT INTO items (repo_id, kind, number, title, body, state, status, author, requested_by, labels, created_at, updated_at) | |
| 577 | − | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) | |
| 578 | − | ON CONFLICT (repo_id, kind, number) DO UPDATE SET title = excluded.title, body = excluded.body, | |
| 579 | − | state = excluded.state, status = excluded.status, author = excluded.author, | |
| 580 | − | requested_by = excluded.requested_by, labels = excluded.labels, updated_at = excluded.updated_at", | |
| 662 | + | UPSERT_ITEM, | |
| 581 | 663 | vec![ | |
| 582 | − | p(repo_id), | |
| 583 | − | p(kind), | |
| 584 | − | Param::Int(i64::from(number)), | |
| 585 | − | p(title), | |
| 586 | − | p(&clip(body, BODY_CHARS)), | |
| 587 | − | p(state), | |
| 588 | − | p(status), | |
| 589 | − | p(&author.to_lowercase()), | |
| 590 | − | requested_by.map_or(Param::Null, |name| p(&name.to_lowercase())), | |
| 591 | − | p(&labels_of(labels)), | |
| 592 | − | p(created_at), | |
| 593 | − | p(updated_at), | |
| 664 | + | p(row.kind), | |
| 665 | + | p(row.key), | |
| 666 | + | Param::opt(row.repo_id), | |
| 667 | + | Param::opt(row.workspace), | |
| 668 | + | Param::Int(i64::from(row.number)), | |
| 669 | + | p(row.repos), | |
| 670 | + | p(text.title), | |
| 671 | + | p(&clip(text.body, BODY_CHARS)), | |
| 672 | + | p(text.state), | |
| 673 | + | p(text.status), | |
| 674 | + | p(&text.author.to_lowercase()), | |
| 675 | + | text.requested_by.map_or(Param::Null, |name| p(&name.to_lowercase())), | |
| 676 | + | p(&labels_of(text.labels)), | |
| 677 | + | p(text.created_at), | |
| 678 | + | p(text.updated_at), | |
| 594 | 679 | ], | |
| 595 | 680 | ) | |
| 596 | 681 | } | |
| 597 | 682 | ||
| 598 | − | /// Indexes one issue or pull request as it is now. | |
| 599 | − | async fn index_item(&self, pull: bool, repo_id: &str, number: u32) -> Result<()> { | |
| 683 | + | async fn known(&self, repo_id: &str) -> Result<bool> { | |
| 684 | + | let found = store::count( | |
| 685 | + | &self.db, | |
| 686 | + | &Sql { text: "SELECT count(*) AS n FROM repos WHERE repo_id = ?".into(), params: vec![p(repo_id)] }, | |
| 687 | + | ) | |
| 688 | + | .await?; | |
| 689 | + | Ok(found > 0) | |
| 690 | + | } | |
| 691 | + | ||
| 692 | + | /// Indexes one pull request as it is now. | |
| 693 | + | async fn index_pull(&self, repo_id: &str, number: u32) -> Result<()> { | |
| 600 | 694 | let Some(path) = self.path_of(repo_id).await? else { | |
| 601 | 695 | return Ok(()); | |
| 602 | 696 | }; | |
| 603 | 697 | let Some(actor) = self.actor(&path.namespace).await? else { | |
| 604 | 698 | return Ok(()); | |
| 605 | 699 | }; | |
| 606 | − | // An issue on a repository not yet indexed brings the repository in. | |
| 607 | − | let known = store::count( | |
| 608 | − | &self.db, | |
| 609 | − | &Sql { text: "SELECT count(*) AS n FROM repos WHERE repo_id = ?".into(), params: vec![p(repo_id)] }, | |
| 610 | − | ) | |
| 611 | − | .await?; | |
| 612 | − | if known == 0 && self.refresh_repo(repo_id).await?.is_none() { | |
| 700 | + | // A pull request on a repository not yet indexed brings the repository in. | |
| 701 | + | if !self.known(repo_id).await? && self.refresh_repo(repo_id).await?.is_none() { | |
| 613 | 702 | return Ok(()); | |
| 614 | 703 | } | |
| 615 | 704 | let view = ViewArgs { repo: path, number, viewer: Some(actor), after_seq: 0 }; | |
| 616 | − | let statement = if pull { | |
| 617 | − | let found: Outcome<PullDetail> = g1t_kit::call(&self.work, "get_pull", &view).await?; | |
| 618 | − | let Outcome::Ok(detail) = found else { return Ok(()) }; | |
| 619 | − | let labels = detail.issue.as_ref().map(|issue| issue.labels.clone()).unwrap_or_default(); | |
| 620 | − | self.pull_row(repo_id, &detail.pull, &labels)? | |
| 621 | − | } else { | |
| 622 | − | let found: Outcome<IssueDetail> = g1t_kit::call(&self.work, "get_issue", &view).await?; | |
| 623 | − | let Outcome::Ok(detail) = found else { return Ok(()) }; | |
| 624 | − | self.issue_row(repo_id, &detail.issue)? | |
| 705 | + | let found: Outcome<PullDetail> = g1t_kit::call(&self.work, "get_pull", &view).await?; | |
| 706 | + | let Outcome::Ok(detail) = found else { return Ok(()) }; | |
| 707 | + | store::run_all(&self.db, vec![self.pull_row(repo_id, &detail.pull)?]).await | |
| 708 | + | } | |
| 709 | + | ||
| 710 | + | /// Brings in the repository a work item was filed in, if the index | |
| 711 | + | /// does not have it yet: who may read the item follows it. | |
| 712 | + | async fn ensure_origin(&self, item: &WorkItem) -> Result<()> { | |
| 713 | + | if let Some(origin) = item.origin() | |
| 714 | + | && !self.known(&origin.repo_id).await? | |
| 715 | + | { | |
| 716 | + | self.refresh_repo(&origin.repo_id).await?; | |
| 717 | + | } | |
| 718 | + | Ok(()) | |
| 719 | + | } | |
| 720 | + | ||
| 721 | + | /// Indexes one work item as it is now. One no longer found is | |
| 722 | + | /// forgotten, when its id is known. | |
| 723 | + | async fn index_work_item(&self, workspace: &str, number: u32, item_id: Option<&str>) -> Result<()> { | |
| 724 | + | if workspace.is_empty() || number == 0 { | |
| 725 | + | return Ok(()); | |
| 726 | + | } | |
| 727 | + | let Some(actor) = self.actor(workspace).await? else { | |
| 728 | + | return Ok(()); | |
| 625 | 729 | }; | |
| 730 | + | let view = WorkItemViewArgs { workspace: workspace.to_lowercase(), number, viewer: Some(actor) }; | |
| 731 | + | let found: Outcome<WorkItemDetail> = g1t_kit::call(&self.work, "get_work_item", &view).await?; | |
| 732 | + | let statement = match found { | |
| 733 | + | Outcome::Ok(detail) => { | |
| 734 | + | self.ensure_origin(&detail.item).await?; | |
| 735 | + | self.work_item_row(&detail.item)? | |
| 736 | + | } | |
| 737 | + | Outcome::Fail(_) => match item_id { | |
| 738 | + | Some(id) => store::prepare(&self.db, "DELETE FROM items WHERE kind = 'item' AND key = ?", vec![p(id)])?, | |
| 739 | + | None => return Ok(()), | |
| 740 | + | }, | |
| 741 | + | }; | |
| 626 | 742 | store::run_all(&self.db, vec![statement]).await | |
| 627 | 743 | } | |
| 628 | 744 | ||
| 629 | − | /// Indexes a repository's issues and pull requests, newest first, as | |
| 630 | − | /// many as one list returns. | |
| 631 | − | async fn index_items(&self, repo_id: &str, path: &RepoPath) -> Result<()> { | |
| 632 | − | let Some(actor) = self.actor(&path.namespace).await? else { | |
| 745 | + | /// Indexes a workspace's work items, newest first, as many as one list | |
| 746 | + | /// returns. | |
| 747 | + | async fn index_work_items(&self, workspace: &str) -> Result<()> { | |
| 748 | + | let Some(actor) = self.actor(workspace).await? else { | |
| 633 | 749 | return Ok(()); | |
| 634 | 750 | }; | |
| 635 | − | let issues: Outcome<Vec<Issue>> = g1t_kit::call( | |
| 751 | + | let items: Outcome<Vec<WorkItem>> = g1t_kit::call( | |
| 636 | 752 | &self.work, | |
| 637 | − | "list_issues", | |
| 638 | − | &ListIssuesArgs { repo: path.clone(), viewer: Some(actor.clone()), state: None, label: None, milestone: None }, | |
| 753 | + | "list_work_items", | |
| 754 | + | &ListWorkItemsArgs { | |
| 755 | + | workspace: workspace.to_lowercase(), | |
| 756 | + | viewer: Some(actor), | |
| 757 | + | repo: None, | |
| 758 | + | role: None, | |
| 759 | + | state: None, | |
| 760 | + | label: None, | |
| 761 | + | milestone: None, | |
| 762 | + | assignee: None, | |
| 763 | + | }, | |
| 639 | 764 | ) | |
| 640 | 765 | .await?; | |
| 766 | + | let Outcome::Ok(items) = items else { return Ok(()) }; | |
| 767 | + | let mut statements = Vec::new(); | |
| 768 | + | for item in items.iter().filter(|item| item.number > 0) { | |
| 769 | + | self.ensure_origin(item).await?; | |
| 770 | + | statements.push(self.work_item_row(item)?); | |
| 771 | + | } | |
| 772 | + | store::run_all(&self.db, statements).await | |
| 773 | + | } | |
| 774 | + | ||
| 775 | + | /// Indexes a repository's pull requests, newest first, as many as one | |
| 776 | + | /// list returns. | |
| 777 | + | async fn index_pulls(&self, repo_id: &str, path: &RepoPath) -> Result<()> { | |
| 778 | + | let Some(actor) = self.actor(&path.namespace).await? else { | |
| 779 | + | return Ok(()); | |
| 780 | + | }; | |
| 641 | 781 | let pulls: Outcome<Vec<Pull>> = g1t_kit::call( | |
| 642 | 782 | &self.work, | |
| 643 | 783 | "list_pulls", | |
| 644 | 784 | &ListPullsArgs { repo: path.clone(), viewer: Some(actor), state: None, label: None, milestone: None, base: None }, | |
| 645 | 785 | ) | |
| 646 | 786 | .await?; | |
| 647 | − | let issues = match issues { | |
| 648 | − | Outcome::Ok(issues) => issues, | |
| 649 | − | Outcome::Fail(_) => Vec::new(), | |
| 650 | − | }; | |
| 651 | − | let labels: HashMap<u32, &Vec<String>> = issues.iter().map(|issue| (issue.number, &issue.labels)).collect(); | |
| 787 | + | let Outcome::Ok(pulls) = pulls else { return Ok(()) }; | |
| 652 | 788 | let mut statements = Vec::new(); | |
| 653 | − | for issue in &issues { | |
| 654 | − | statements.push(self.issue_row(repo_id, issue)?); | |
| 655 | − | } | |
| 656 | − | if let Outcome::Ok(pulls) = pulls { | |
| 657 | − | for pull in &pulls { | |
| 658 | − | let labels = pull.issue.and_then(|number| labels.get(&number)).map(|l| l.as_slice()).unwrap_or_default(); | |
| 659 | − | statements.push(self.pull_row(repo_id, pull, labels)?); | |
| 660 | − | } | |
| 789 | + | for pull in &pulls { | |
| 790 | + | statements.push(self.pull_row(repo_id, pull)?); | |
| 661 | 791 | } | |
| 662 | 792 | store::run_all(&self.db, statements).await | |
| 663 | 793 | } | |
| ⋯ | |||
| 794 | 924 | } | |
| 795 | 925 | } | |
| 796 | 926 | // A transfer changes the repository's path; refreshing reads it | |
| 797 | − | // again by id, issues and pull requests with it. | |
| 927 | + | // again by id. | |
| 798 | 928 | // Archiving changes nothing searched, but the details are read again. | |
| 799 | 929 | "repo.created" | |
| 800 | 930 | | "repo.updated" | |
| ⋯ | |||
| 824 | 954 | self.purge(&repo.repo_id).await?; | |
| 825 | 955 | } | |
| 826 | 956 | } | |
| 827 | − | // A restored one is indexed again, whole: its code, issues and | |
| 828 | − | // pull requests. | |
| 957 | + | // A restored one is indexed again, whole: its code and pull | |
| 958 | + | // requests. | |
| 829 | 959 | "repo.restored" => { | |
| 830 | 960 | if let Ok(repo) = serde_json::from_value::<RepoEvent>(data.clone()) { | |
| 831 | 961 | self.enqueue(vec![Job::Repo { repo_id: repo.repo_id }]).await?; | |
| 832 | 962 | } | |
| 833 | 963 | } | |
| 834 | − | "issue.opened" | "issue.updated" | "issue.closed" | "issue.reopened" | "issue.assigned" => { | |
| 835 | − | if let Ok(item) = serde_json::from_value::<NumberEvent>(data.clone()) { | |
| 836 | − | self.index_item(false, &item.repo_id, item.number).await?; | |
| 964 | + | // Every change to a work item: its text, state, labels and the | |
| 965 | + | // repositories it is linked to are all indexed. | |
| 966 | + | kind if kind.starts_with("work_item.") => { | |
| 967 | + | if let Ok(item) = serde_json::from_value::<ItemEvent>(data.clone()) { | |
| 968 | + | self.index_work_item(&item.workspace, item.number, Some(&item.item_id)).await?; | |
| 969 | + | } | |
| 970 | + | } | |
| 971 | + | // Issues moved into work items keep their ids: each is read | |
| 972 | + | // again with its workspace and new number. The SQL migration | |
| 973 | + | // dropped what was kept of them as issues. | |
| 974 | + | "work.migrated" => { | |
| 975 | + | if let Ok(migrated) = serde_json::from_value::<WorkMigrated>(data.clone()) { | |
| 976 | + | self.enqueue(migrated_jobs(&migrated)).await?; | |
| 837 | 977 | } | |
| 838 | 978 | } | |
| 839 | 979 | "pull.opened" | "pull.ready" | "pull.updated" | "pull.closed" | "pull.reopened" | |
| 840 | − | | "pull.converted_to_draft" | "pull.merged" => { | |
| 841 | − | if let Ok(item) = serde_json::from_value::<NumberEvent>(data.clone()) { | |
| 842 | − | self.index_item(true, &item.repo_id, item.number).await?; | |
| 980 | + | | "pull.converted_to_draft" | "pull.merged" | "pull.labeled" | "pull.unlabeled" => { | |
| 981 | + | if let Ok(pull) = serde_json::from_value::<NumberEvent>(data.clone()) { | |
| 982 | + | self.index_pull(&pull.repo_id, pull.number).await?; | |
| 843 | 983 | } | |
| 844 | 984 | } | |
| 845 | 985 | // A deleted account's profile is not found, so indexing it | |
| ⋯ | |||
| 886 | 1026 | event, | |
| 887 | 1027 | &[ | |
| 888 | 1028 | "UPDATE repos SET namespace = ?1 WHERE namespace = ?2", | |
| 1029 | + | "UPDATE items SET workspace = ?1 WHERE workspace = ?2", | |
| 889 | 1030 | "UPDATE people SET slug = ?1 WHERE kind = 'workspace' AND slug = ?2", | |
| 890 | 1031 | ], | |
| 891 | 1032 | ) | |
| ⋯ | |||
| 920 | 1061 | ) | |
| 921 | 1062 | .await?; | |
| 922 | 1063 | } | |
| 923 | − | self.index_items(&repo_id, &path).await?; | |
| 1064 | + | self.index_pulls(&repo_id, &path).await?; | |
| 924 | 1065 | self.reconcile(&repo_id).await | |
| 925 | 1066 | } | |
| 926 | 1067 | Job::Push { repo_id, before, after } => { | |
| ⋯ | |||
| 976 | 1117 | statements.push(self.person_row(&kind, &reference, entry)?); | |
| 977 | 1118 | } | |
| 978 | 1119 | store::run_all(&self.db, statements).await?; | |
| 1120 | + | // Each workspace's work items, a job each. | |
| 1121 | + | let mut jobs: Vec<Job> = if kind == "workspace" { | |
| 1122 | + | page.entries.iter().map(|entry| Job::Workspace { workspace: entry.slug.to_lowercase() }).collect() | |
| 1123 | + | } else { | |
| 1124 | + | Vec::new() | |
| 1125 | + | }; | |
| 979 | 1126 | if let Some(next) = page.next { | |
| 980 | − | self.enqueue(vec![Job::Directory { kind, after: Some(next) }]).await?; | |
| 1127 | + | jobs.push(Job::Directory { kind, after: Some(next) }); | |
| 981 | 1128 | } | |
| 1129 | + | self.enqueue(jobs).await | |
| 1130 | + | } | |
| 1131 | + | Job::Workspace { workspace } => self.index_work_items(&workspace).await, | |
| 1132 | + | Job::Items { workspace, numbers } => { | |
| 1133 | + | for number in numbers { | |
| 1134 | + | self.index_work_item(&workspace, number, None).await?; | |
| 1135 | + | } | |
| 982 | 1136 | Ok(()) | |
| 983 | 1137 | } | |
| 984 | 1138 | } | |
| ⋯ | |||
| 1023 | 1177 | } | |
| 1024 | 1178 | ||
| 1025 | 1179 | #[test] | |
| 1180 | + | fn migrated_items_are_read_again_by_workspace_a_few_at_a_time() { | |
| 1181 | + | use g1t_contracts::work::MigratedItem; | |
| 1182 | + | let moved = |workspace: &str, number: u32| MigratedItem { | |
| 1183 | + | repo_id: "rep_1".into(), | |
| 1184 | + | legacy_number: number, | |
| 1185 | + | workspace: workspace.into(), | |
| 1186 | + | number: number + 100, | |
| 1187 | + | item_id: format!("iss_{number}"), | |
| 1188 | + | }; | |
| 1189 | + | let mut items: Vec<MigratedItem> = (1..=25).map(|n| moved("acme", n)).collect(); | |
| 1190 | + | items.insert(3, moved("Beta", 7)); | |
| 1191 | + | let jobs = migrated_jobs(&WorkMigrated { page: 1, items, done: false }); | |
| 1192 | + | assert_eq!(jobs.len(), 3); | |
| 1193 | + | assert_eq!(jobs[0], Job::Items { workspace: "acme".into(), numbers: (101..=120).collect() }); | |
| 1194 | + | assert_eq!(jobs[1], Job::Items { workspace: "acme".into(), numbers: (121..=125).collect() }); | |
| 1195 | + | assert_eq!(jobs[2], Job::Items { workspace: "beta".into(), numbers: vec![107] }); | |
| 1196 | + | let value = serde_json::to_value(&jobs[2]).unwrap(); | |
| 1197 | + | assert_eq!(value, serde_json::json!({ "type": "items", "workspace": "beta", "numbers": [107] })); | |
| 1198 | + | } | |
| 1199 | + | ||
| 1200 | + | #[test] | |
| 1201 | + | fn keys_and_linked_repositories() { | |
| 1202 | + | assert_eq!(pull_key("rep_1", 12), "rep_1/12"); | |
| 1203 | + | assert_eq!(repos_of(&["rep_1", "rep_2"]), "|rep_1|rep_2|"); | |
| 1204 | + | assert_eq!(repos_of(&[]), "|"); | |
| 1205 | + | } | |
| 1206 | + | ||
| 1207 | + | #[test] | |
| 1026 | 1208 | fn pull_statuses_have_names() { | |
| 1027 | 1209 | assert_eq!(PullStatus::Merged.as_str(), "merged"); | |
| 1028 | 1210 | } | |
| 2 | 2 | //! | |
| 3 | 3 | //! - **What it searches.** Repositories (name, description, topics, the | |
| 4 | 4 | //! opening of the README), code on default branches (paths and | |
| 5 | − | //! contents), issues and pull requests (titles and bodies), people and | |
| 6 | − | //! workspaces. Everything public, and private content the viewer is a | |
| 7 | − | //! member of. | |
| 5 | + | //! contents), work items and pull requests (titles and bodies), people | |
| 6 | + | //! and workspaces. Everything public, and private content the viewer is | |
| 7 | + | //! a member of. | |
| 8 | 8 | //! - **The index.** D1 with FTS5, a database that costs nothing while no | |
| 9 | 9 | //! one uses it: words for prose (`unicode61`), every run of three | |
| 10 | 10 | //! characters for code (`trigram`), each table kept in step with its | |
| ⋯ | |||
| 20 | 20 | //! `g1t_contracts::search`. | |
| 21 | 21 | ||
| 22 | 22 | mod index; | |
| 23 | + | #[cfg(test)] | |
| 24 | + | mod migrated; | |
| 23 | 25 | mod query; | |
| 24 | 26 | mod read; | |
| 25 | 27 | mod rules; | |
| 1 | + | //! The search database as the migrations make it, for tests: SQLite (which | |
| 2 | + | //! D1 is), each migration in a transaction of its own as D1 applies it. | |
| 3 | + | //! What each search asks of it is run here too. | |
| 4 | + | ||
| 5 | + | use rusqlite::Connection; | |
| 6 | + | use rusqlite::types::Value; | |
| 7 | + | ||
| 8 | + | use crate::sql::{Param, Sql}; | |
| 9 | + | ||
| 10 | + | /// The migrations, in order: each file's name and SQL. | |
| 11 | + | fn migrations() -> Vec<(String, String)> { | |
| 12 | + | let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("migrations"); | |
| 13 | + | let mut names: Vec<String> = std::fs::read_dir(&dir) | |
| 14 | + | .unwrap() | |
| 15 | + | .map(|entry| entry.unwrap().file_name().into_string().unwrap()) | |
| 16 | + | .filter(|name| name.ends_with(".sql")) | |
| 17 | + | .collect(); | |
| 18 | + | names.sort(); | |
| 19 | + | names.into_iter().map(|name| (name.clone(), std::fs::read_to_string(dir.join(&name)).unwrap())).collect() | |
| 20 | + | } | |
| 21 | + | ||
| 22 | + | fn apply(db: &Connection, name: &str, sql: &str) { | |
| 23 | + | db.execute_batch(&format!("BEGIN;\n{sql}\nCOMMIT;")).unwrap_or_else(|error| panic!("{name}: {error}")); | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | /// A database with the migrations before `name` applied. | |
| 27 | + | fn database_before(name: &str) -> Connection { | |
| 28 | + | let db = Connection::open_in_memory().unwrap(); | |
| 29 | + | for (file, sql) in migrations() { | |
| 30 | + | if file.as_str() >= name { | |
| 31 | + | break; | |
| 32 | + | } | |
| 33 | + | apply(&db, &file, &sql); | |
| 34 | + | } | |
| 35 | + | db | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | fn migrate_from(db: &Connection, name: &str) { | |
| 39 | + | for (file, sql) in migrations() { | |
| 40 | + | if file.as_str() >= name { | |
| 41 | + | apply(db, &file, &sql); | |
| 42 | + | } | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | fn database() -> Connection { | |
| 47 | + | let db = Connection::open_in_memory().unwrap(); | |
| 48 | + | for (file, sql) in migrations() { | |
| 49 | + | apply(&db, &file, &sql); | |
| 50 | + | } | |
| 51 | + | db | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | fn values(params: &[Param]) -> Vec<Value> { | |
| 55 | + | params | |
| 56 | + | .iter() | |
| 57 | + | .map(|param| match param { | |
| 58 | + | Param::Text(text) => Value::Text(text.clone()), | |
| 59 | + | Param::Int(number) => Value::Integer(*number), | |
| 60 | + | Param::Null => Value::Null, | |
| 61 | + | }) | |
| 62 | + | .collect() | |
| 63 | + | } | |
| 64 | + | ||
| 65 | + | fn run(db: &Connection, text: &str, params: Vec<Param>) { | |
| 66 | + | db.execute(text, rusqlite::params_from_iter(values(¶ms))).unwrap_or_else(|error| panic!("{text}: {error}")); | |
| 67 | + | } | |
| 68 | + | ||
| 69 | + | /// The `number` column of every row `sql` returns, in order. | |
| 70 | + | fn numbers(db: &Connection, sql: &Sql) -> Vec<i64> { | |
| 71 | + | let mut statement = db.prepare(&sql.text).unwrap_or_else(|error| panic!("{}: {error}", sql.text)); | |
| 72 | + | let index = statement.column_index("number").unwrap(); | |
| 73 | + | statement | |
| 74 | + | .query_map(rusqlite::params_from_iter(values(&sql.params)), |row| row.get::<_, i64>(index)) | |
| 75 | + | .unwrap() | |
| 76 | + | .map(Result::unwrap) | |
| 77 | + | .collect() | |
| 78 | + | } | |
| 79 | + | ||
| 80 | + | #[cfg(test)] | |
| 81 | + | mod tests { | |
| 82 | + | use g1t_contracts::access::{BasePermission, RepoGrant, RepoRole}; | |
| 83 | + | use g1t_contracts::{Membership, User, Viewer}; | |
| 84 | + | ||
| 85 | + | use super::*; | |
| 86 | + | use crate::index::UPSERT_ITEM; | |
| 87 | + | use crate::query::{ItemKind, Query}; | |
| 88 | + | use crate::sql::{Page, items_count, items_page}; | |
| 89 | + | use crate::visibility::Reader; | |
| 90 | + | ||
| 91 | + | const PAGE: Page = Page { limit: 20, offset: 0 }; | |
| 92 | + | ||
| 93 | + | fn repo(db: &Connection, id: &str, namespace: &str, name: &str, private: bool) { | |
| 94 | + | run( | |
| 95 | + | db, | |
| 96 | + | "INSERT INTO repos (repo_id, namespace, name, private, created_at, updated_at) VALUES (?, ?, ?, ?, '', '')", | |
| 97 | + | vec![Param::from(id), Param::from(namespace), Param::from(name), Param::Int(i64::from(private))], | |
| 98 | + | ); | |
| 99 | + | } | |
| 100 | + | ||
| 101 | + | /// A work item of acme numbered `number`, filed in `origin`, linked to `repos`. | |
| 102 | + | fn item(db: &Connection, number: i64, origin: Option<&str>, repos: &str, title: &str) { | |
| 103 | + | run( | |
| 104 | + | db, | |
| 105 | + | UPSERT_ITEM, | |
| 106 | + | vec![ | |
| 107 | + | Param::from("item"), | |
| 108 | + | Param::Text(format!("wki_{number}")), | |
| 109 | + | Param::opt(origin), | |
| 110 | + | Param::from("acme"), | |
| 111 | + | Param::Int(number), | |
| 112 | + | Param::from(repos), | |
| 113 | + | Param::from(title), | |
| 114 | + | Param::from("the body"), | |
| 115 | + | Param::from("open"), | |
| 116 | + | Param::from("open"), | |
| 117 | + | Param::from("ana"), | |
| 118 | + | Param::Null, | |
| 119 | + | Param::from("|bug|"), | |
| 120 | + | Param::from("2026-10-01T00:00:00Z"), | |
| 121 | + | Param::Text(format!("2026-10-01T00:00:{number:02}Z")), | |
| 122 | + | ], | |
| 123 | + | ); | |
| 124 | + | } | |
| 125 | + | ||
| 126 | + | fn member(base: BasePermission) -> Viewer { | |
| 127 | + | Some(User { | |
| 128 | + | id: "usr_1".into(), | |
| 129 | + | username: "ana".into(), | |
| 130 | + | workspaces: vec![Membership { base_permission: Some(base), ..Membership::member("acme") }], | |
| 131 | + | ..User::default() | |
| 132 | + | }) | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | fn outsider(grants: &[&str]) -> Viewer { | |
| 136 | + | Some(User { | |
| 137 | + | id: "usr_2".into(), | |
| 138 | + | username: "bo".into(), | |
| 139 | + | grants: grants | |
| 140 | + | .iter() | |
| 141 | + | .map(|id| RepoGrant { repo_id: (*id).into(), workspace: "acme".into(), role: RepoRole::Read, team: None }) | |
| 142 | + | .collect(), | |
| 143 | + | ..User::default() | |
| 144 | + | }) | |
| 145 | + | } | |
| 146 | + | ||
| 147 | + | /// acme/web is public, acme/api private; item 1 was filed in web, 2 in | |
| 148 | + | /// api (and targets web), 3 in no repository (and targets web). | |
| 149 | + | fn filled() -> Connection { | |
| 150 | + | let db = database(); | |
| 151 | + | repo(&db, "rep_web", "acme", "web", false); | |
| 152 | + | repo(&db, "rep_api", "acme", "api", true); | |
| 153 | + | item(&db, 1, Some("rep_web"), "|rep_web|", "crash on login"); | |
| 154 | + | item(&db, 2, Some("rep_api"), "|rep_api|rep_web|", "crash in the api"); | |
| 155 | + | item(&db, 3, None, "|rep_web|", "crash everywhere"); | |
| 156 | + | db | |
| 157 | + | } | |
| 158 | + | ||
| 159 | + | fn found(db: &Connection, text: &str, viewer: &Viewer) -> Vec<i64> { | |
| 160 | + | let query = Query::parse(text); | |
| 161 | + | let reader = Reader::of(viewer); | |
| 162 | + | let mut page = numbers(db, &items_page(&query, &reader, ItemKind::Item, PAGE)); | |
| 163 | + | page.sort(); | |
| 164 | + | page | |
| 165 | + | } | |
| 166 | + | ||
| 167 | + | #[test] | |
| 168 | + | fn issues_are_dropped_and_pull_requests_kept() { | |
| 169 | + | let db = database_before("0003"); | |
| 170 | + | repo(&db, "rep_web", "acme", "web", false); | |
| 171 | + | for (kind, number, title) in [("issue", 1, "crash on login"), ("pull", 2, "fix the crash")] { | |
| 172 | + | run( | |
| 173 | + | &db, | |
| 174 | + | "INSERT INTO items (repo_id, kind, number, title, body, state, status, author, labels, created_at, updated_at, requested_by) | |
| 175 | + | VALUES ('rep_web', ?, ?, ?, 'body', 'open', 'open', 'ana', '|bug|', '', '', 'bo')", | |
| 176 | + | vec![Param::from(kind), Param::Int(number), Param::from(title)], | |
| 177 | + | ); | |
| 178 | + | } | |
| 179 | + | migrate_from(&db, "0003"); | |
| 180 | + | let rows: Vec<(String, String, Option<String>, String, String)> = db | |
| 181 | + | .prepare("SELECT kind, key, workspace, repos, requested_by FROM items") | |
| 182 | + | .unwrap() | |
| 183 | + | .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?))) | |
| 184 | + | .unwrap() | |
| 185 | + | .map(Result::unwrap) | |
| 186 | + | .collect(); | |
| 187 | + | assert_eq!(rows, vec![("pull".into(), "rep_web/2".into(), None, "|".into(), "bo".into())]); | |
| 188 | + | // The full-text index holds the pull request, and nothing of the issue. | |
| 189 | + | let pulls = numbers(&db, &items_page(&Query::parse("crash"), &Reader::default(), ItemKind::Pull, PAGE)); | |
| 190 | + | assert_eq!(pulls, vec![2]); | |
| 191 | + | let matched: i64 = db.query_row("SELECT count(*) FROM items_fts WHERE items_fts MATCH 'login'", [], |row| row.get(0)).unwrap(); | |
| 192 | + | assert_eq!(matched, 0); | |
| 193 | + | } | |
| 194 | + | ||
| 195 | + | #[test] | |
| 196 | + | fn a_work_item_indexed_again_is_updated_in_place() { | |
| 197 | + | let db = filled(); | |
| 198 | + | item(&db, 1, Some("rep_web"), "|rep_web|rep_api|", "a new title"); | |
| 199 | + | let rows: i64 = db.query_row("SELECT count(*) FROM items WHERE key = 'wki_1'", [], |row| row.get(0)).unwrap(); | |
| 200 | + | assert_eq!(rows, 1); | |
| 201 | + | assert_eq!(found(&db, "title", &None), vec![1]); | |
| 202 | + | assert!(found(&db, "login", &None).is_empty()); | |
| 203 | + | } | |
| 204 | + | ||
| 205 | + | #[test] | |
| 206 | + | fn members_see_every_work_item() { | |
| 207 | + | let db = filled(); | |
| 208 | + | // Even a member whose base permission reads no repository. | |
| 209 | + | assert_eq!(found(&db, "crash", &member(BasePermission::None)), vec![1, 2, 3]); | |
| 210 | + | assert_eq!(found(&db, "crash", &member(BasePermission::Read)), vec![1, 2, 3]); | |
| 211 | + | } | |
| 212 | + | ||
| 213 | + | #[test] | |
| 214 | + | fn others_see_what_was_filed_where_they_read() { | |
| 215 | + | let db = filled(); | |
| 216 | + | // Signed out, or in no way part of acme: what was filed in its public repository. | |
| 217 | + | assert_eq!(found(&db, "crash", &None), vec![1]); | |
| 218 | + | assert_eq!(found(&db, "crash", &outsider(&[])), vec![1]); | |
| 219 | + | // Given a role on the private one: what was filed there too, never | |
| 220 | + | // what was filed in no repository. | |
| 221 | + | assert_eq!(found(&db, "crash", &outsider(&["rep_api"])), vec![1, 2]); | |
| 222 | + | let count = items_count(&Query::parse("crash"), &Reader::of(&None), ItemKind::Item); | |
| 223 | + | assert_eq!(numbers_count(&db, &count), 1); | |
| 224 | + | } | |
| 225 | + | ||
| 226 | + | #[test] | |
| 227 | + | fn repo_matches_any_linked_repository_and_org_the_workspace() { | |
| 228 | + | let db = filled(); | |
| 229 | + | let viewer = member(BasePermission::Read); | |
| 230 | + | assert_eq!(found(&db, "crash repo:acme/web", &viewer), vec![1, 2, 3]); | |
| 231 | + | assert_eq!(found(&db, "crash repo:acme/api", &viewer), vec![2]); | |
| 232 | + | assert_eq!(found(&db, "crash -repo:acme/api", &viewer), vec![1, 3]); | |
| 233 | + | assert_eq!(found(&db, "crash org:acme", &viewer), vec![1, 2, 3]); | |
| 234 | + | assert_eq!(found(&db, "workspace:other", &viewer), Vec::<i64>::new()); | |
| 235 | + | // Public: filed in a public repository. Private: anything else. | |
| 236 | + | assert_eq!(found(&db, "crash is:public", &viewer), vec![1]); | |
| 237 | + | assert_eq!(found(&db, "crash is:private", &viewer), vec![2, 3]); | |
| 238 | + | // Without words, by qualifiers alone. | |
| 239 | + | assert_eq!(found(&db, "is:open label:bug", &viewer), vec![1, 2, 3]); | |
| 240 | + | } | |
| 241 | + | ||
| 242 | + | fn numbers_count(db: &Connection, sql: &Sql) -> i64 { | |
| 243 | + | db.query_row(&sql.text, rusqlite::params_from_iter(values(&sql.params)), |row| row.get(0)).unwrap() | |
| 244 | + | } | |
| 245 | + | } |
| 38 | 38 | ||
| 39 | 39 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] | |
| 40 | 40 | pub enum ItemKind { | |
| 41 | − | Issue, | |
| 41 | + | Item, | |
| 42 | 42 | Pull, | |
| 43 | 43 | } | |
| 44 | 44 | ||
| ⋯ | |||
| 201 | 201 | return; | |
| 202 | 202 | } | |
| 203 | 203 | match lower { | |
| 204 | − | "issue" | "issues" => self.kind = Some(ItemKind::Issue), | |
| 204 | + | "item" | "items" => self.kind = Some(ItemKind::Item), | |
| 205 | 205 | "pr" | "pull" | "pulls" | "pull-request" => self.kind = Some(ItemKind::Pull), | |
| 206 | 206 | "open" => self.state = Some(ItemState::Open), | |
| 207 | 207 | "closed" => self.state = Some(ItemState::Closed), | |
| ⋯ | |||
| 244 | 244 | } | |
| 245 | 245 | ||
| 246 | 246 | /// The tab a query is about when none is named: `type:` if given, then | |
| 247 | − | /// pull requests for `is:pr`, issues for `is:issue` or anything only | |
| 248 | − | /// issues have, code for `path:`, else repositories. | |
| 247 | + | /// pull requests for `is:pr`, work items for `is:item` or anything only | |
| 248 | + | /// work items and pull requests have, code for `path:`, else | |
| 249 | + | /// repositories. | |
| 249 | 250 | pub fn default_type(&self) -> SearchType { | |
| 250 | 251 | if let Some(kind) = self.type_hint { | |
| 251 | 252 | return kind; | |
| 252 | 253 | } | |
| 253 | 254 | match self.kind { | |
| 254 | 255 | Some(ItemKind::Pull) => return SearchType::Pulls, | |
| 255 | − | Some(ItemKind::Issue) => return SearchType::Issues, | |
| 256 | + | Some(ItemKind::Item) => return SearchType::Items, | |
| 256 | 257 | None => {} | |
| 257 | 258 | } | |
| 258 | 259 | if !self.paths.is_empty() { | |
| 259 | 260 | return SearchType::Code; | |
| 260 | 261 | } | |
| 261 | 262 | if self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty() { | |
| 262 | − | return SearchType::Issues; | |
| 263 | + | return SearchType::Items; | |
| 263 | 264 | } | |
| 264 | 265 | SearchType::Repositories | |
| 265 | 266 | } | |
| 266 | 267 | ||
| 267 | 268 | /// Whether the query can have results of `kind`: qualifiers only one | |
| 268 | 269 | /// kind has rule the others out (`path:` is only code, `label:` only | |
| 269 | − | /// issues and pull requests). | |
| 270 | + | /// work items and pull requests). | |
| 270 | 271 | pub fn applies_to(&self, kind: SearchType) -> bool { | |
| 271 | 272 | let item_only = self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty(); | |
| 272 | 273 | let repo_scoped = !self.repos.is_empty() || !self.owners.is_empty() || self.visibility.is_some(); | |
| 273 | 274 | match kind { | |
| 274 | 275 | SearchType::Repositories => self.paths.is_empty() && self.kind.is_none() && !item_only, | |
| 275 | 276 | SearchType::Code => self.kind.is_none() && !item_only, | |
| 276 | − | SearchType::Issues => { | |
| 277 | + | SearchType::Items => { | |
| 277 | 278 | self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Pull) | |
| 278 | 279 | && !matches!(self.state, Some(ItemState::Merged | ItemState::Draft)) | |
| 279 | 280 | } | |
| 280 | − | SearchType::Pulls => self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Issue), | |
| 281 | + | SearchType::Pulls => self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Item), | |
| 281 | 282 | SearchType::People => { | |
| 282 | 283 | self.paths.is_empty() && self.languages.is_empty() && self.kind.is_none() && !item_only && !repo_scoped | |
| 283 | 284 | } | |
| 284 | 285 | } | |
| 285 | 286 | } | |
| 286 | 287 | ||
| 287 | − | /// The full-text expression for prose (repositories, issues, people): | |
| 288 | + | /// The full-text expression for prose (repositories, work items, people): | |
| 288 | 289 | /// every word, each a prefix as people type it, phrases exact, `-` | |
| 289 | 290 | /// words left out. `None` when there are no words to match. | |
| 290 | 291 | pub fn prose_match(&self) -> Option<String> { | |
| ⋯ | |||
| 358 | 359 | qualifier("author", &self.authors); | |
| 359 | 360 | qualifier("label", &self.labels); | |
| 360 | 361 | match self.kind { | |
| 361 | − | Some(ItemKind::Issue) => out.push("is:issue".into()), | |
| 362 | + | Some(ItemKind::Item) => out.push("is:item".into()), | |
| 362 | 363 | Some(ItemKind::Pull) if !matches!(self.state, Some(ItemState::Merged | ItemState::Draft)) => out.push("is:pr".into()), | |
| 363 | 364 | _ => {} | |
| 364 | 365 | } | |
| ⋯ | |||
| 463 | 464 | } | |
| 464 | 465 | ||
| 465 | 466 | #[test] | |
| 467 | + | fn work_items_are_items() { | |
| 468 | + | assert_eq!(Query::parse("is:item").kind, Some(ItemKind::Item)); | |
| 469 | + | // The old names are not kept. | |
| 470 | + | let old = Query::parse("is:issue type:issues"); | |
| 471 | + | assert_eq!(old.kind, None); | |
| 472 | + | assert_eq!(old.type_hint, None); | |
| 473 | + | assert_eq!(old.notes.len(), 2); | |
| 474 | + | } | |
| 475 | + | ||
| 476 | + | #[test] | |
| 466 | 477 | fn mistakes_are_said() { | |
| 467 | 478 | let query = Query::parse("repo:acme is:spaceship label:"); | |
| 468 | 479 | assert_eq!(query.notes.len(), 3); | |
| ⋯ | |||
| 474 | 485 | assert_eq!(Query::parse("router").default_type(), SearchType::Repositories); | |
| 475 | 486 | assert_eq!(Query::parse("router path:src").default_type(), SearchType::Code); | |
| 476 | 487 | assert_eq!(Query::parse("crash is:pr").default_type(), SearchType::Pulls); | |
| 477 | − | assert_eq!(Query::parse("crash is:open").default_type(), SearchType::Issues); | |
| 478 | − | assert_eq!(Query::parse("crash label:bug").default_type(), SearchType::Issues); | |
| 488 | + | assert_eq!(Query::parse("crash is:open").default_type(), SearchType::Items); | |
| 489 | + | assert_eq!(Query::parse("crash is:item").default_type(), SearchType::Items); | |
| 490 | + | assert_eq!(Query::parse("crash type:items").default_type(), SearchType::Items); | |
| 491 | + | assert_eq!(Query::parse("crash label:bug").default_type(), SearchType::Items); | |
| 479 | 492 | assert_eq!(Query::parse("crash type:code").default_type(), SearchType::Code); | |
| 480 | 493 | } | |
| 481 | 494 | ||
| ⋯ | |||
| 484 | 497 | let code = Query::parse("x path:src"); | |
| 485 | 498 | assert!(code.applies_to(SearchType::Code)); | |
| 486 | 499 | assert!(!code.applies_to(SearchType::Repositories)); | |
| 487 | − | assert!(!code.applies_to(SearchType::Issues)); | |
| 488 | − | let issues = Query::parse("x label:bug"); | |
| 489 | − | assert!(issues.applies_to(SearchType::Issues) && issues.applies_to(SearchType::Pulls)); | |
| 490 | − | assert!(!issues.applies_to(SearchType::Code) && !issues.applies_to(SearchType::People)); | |
| 500 | + | assert!(!code.applies_to(SearchType::Items)); | |
| 501 | + | let items = Query::parse("x label:bug"); | |
| 502 | + | assert!(items.applies_to(SearchType::Items) && items.applies_to(SearchType::Pulls)); | |
| 503 | + | assert!(!items.applies_to(SearchType::Code) && !items.applies_to(SearchType::People)); | |
| 491 | 504 | let merged = Query::parse("x is:merged"); | |
| 492 | − | assert!(merged.applies_to(SearchType::Pulls) && !merged.applies_to(SearchType::Issues)); | |
| 505 | + | assert!(merged.applies_to(SearchType::Pulls) && !merged.applies_to(SearchType::Items)); | |
| 493 | 506 | let scoped = Query::parse("x repo:a/b"); | |
| 494 | 507 | assert!(!scoped.applies_to(SearchType::People)); | |
| 495 | 508 | assert!(Query::parse("ana").applies_to(SearchType::People)); | |
| ⋯ | |||
| 528 | 541 | ||
| 529 | 542 | #[test] | |
| 530 | 543 | fn the_query_is_written_back_tidied() { | |
| 531 | − | let query = Query::parse(" Parse REPO:acme/web \"two words\" is:issue "); | |
| 532 | − | assert_eq!(query.display(), "Parse \"two words\" repo:acme/web is:issue"); | |
| 544 | + | let query = Query::parse(" Parse REPO:acme/web \"two words\" is:item "); | |
| 545 | + | assert_eq!(query.display(), "Parse \"two words\" repo:acme/web is:item"); | |
| 533 | 546 | } | |
| 534 | 547 | ||
| 535 | 548 | #[test] | |
| 94 | 94 | #[derive(Deserialize)] | |
| 95 | 95 | struct ItemRow { | |
| 96 | 96 | kind: String, | |
| 97 | + | #[serde(default)] | |
| 98 | + | workspace: Option<String>, | |
| 97 | 99 | number: u32, | |
| 98 | 100 | title: String, | |
| 99 | 101 | state: String, | |
| ⋯ | |||
| 103 | 105 | requested_by: Option<String>, | |
| 104 | 106 | labels: String, | |
| 105 | 107 | updated_at: String, | |
| 106 | − | repo_id: String, | |
| 107 | − | namespace: String, | |
| 108 | − | name: String, | |
| 109 | − | private: u8, | |
| 108 | + | // A work item's origin, if it has one, as the index holds it. | |
| 109 | + | #[serde(default)] | |
| 110 | + | repo_id: Option<String>, | |
| 111 | + | #[serde(default)] | |
| 112 | + | namespace: Option<String>, | |
| 113 | + | #[serde(default)] | |
| 114 | + | name: Option<String>, | |
| 115 | + | #[serde(default)] | |
| 116 | + | private: Option<u8>, | |
| 110 | 117 | body: Option<String>, | |
| 111 | 118 | } | |
| 112 | 119 | ||
| ⋯ | |||
| 119 | 126 | avatar: Option<String>, | |
| 120 | 127 | } | |
| 121 | 128 | ||
| 122 | − | /// A result, and the repository whose visibility decides whether it is shown. | |
| 129 | + | /// A result, and what decides whether it is shown: the repository whose | |
| 130 | + | /// visibility it has, and for a work item its workspace, whose members see | |
| 131 | + | /// it whatever its repository (with none, only they do). | |
| 123 | 132 | struct Found { | |
| 124 | 133 | hit: Hit, | |
| 125 | 134 | repo: Option<Indexed>, | |
| 135 | + | workspace: Option<String>, | |
| 126 | 136 | } | |
| 127 | 137 | ||
| 138 | + | impl Found { | |
| 139 | + | fn shown(&self, reader: &Reader, keep: &HashSet<String>) -> bool { | |
| 140 | + | if let Some(workspace) = &self.workspace | |
| 141 | + | && reader.member_of(workspace) | |
| 142 | + | { | |
| 143 | + | return true; | |
| 144 | + | } | |
| 145 | + | match &self.repo { | |
| 146 | + | Some(repo) => keep.contains(&repo.repo_id), | |
| 147 | + | None => self.workspace.is_none(), | |
| 148 | + | } | |
| 149 | + | } | |
| 150 | + | } | |
| 151 | + | ||
| 128 | 152 | fn repo_hit(row: RepoRow, terms: &[String]) -> Found { | |
| 129 | 153 | let full = format!("{}/{}", row.namespace, row.name); | |
| 130 | 154 | let mut hit = Hit::new(HitKind::Repository, full.clone(), format!("/{}/{}", row.namespace, row.name)); | |
| ⋯ | |||
| 146 | 170 | namespace: row.namespace, | |
| 147 | 171 | private: row.private != 0, | |
| 148 | 172 | }), | |
| 173 | + | workspace: None, | |
| 149 | 174 | } | |
| 150 | 175 | } | |
| 151 | 176 | ||
| 152 | − | fn item_hit(row: ItemRow, terms: &[String]) -> Found { | |
| 177 | + | fn item_hit(row: ItemRow, terms: &[String]) -> Option<Found> { | |
| 153 | 178 | let pull = row.kind == "pull"; | |
| 154 | − | let full = format!("{}/{}", row.namespace, row.name); | |
| 155 | − | let url = format!("/{full}/{}/{}", if pull { "pull" } else { "issues" }, row.number); | |
| 156 | − | let mut hit = Hit::new(if pull { HitKind::Pull } else { HitKind::Issue }, row.title, url); | |
| 157 | − | hit.repo = Some(full); | |
| 158 | − | hit.private = row.private != 0; | |
| 179 | + | let repo = match (row.repo_id, row.namespace, row.name) { | |
| 180 | + | (Some(repo_id), Some(namespace), Some(name)) => { | |
| 181 | + | Some((Indexed { repo_id, namespace, private: row.private.unwrap_or(1) != 0 }, name)) | |
| 182 | + | } | |
| 183 | + | _ => None, | |
| 184 | + | }; | |
| 185 | + | let full = repo.as_ref().map(|(repo, name)| format!("{}/{name}", repo.namespace)); | |
| 186 | + | let (kind, url, workspace) = if pull { | |
| 187 | + | (HitKind::Pull, format!("/{}/pull/{}", full.as_deref()?, row.number), None) | |
| 188 | + | } else { | |
| 189 | + | let workspace = row.workspace?; | |
| 190 | + | (HitKind::Item, format!("/{workspace}/-/work/{}", row.number), Some(workspace)) | |
| 191 | + | }; | |
| 192 | + | let mut hit = Hit::new(kind, row.title, url); | |
| 193 | + | hit.repo = full; | |
| 194 | + | hit.workspace = workspace.clone(); | |
| 195 | + | // A work item filed in no repository is for its workspace's members. | |
| 196 | + | hit.private = repo.as_ref().is_none_or(|(repo, _)| repo.private); | |
| 159 | 197 | hit.snippet = snippet::prose(row.body.as_deref().unwrap_or_default(), terms); | |
| 160 | 198 | hit.number = Some(row.number); | |
| 161 | 199 | hit.state = Some(if pull { row.status } else { row.state }); | |
| ⋯ | |||
| 163 | 201 | hit.requested_by = row.requested_by; | |
| 164 | 202 | hit.labels = split_labels(&row.labels); | |
| 165 | 203 | hit.updated_at = Some(row.updated_at); | |
| 166 | − | Found { | |
| 167 | − | hit, | |
| 168 | − | repo: Some(Indexed { | |
| 169 | − | repo_id: row.repo_id, | |
| 170 | − | namespace: row.namespace, | |
| 171 | − | private: row.private != 0, | |
| 172 | − | }), | |
| 173 | − | } | |
| 204 | + | Some(Found { hit, repo: repo.map(|(repo, _)| repo), workspace }) | |
| 174 | 205 | } | |
| 175 | 206 | ||
| 176 | 207 | fn person_hit(row: PersonRow, terms: &[String]) -> Found { | |
| ⋯ | |||
| 182 | 213 | hit.snippet = row.bio.as_deref().map(|bio| snippet::prose(bio, terms)).unwrap_or_default(); | |
| 183 | 214 | hit.description = row.bio; | |
| 184 | 215 | hit.avatar = row.avatar; | |
| 185 | − | Found { hit, repo: None } | |
| 216 | + | Found { hit, repo: None, workspace: None } | |
| 186 | 217 | } | |
| 187 | 218 | ||
| 188 | 219 | impl Search { | |
| ⋯ | |||
| 240 | 271 | async fn admit(&self, viewer: &Viewer, found: Vec<Found>) -> Vec<Hit> { | |
| 241 | 272 | let indexed: Vec<Indexed> = found.iter().filter_map(|f| f.repo.clone()).collect(); | |
| 242 | 273 | let (keep, _) = self.readable(viewer, &indexed).await; | |
| 274 | + | let reader = Reader::of(viewer); | |
| 243 | 275 | found | |
| 244 | 276 | .into_iter() | |
| 245 | − | .filter(|f| f.repo.as_ref().is_none_or(|repo| keep.contains(&repo.repo_id))) | |
| 277 | + | .filter(|f| f.shown(&reader, &keep)) | |
| 246 | 278 | .map(|f| f.hit) | |
| 247 | 279 | .collect() | |
| 248 | 280 | } | |
| ⋯ | |||
| 254 | 286 | let sql = match kind { | |
| 255 | 287 | SearchType::Repositories => Some(sql::repos_count(query, reader)), | |
| 256 | 288 | SearchType::Code => sql::code_plan(query).ok().map(|plan| sql::code_count(&plan, query, reader)), | |
| 257 | − | SearchType::Issues => Some(sql::items_count(query, reader, ItemKind::Issue)), | |
| 289 | + | SearchType::Items => Some(sql::items_count(query, reader, ItemKind::Item)), | |
| 258 | 290 | SearchType::Pulls => Some(sql::items_count(query, reader, ItemKind::Pull)), | |
| 259 | 291 | SearchType::People => sql::people_select(query).map(|select| sql::count(&select)), | |
| 260 | 292 | }; | |
| ⋯ | |||
| 316 | 348 | namespace: file.namespace.clone(), | |
| 317 | 349 | private: file.private != 0, | |
| 318 | 350 | }), | |
| 351 | + | workspace: None, | |
| 319 | 352 | }); | |
| 320 | 353 | } | |
| 321 | 354 | Ok(found) | |
| ⋯ | |||
| 335 | 368 | Ok(plan) => self.code_hits(&plan, query, reader, page, terms).await?, | |
| 336 | 369 | Err(_) => Vec::new(), | |
| 337 | 370 | }, | |
| 338 | − | SearchType::Issues | SearchType::Pulls => { | |
| 339 | − | let item = if kind == SearchType::Issues { ItemKind::Issue } else { ItemKind::Pull }; | |
| 371 | + | SearchType::Items | SearchType::Pulls => { | |
| 372 | + | let item = if kind == SearchType::Items { ItemKind::Item } else { ItemKind::Pull }; | |
| 340 | 373 | store::all::<ItemRow>(&self.db, &sql::items_page(query, reader, item, page)) | |
| 341 | 374 | .await? | |
| 342 | 375 | .into_iter() | |
| 343 | − | .map(|row| item_hit(row, terms)) | |
| 376 | + | .filter_map(|row| item_hit(row, terms)) | |
| 344 | 377 | .collect() | |
| 345 | 378 | } | |
| 346 | 379 | SearchType::People => match sql::people_page(query, page) { | |
| ⋯ | |||
| 412 | 445 | Ok(Outcome::Ok(results)) | |
| 413 | 446 | } | |
| 414 | 447 | ||
| 415 | − | /// A few repositories, issues, pull requests and people, for the | |
| 448 | + | /// A few repositories, work items, pull requests and people, for the | |
| 416 | 449 | /// command palette. Never code: that is a full search away. | |
| 417 | 450 | pub async fn suggest(&self, a: SuggestArgs) -> Result<Vec<Hit>> { | |
| 418 | 451 | let query = Query::parse(&a.query); | |
| ⋯ | |||
| 423 | 456 | let page = Page { limit: SUGGEST_EACH, offset: 0 }; | |
| 424 | 457 | let terms = query.highlights(); | |
| 425 | 458 | let mut found = Vec::new(); | |
| 426 | − | for kind in [SearchType::Repositories, SearchType::Issues, SearchType::Pulls, SearchType::People] { | |
| 459 | + | for kind in [SearchType::Repositories, SearchType::Items, SearchType::Pulls, SearchType::People] { | |
| 427 | 460 | match self.page_of(kind, &query, &reader, page, &terms).await { | |
| 428 | 461 | Ok(more) => found.extend(more), | |
| 429 | 462 | Err(error) => worker::console_error!("search: suggesting {} failed: {error}", kind.as_str()), | |
| ⋯ | |||
| 485 | 518 | assert_eq!(encode_path("src/ünï.rs"), "src/%C3%BCn%C3%AF.rs"); | |
| 486 | 519 | } | |
| 487 | 520 | ||
| 521 | + | fn row(kind: &str, workspace: Option<&str>, origin: Option<(&str, &str, bool)>) -> ItemRow { | |
| 522 | + | ItemRow { | |
| 523 | + | kind: kind.into(), | |
| 524 | + | workspace: workspace.map(str::to_owned), | |
| 525 | + | number: 12, | |
| 526 | + | title: "Fix it".into(), | |
| 527 | + | state: "open".into(), | |
| 528 | + | status: "open".into(), | |
| 529 | + | author: "ana".into(), | |
| 530 | + | requested_by: None, | |
| 531 | + | labels: "|bug|".into(), | |
| 532 | + | updated_at: String::new(), | |
| 533 | + | repo_id: origin.map(|_| "rep_1".to_owned()), | |
| 534 | + | namespace: origin.map(|(namespace, _, _)| namespace.to_owned()), | |
| 535 | + | name: origin.map(|(_, name, _)| name.to_owned()), | |
| 536 | + | private: origin.map(|(_, _, private)| u8::from(private)), | |
| 537 | + | body: None, | |
| 538 | + | } | |
| 539 | + | } | |
| 540 | + | ||
| 541 | + | #[test] | |
| 542 | + | fn work_items_are_at_their_workspace() { | |
| 543 | + | let found = item_hit(row("item", Some("acme"), Some(("acme", "web", false))), &[]).unwrap(); | |
| 544 | + | assert_eq!(found.hit.kind, HitKind::Item); | |
| 545 | + | assert_eq!(found.hit.url, "/acme/-/work/12"); | |
| 546 | + | assert_eq!(found.hit.repo.as_deref(), Some("acme/web")); | |
| 547 | + | assert_eq!(found.hit.workspace.as_deref(), Some("acme")); | |
| 548 | + | assert!(!found.hit.private); | |
| 549 | + | // Filed in no repository: for members only. | |
| 550 | + | let found = item_hit(row("item", Some("acme"), None), &[]).unwrap(); | |
| 551 | + | assert_eq!(found.hit.repo, None); | |
| 552 | + | assert!(found.hit.private); | |
| 553 | + | let pull = item_hit(row("pull", None, Some(("acme", "web", false))), &[]).unwrap(); | |
| 554 | + | assert_eq!(pull.hit.url, "/acme/web/pull/12"); | |
| 555 | + | } | |
| 556 | + | ||
| 557 | + | #[test] | |
| 558 | + | fn the_second_check_lets_members_see_their_work_items() { | |
| 559 | + | use g1t_contracts::{Membership, User}; | |
| 560 | + | let member = Reader::of(&Some(User { workspaces: vec![Membership::member("acme")], ..User::default() })); | |
| 561 | + | let none = HashSet::new(); | |
| 562 | + | let readable = HashSet::from(["rep_1".to_owned()]); | |
| 563 | + | let filed = item_hit(row("item", Some("acme"), Some(("acme", "web", true))), &[]).unwrap(); | |
| 564 | + | let loose = item_hit(row("item", Some("acme"), None), &[]).unwrap(); | |
| 565 | + | assert!(filed.shown(&member, &none) && loose.shown(&member, &none)); | |
| 566 | + | // Others, only where the repos service says they read its origin. | |
| 567 | + | let outsider = Reader::default(); | |
| 568 | + | assert!(!filed.shown(&outsider, &none)); | |
| 569 | + | assert!(filed.shown(&outsider, &readable)); | |
| 570 | + | assert!(!loose.shown(&outsider, &readable)); | |
| 571 | + | } | |
| 572 | + | ||
| 488 | 573 | #[test] | |
| 489 | 574 | fn labels_and_topics_are_split() { | |
| 490 | 575 | assert_eq!(split_labels("|bug|good first issue|"), vec!["bug", "good first issue"]); | |
| 7 | 7 | use g1t_contracts::search::COUNT_CAP; | |
| 8 | 8 | ||
| 9 | 9 | use crate::query::{Filter, ItemKind, ItemState, Query, Visibility, path_pattern}; | |
| 10 | − | use crate::visibility::{Reader, clause}; | |
| 10 | + | use crate::visibility::{Reader, clause, item_clause}; | |
| 11 | 11 | ||
| 12 | 12 | /// A value bound to a `?`. | |
| 13 | 13 | #[derive(Clone, Debug, PartialEq, Serialize)] | |
| ⋯ | |||
| 321 | 321 | } | |
| 322 | 322 | } | |
| 323 | 323 | ||
| 324 | − | // ---- Issues and pull requests ---------------------------------------------- | |
| 324 | + | // ---- Work items and pull requests ------------------------------------------ | |
| 325 | 325 | ||
| 326 | − | const ITEM_COLUMNS: &str = "i.kind, i.number, i.title, i.state, i.status, i.author, i.requested_by, i.labels, i.updated_at, \ | |
| 327 | − | r.repo_id, r.namespace, r.name, r.private"; | |
| 326 | + | const ITEM_COLUMNS: &str = "i.kind, i.workspace, i.number, i.title, i.state, i.status, i.author, i.requested_by, i.labels, i.updated_at, r.repo_id, r.namespace, r.name, r.private"; | |
| 328 | 327 | ||
| 328 | + | /// What every query of work items starts with: the visibility check (a | |
| 329 | + | /// member of the workspace, or a reader of the origin), then `repo:` | |
| 330 | + | /// (any repository it is linked to), `org:` (its workspace) and | |
| 331 | + | /// `is:public`/`is:private` (whether its origin is public). | |
| 332 | + | fn item_scope(query: &Query, reader: &Reader) -> Where { | |
| 333 | + | let mut out = Where::default(); | |
| 334 | + | out.add( | |
| 335 | + | item_clause("i", "r"), | |
| 336 | + | vec![Param::Text(reader.members_json()), Param::Text(reader.namespaces_json()), Param::Text(reader.granted_json())], | |
| 337 | + | ); | |
| 338 | + | let linked = "SELECT 1 FROM repos lr WHERE (lr.namespace || '/' || lr.name) IN (SELECT value FROM json_each(?)) AND instr(i.repos, '|' || lr.repo_id || '|') > 0"; | |
| 339 | + | if !query.repos.include.is_empty() { | |
| 340 | + | out.add(format!("EXISTS ({linked})"), vec![json(&query.repos.include)]); | |
| 341 | + | } | |
| 342 | + | if !query.repos.exclude.is_empty() { | |
| 343 | + | out.add(format!("NOT EXISTS ({linked})"), vec![json(&query.repos.exclude)]); | |
| 344 | + | } | |
| 345 | + | in_list(&query.owners, "i.workspace", &mut out); | |
| 346 | + | match query.visibility { | |
| 347 | + | Some(Visibility::Public) => out.add("r.private = 0", vec![]), | |
| 348 | + | Some(Visibility::Private) => out.add("COALESCE(r.private, 1) = 1", vec![]), | |
| 349 | + | None => {} | |
| 350 | + | } | |
| 351 | + | out | |
| 352 | + | } | |
| 353 | + | ||
| 329 | 354 | fn items_select(query: &Query, reader: &Reader, kind: ItemKind) -> (Sql, bool) { | |
| 330 | − | let mut filters = scope(query, reader, "r"); | |
| 331 | − | filters.add("i.kind = ?", vec![Param::Text(if kind == ItemKind::Issue { "issue" } else { "pull" }.into())]); | |
| 355 | + | // A pull request belongs to its repository; a work item to its | |
| 356 | + | // workspace, with its origin, if any, joined for who may read it. | |
| 357 | + | let (mut filters, join) = match kind { | |
| 358 | + | ItemKind::Item => (item_scope(query, reader), "LEFT JOIN repos r ON r.repo_id = i.repo_id"), | |
| 359 | + | ItemKind::Pull => (scope(query, reader, "r"), "JOIN repos r ON r.repo_id = i.repo_id"), | |
| 360 | + | }; | |
| 361 | + | filters.add("i.kind = ?", vec![Param::Text(if kind == ItemKind::Item { "item" } else { "pull" }.into())]); | |
| 332 | 362 | match query.state { | |
| 333 | 363 | Some(ItemState::Open) => filters.add("i.state = 'open'", vec![]), | |
| 334 | 364 | Some(ItemState::Closed) => filters.add("i.state = 'closed'", vec![]), | |
| ⋯ | |||
| 350 | 380 | ( | |
| 351 | 381 | Sql { | |
| 352 | 382 | text: format!( | |
| 353 | − | "SELECT {ITEM_COLUMNS}, snippet(items_fts, 1, '', '', '…', 40) AS body \ | |
| 354 | − | FROM items_fts JOIN items i ON i.iid = items_fts.rowid JOIN repos r ON r.repo_id = i.repo_id \ | |
| 355 | − | WHERE items_fts MATCH ? AND {}", | |
| 383 | + | "SELECT {ITEM_COLUMNS}, snippet(items_fts, 1, '', '', '…', 40) AS body FROM items_fts JOIN items i ON i.iid = items_fts.rowid {join} WHERE items_fts MATCH ? AND {}", | |
| 356 | 384 | filters.text() | |
| 357 | 385 | ), | |
| 358 | 386 | params, | |
| ⋯ | |||
| 363 | 391 | None => ( | |
| 364 | 392 | Sql { | |
| 365 | 393 | text: format!( | |
| 366 | − | "SELECT {ITEM_COLUMNS}, substr(i.body, 1, 400) AS body \ | |
| 367 | − | FROM items i JOIN repos r ON r.repo_id = i.repo_id WHERE {}", | |
| 394 | + | "SELECT {ITEM_COLUMNS}, substr(i.body, 1, 400) AS body FROM items i {join} WHERE {}", | |
| 368 | 395 | filters.text() | |
| 369 | 396 | ), | |
| 370 | 397 | params: filters.params, | |
| ⋯ | |||
| 465 | 492 | repos_count(&query, &reader), | |
| 466 | 493 | code_page(&plan, &query, &reader, PAGE), | |
| 467 | 494 | code_count(&plan, &query, &reader), | |
| 468 | − | items_page(&query, &reader, ItemKind::Issue, PAGE), | |
| 495 | + | items_page(&query, &reader, ItemKind::Item, PAGE), | |
| 496 | + | items_count(&query, &reader, ItemKind::Item), | |
| 497 | + | items_page(&query, &reader, ItemKind::Pull, PAGE), | |
| 469 | 498 | items_count(&query, &reader, ItemKind::Pull), | |
| 470 | 499 | ] { | |
| 471 | 500 | assert!(sql.text.contains("r.private = 0 OR r.namespace IN"), "{}", sql.text); | |
| ⋯ | |||
| 496 | 525 | } | |
| 497 | 526 | ||
| 498 | 527 | #[test] | |
| 499 | − | fn issue_qualifiers_become_conditions() { | |
| 528 | + | fn work_items_are_for_members_and_readers_of_their_origin() { | |
| 529 | + | let query = Query::parse("crash repo:acme/web -repo:acme/old org:acme is:private"); | |
| 530 | + | let sql = items_page(&query, &reader(&["acme"]), ItemKind::Item, PAGE); | |
| 531 | + | assert!(sql.text.contains("LEFT JOIN repos r")); | |
| 532 | + | assert!(sql.text.contains("i.workspace IN (SELECT value FROM json_each(?)) OR ((r.repo_id IS NOT NULL)")); | |
| 533 | + | // repo: is any linked repository, org: the workspace. | |
| 534 | + | assert!(sql.text.contains("EXISTS (SELECT 1 FROM repos lr")); | |
| 535 | + | assert!(sql.text.contains("NOT EXISTS (SELECT 1 FROM repos lr")); | |
| 536 | + | assert!(sql.text.contains("i.workspace IN (SELECT value FROM json_each(?))")); | |
| 537 | + | assert!(sql.text.contains("COALESCE(r.private, 1) = 1")); | |
| 538 | + | assert!(sql.params.contains(&Param::Text(r#"["acme/web"]"#.into()))); | |
| 539 | + | assert!(sql.params.contains(&Param::Text("item".into()))); | |
| 540 | + | assert_eq!(placeholders(&sql), sql.params.len(), "{}", sql.text); | |
| 541 | + | } | |
| 542 | + | ||
| 543 | + | #[test] | |
| 544 | + | fn item_qualifiers_become_conditions() { | |
| 500 | 545 | let query = Query::parse("crash is:closed author:ana label:bug -label:wontfix"); | |
| 501 | − | let sql = items_page(&query, &reader(&[]), ItemKind::Issue, PAGE); | |
| 546 | + | let sql = items_page(&query, &reader(&[]), ItemKind::Item, PAGE); | |
| 502 | 547 | assert!(sql.text.contains("i.state = 'closed'")); | |
| 503 | 548 | assert!(sql.text.contains("i.author IN")); | |
| 504 | 549 | assert!(sql.params.contains(&Param::Text("|bug|".into()))); | |
| ⋯ | |||
| 554 | 599 | "fn repo:acme/web", | |
| 555 | 600 | "repo:acme/web path:*.rs", | |
| 556 | 601 | "crash is:open label:bug author:ana", | |
| 602 | + | "crash repo:acme/web -repo:acme/old org:acme is:private", | |
| 557 | 603 | "language:rust", | |
| 558 | 604 | "ana", | |
| 559 | 605 | ] { | |
| ⋯ | |||
| 565 | 611 | out.push(code_count(&plan, &query, &reader)); | |
| 566 | 612 | out.push(code_pieces(&plan, &[1, 2], 10)); | |
| 567 | 613 | } | |
| 568 | − | out.push(items_page(&query, &reader, ItemKind::Issue, PAGE)); | |
| 614 | + | out.push(items_page(&query, &reader, ItemKind::Item, PAGE)); | |
| 615 | + | out.push(items_count(&query, &reader, ItemKind::Item)); | |
| 616 | + | out.push(items_page(&query, &reader, ItemKind::Pull, PAGE)); | |
| 569 | 617 | out.push(items_count(&query, &reader, ItemKind::Pull)); | |
| 570 | 618 | if let Some(sql) = people_page(&query, PAGE) { | |
| 571 | 619 | out.push(sql); | |
| 4 | 4 | //! row is read if its repository is public, as the index last heard, | |
| 5 | 5 | //! in a workspace whose private repositories the viewer reads (an | |
| 6 | 6 | //! owner, or a member whose base permission gives a role), or one they | |
| 7 | − | //! were given a role on directly. | |
| 7 | + | //! were given a role on directly. A work item is read by every member | |
| 8 | + | //! of its workspace, and by anyone who reads the repository it was filed | |
| 9 | + | //! in (`access::item_role`); one filed in no repository, by members only. | |
| 8 | 10 | //! 2. On the page about to be returned, against the repos service, which | |
| 9 | 11 | //! owns visibility: anything it does not say the viewer may read now is | |
| 10 | 12 | //! dropped, and the index is corrected. A repository made private a | |
| ⋯ | |||
| 30 | 32 | /// Repositories the reader was given a role on directly, by id: an | |
| 31 | 33 | /// outside collaborator's, or a member's beyond the base permission. | |
| 32 | 34 | pub granted: Vec<String>, | |
| 35 | + | /// Workspaces, by slug, the reader is a member of, so sees every work | |
| 36 | + | /// item of (`access::workspace_role`). | |
| 37 | + | pub members_of: Vec<String>, | |
| 33 | 38 | } | |
| 34 | 39 | ||
| 35 | 40 | impl Reader { | |
| ⋯ | |||
| 47 | 52 | .filter(|slug| access::granted(user, RepoRef { id: "", namespace: slug, private: true }).is_some()) | |
| 48 | 53 | .collect(), | |
| 49 | 54 | granted: user.grants.iter().map(|grant| grant.repo_id.clone()).collect(), | |
| 55 | + | members_of: user | |
| 56 | + | .workspaces | |
| 57 | + | .iter() | |
| 58 | + | .map(|membership| membership.slug.to_lowercase()) | |
| 59 | + | .filter(|slug| access::workspace_role(user, slug).is_some()) | |
| 60 | + | .collect(), | |
| 50 | 61 | } | |
| 51 | 62 | } | |
| 52 | 63 | ||
| ⋯ | |||
| 76 | 87 | pub fn granted_json(&self) -> String { | |
| 77 | 88 | serde_json::to_string(&self.granted).unwrap_or_else(|_| "[]".into()) | |
| 78 | 89 | } | |
| 90 | + | ||
| 91 | + | /// Whether the reader sees every work item of `workspace`. | |
| 92 | + | pub fn member_of(&self, workspace: &str) -> bool { | |
| 93 | + | self.members_of.iter().any(|slug| slug.eq_ignore_ascii_case(workspace)) | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | /// [`Reader::members_of`] as JSON, likewise. | |
| 97 | + | pub fn members_json(&self) -> String { | |
| 98 | + | serde_json::to_string(&self.members_of).unwrap_or_else(|_| "[]".into()) | |
| 99 | + | } | |
| 79 | 100 | } | |
| 80 | 101 | ||
| 81 | 102 | /// The SQL the first check adds to every query, given the repository's | |
| ⋯ | |||
| 87 | 108 | ) | |
| 88 | 109 | } | |
| 89 | 110 | ||
| 111 | + | /// The first check for work items, given the item's and its origin's table | |
| 112 | + | /// aliases: a member of its workspace, or a reader of its origin, which is | |
| 113 | + | /// joined with a `LEFT JOIN` (none: no origin, members only). Its three | |
| 114 | + | /// parameters are [`Reader::members_json`], [`Reader::namespaces_json`] and | |
| 115 | + | /// [`Reader::granted_json`], in that order. | |
| 116 | + | pub fn item_clause(item: &str, origin: &str) -> String { | |
| 117 | + | format!( | |
| 118 | + | "({item}.workspace IN (SELECT value FROM json_each(?)) OR (({origin}.repo_id IS NOT NULL) AND {}))", | |
| 119 | + | clause(origin) | |
| 120 | + | ) | |
| 121 | + | } | |
| 122 | + | ||
| 90 | 123 | /// A repository as the index holds it. | |
| 91 | 124 | #[derive(Clone, Debug, PartialEq, Eq)] | |
| 92 | 125 | pub struct Indexed { | |
| ⋯ | |||
| 347 | 380 | } | |
| 348 | 381 | ||
| 349 | 382 | #[test] | |
| 383 | + | fn members_see_every_work_item_of_their_workspace() { | |
| 384 | + | // A member whose base permission gives no role on its repositories | |
| 385 | + | // still sees its work items. | |
| 386 | + | let reader = Reader::of(&member_with(BasePermission::None, &[])); | |
| 387 | + | assert!(reader.member_of("ACME")); | |
| 388 | + | assert_eq!(reader.members_json(), r#"["acme"]"#); | |
| 389 | + | assert_eq!(reader.namespaces_json(), "[]"); | |
| 390 | + | assert!(!Reader::of(&None).member_of("acme")); | |
| 391 | + | assert!(!Reader::of(&viewer(&["other"])).member_of("acme")); | |
| 392 | + | assert_eq!( | |
| 393 | + | item_clause("i", "r"), | |
| 394 | + | "(i.workspace IN (SELECT value FROM json_each(?)) OR ((r.repo_id IS NOT NULL) AND (r.private = 0 OR r.namespace IN (SELECT value FROM json_each(?)) OR r.repo_id IN (SELECT value FROM json_each(?)))))" | |
| 395 | + | ); | |
| 396 | + | } | |
| 397 | + | ||
| 398 | + | #[test] | |
| 350 | 399 | fn a_moved_repository_is_corrected() { | |
| 351 | 400 | let reader = Reader::of(&None); | |
| 352 | 401 | let mut moved = repo("rep_4", "newname", false); | |
Comments
Sign in to comment.