Skip to content

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

Commit

Index work items in search

Search indexes work items per workspace (kind item) and finds them with is:item and type:items; repo: matches any repository an item is linked to and workspace: its workspace. Members of the workspace see all of its items, anyone who reads an item's origin sees those filed there, and items with no origin are for members only, in the query and again on the page. Results link to /{workspace}/-/work/{number}. Events are work_item.* in place of issue.*; work.migrated reads the moved items again by workspace and number, and migration 0003 rekeys the items table (work items by id, pull requests by repository and number), keeps pull requests and drops the old issue rows. A new deployment's backfill lists each workspace's work items.

G1T-Session:
ses_01m566c35sewwsq09vknpxzh49
syntaqxcommitted Parente1c5273Browse files
10 files+885−1810/10 viewed
+1−0
16541654 dependencies = [
16551655 "g1t-contracts",
16561656 "g1t-kit",
1657+ "rusqlite",
16571658 "serde",
16581659 "serde_json",
16591660 "worker",
+4−1
33 version = "0.1.0"
44 edition.workspace = true
55 license.workspace = true
6−description = "Search across all of g1t: repositories, code, issues, pull requests, people and workspaces; and Explore."
6+description = "Search across all of g1t: repositories, code, work items, pull requests, people and workspaces; and Explore."
77
88 [lib]
99 crate-type = ["cdylib"]
1515 serde_json.workspace = true
1616 worker.workspace = true
1717
18+[dev-dependencies]
19+rusqlite = { version = "0.32", features = ["bundled"] }
20+
1821 # wasm-opt at -O1: about the same gzipped size as -O in a tenth of the
1922 # time.
2023 [package.metadata.wasm-pack.profile.release]
+76−0
1+-- Issues became work items of a workspace (the work service's
2+-- 0049_work_items). Items are kept by key: a work item by its id, a pull
3+-- request by its repository and number. A work item keeps its workspace,
4+-- its number there, its origin (where it was filed, if anywhere) and every
5+-- repository it is linked to.
6+--
7+-- Pull requests are copied as they are. Issues are dropped: they are
8+-- indexed again as work items, by id, when the work service's move
9+-- announces them (`work.migrated`), with their workspace and new number.
10+
11+DROP TRIGGER items_ai;
12+DROP TRIGGER items_ad;
13+DROP TRIGGER items_au;
14+DROP TABLE items_fts;
15+
16+CREATE TABLE items_next (
17+ iid INTEGER PRIMARY KEY,
18+ -- item or pull.
19+ kind TEXT NOT NULL,
20+ -- A work item's id; a pull request's repository id and number, `rep_…/12`.
21+ key TEXT NOT NULL UNIQUE,
22+ -- A pull request's repository; a work item's origin, if it has one.
23+ repo_id TEXT,
24+ -- A work item's workspace, lowercase.
25+ workspace TEXT,
26+ number INTEGER NOT NULL,
27+ title TEXT NOT NULL,
28+ body TEXT NOT NULL DEFAULT '',
29+ -- open or closed.
30+ state TEXT NOT NULL,
31+ -- A work item: open, completed or not_planned. A pull request: draft,
32+ -- open, merged or closed.
33+ status TEXT NOT NULL,
34+ -- Lowercase username.
35+ author TEXT NOT NULL,
36+ requested_by TEXT,
37+ -- Lowercase, each between bars: |bug|good first issue|.
38+ labels TEXT NOT NULL DEFAULT '|',
39+ -- A work item's linked repositories by id, each between bars:
40+ -- |rep_1|rep_2|.
41+ repos TEXT NOT NULL DEFAULT '|',
42+ created_at TEXT NOT NULL,
43+ updated_at TEXT NOT NULL
44+);
45+
46+INSERT INTO items_next
47+ (iid, kind, key, repo_id, workspace, number, title, body, state, status, author, requested_by, labels, repos,
48+ created_at, updated_at)
49+SELECT iid, kind, repo_id || '/' || number, repo_id, NULL, number, title, body, state, status, author, requested_by,
50+ labels, '|', created_at, updated_at
51+FROM items WHERE kind = 'pull';
52+
53+DROP TABLE items;
54+ALTER TABLE items_next RENAME TO items;
55+
56+CREATE INDEX items_recent ON items (kind, updated_at);
57+CREATE INDEX items_by_author ON items (author);
58+CREATE INDEX items_by_repo ON items (repo_id);
59+CREATE INDEX items_by_workspace ON items (workspace, number);
60+
61+CREATE VIRTUAL TABLE items_fts USING fts5(
62+ title, body,
63+ content = 'items', content_rowid = 'iid',
64+ tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
65+);
66+INSERT INTO items_fts (items_fts) VALUES ('rebuild');
67+CREATE TRIGGER items_ai AFTER INSERT ON items BEGIN
68+ INSERT INTO items_fts (rowid, title, body) VALUES (new.iid, new.title, new.body);
69+END;
70+CREATE TRIGGER items_ad AFTER DELETE ON items BEGIN
71+ INSERT INTO items_fts (items_fts, rowid, title, body) VALUES ('delete', old.iid, old.title, old.body);
72+END;
73+CREATE TRIGGER items_au AFTER UPDATE OF title, body ON items BEGIN
74+ INSERT INTO items_fts (items_fts, rowid, title, body) VALUES ('delete', old.iid, old.title, old.body);
75+ INSERT INTO items_fts (rowid, title, body) VALUES (new.iid, new.title, new.body);
76+END;
+296−114
1111
1212 use std::collections::{HashMap, HashSet};
1313
14−use g1t_contracts::events::{Event, ProjectUpdated};
14+use g1t_contracts::events::{Event, ProjectUpdated, WorkMigrated};
1515 use g1t_contracts::identity::{DirectoryArgs, DirectoryEntry, DirectoryPage, Profile, SlugArgs, UsernameArgs, Workspace};
1616 use g1t_contracts::repos::{
1717 AllIdsArgs, BlobText, ChangedFilesArgs, FileList, IdPage, ListFilesArgs, PathByIdArgs, ReadBlobsArgs, RepoPath, TreeArgs,
1919 };
2020 use g1t_contracts::time::rfc3339;
2121 use g1t_contracts::work::{
22− Issue, IssueDetail, ListIssuesArgs, ListPullsArgs, Pull, PullDetail, State, ViewArgs,
22+ ListPullsArgs, ListWorkItemsArgs, Pull, PullDetail, State, ViewArgs, WorkItem, WorkItemDetail, WorkItemViewArgs,
2323 };
2424 use g1t_contracts::{Membership, Outcome, PrincipalKind, User};
2525 use g1t_kit::now_ms;
4444 const READ_GROUP: usize = 20;
4545 /// Pieces of a file written in one statement.
4646 const PIECES_PER_STATEMENT: usize = 8;
47−/// The most of an issue's or pull request's body indexed.
47+/// The most of a work item's or pull request's body indexed.
4848 const BODY_CHARS: usize = 32 * 1024;
4949 /// The most of a README indexed.
5050 const README_CHARS: usize = 4000;
5252 const BACKFILL_PAGE: u32 = 100;
5353 /// Accounts or workspaces per directory job.
5454 const DIRECTORY_PAGE: u32 = 200;
55+/// Work items read one at a time in one job, at most.
56+const ITEMS_PER_JOB: usize = 20;
57+
58+/// Stores a work item or pull request by its key, as it is now.
59+pub const UPSERT_ITEM: &str = "INSERT INTO items (kind, key, repo_id, workspace, number, repos, title, body, state, status,
60+ author, requested_by, labels, created_at, updated_at)
61+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
62+ ON CONFLICT (key) DO UPDATE SET repo_id = excluded.repo_id, workspace = excluded.workspace,
63+ number = excluded.number, repos = excluded.repos, title = excluded.title, body = excluded.body,
64+ state = excluded.state, status = excluded.status, author = excluded.author,
65+ requested_by = excluded.requested_by, labels = excluded.labels, updated_at = excluded.updated_at";
5566
5667 /// Where `park` keeps a backfill page in `meta`, by its key's prefix.
5768 const PARKED: &str = "parked_job:";
7687 after: Option<String>,
7788 },
7889 /// Compares a repository's default branch with the index, whole, and
79− /// indexes its issues and pull requests.
90+ /// indexes its pull requests.
8091 Repo { repo_id: String },
8192 /// Indexes what a push to the default branch changed.
8293 Push {
93104 #[serde(default)]
94105 after: Option<String>,
95106 },
107+ /// Indexes a workspace's work items, as many as one list returns.
108+ Workspace { workspace: String },
109+ /// Indexes these work items of a workspace, one at a time.
110+ Items { workspace: String, numbers: Vec<u32> },
96111 }
97112
98113 /// What this service reads of the events it handles.
121136 }
122137
123138 #[derive(Deserialize)]
139+#[serde(rename_all = "camelCase")]
140+struct ItemEvent {
141+ item_id: String,
142+ workspace: String,
143+ number: u32,
144+}
145+
146+#[derive(Deserialize)]
124147 struct UserEvent {
125148 username: String,
126149 }
147170 state: String,
148171 }
149172
173+/// Where a row of `items` belongs and what it is linked to.
174+struct Row<'a> {
175+ kind: &'a str,
176+ key: &'a str,
177+ repo_id: Option<&'a str>,
178+ workspace: Option<&'a str>,
179+ number: u32,
180+ repos: &'a str,
181+}
182+
183+/// What a row of `items` says.
184+struct Text<'a> {
185+ title: &'a str,
186+ body: &'a str,
187+ state: &'a str,
188+ status: &'a str,
189+ /// Who opened it, and for g1t's work, who asked for it.
190+ author: &'a str,
191+ requested_by: Option<&'a str>,
192+ labels: &'a [String],
193+ created_at: &'a str,
194+ updated_at: &'a str,
195+}
196+
150197 fn now() -> String {
151198 rfc3339(now_ms())
152199 }
173220 text.chars().take(max).collect()
174221 }
175222
223+/// Repository ids between bars, as `items.repos` keeps them: `|rep_1|rep_2|`.
224+fn repos_of(ids: &[&str]) -> String {
225+ let mut out = String::from("|");
226+ for id in ids {
227+ out.push_str(id);
228+ out.push('|');
229+ }
230+ out
231+}
232+
233+/// A pull request's key in `items`.
234+fn pull_key(repo_id: &str, number: u32) -> String {
235+ format!("{repo_id}/{number}")
236+}
237+
238+/// The jobs that index the work items `work.migrated` announces: grouped
239+/// by workspace, a few to a job.
240+fn migrated_jobs(migrated: &WorkMigrated) -> Vec<Job> {
241+ let mut by_workspace: Vec<(String, Vec<u32>)> = Vec::new();
242+ for item in &migrated.items {
243+ let workspace = item.workspace.to_lowercase();
244+ match by_workspace.iter_mut().find(|(slug, _)| *slug == workspace) {
245+ Some((_, numbers)) => numbers.push(item.number),
246+ None => by_workspace.push((workspace, vec![item.number])),
247+ }
248+ }
249+ let mut jobs = Vec::new();
250+ for (workspace, numbers) in by_workspace {
251+ for group in numbers.chunks(ITEMS_PER_JOB) {
252+ jobs.push(Job::Items { workspace: workspace.clone(), numbers: group.to_vec() });
253+ }
254+ }
255+ jobs
256+}
257+
176258 impl Search {
177259 /// The workspace itself, through no one: how this service reads a
178260 /// repository, private or not, to index it.
272354 store::prepare(&self.db, "DELETE FROM chunks WHERE fid IN (SELECT fid FROM files WHERE repo_id = ?)", id())?,
273355 store::prepare(&self.db, "DELETE FROM files WHERE repo_id = ?", id())?,
274356 store::prepare(&self.db, "DELETE FROM pending WHERE repo_id = ?", id())?,
275− store::prepare(&self.db, "DELETE FROM items WHERE repo_id = ?", id())?,
357+ // Its work items stay: they are their workspace's, now for
358+ // its members only.
359+ store::prepare(&self.db, "DELETE FROM items WHERE kind = 'pull' AND repo_id = ?", id())?,
276360 store::prepare(&self.db, "DELETE FROM repo_topics WHERE repo_id = ?", id())?,
277361 store::prepare(&self.db, "DELETE FROM repos WHERE repo_id = ?", id())?,
278362 ],
517601 Ok(statements)
518602 }
519603
520− // ---- Issues and pull requests ------------------------------------------
604+ // ---- Work items and pull requests -------------------------------------
521605
522− fn issue_row(&self, repo_id: &str, issue: &Issue) -> Result<worker::D1PreparedStatement> {
523− let status = match issue.state {
606+ fn work_item_row(&self, item: &WorkItem) -> Result<worker::D1PreparedStatement> {
607+ let status = match item.state {
524608 State::Open => "open",
525− State::Closed => issue.reason.map_or("completed", |reason| reason.as_str()),
609+ State::Closed => item.reason.map_or("completed", |reason| reason.as_str()),
526610 };
611+ let linked: Vec<&str> = item.repos.iter().map(|repo| repo.repo_id.as_str()).collect();
527612 self.item_row(
528− repo_id,
529− "issue",
530− issue.number,
531− &issue.title,
532− &issue.body,
533− if issue.state == State::Open { "open" } else { "closed" },
534− status,
535− (&issue.author.username, issue.requested_by.as_ref().map(|user| user.username.as_str())),
536− &issue.labels,
537− &issue.created_at,
538− &issue.updated_at,
613+ Row {
614+ kind: "item",
615+ key: &item.id,
616+ repo_id: item.origin().map(|repo| repo.repo_id.as_str()),
617+ workspace: Some(&item.workspace.to_lowercase()),
618+ number: item.number,
619+ repos: &repos_of(&linked),
620+ },
621+ Text {
622+ title: &item.title,
623+ body: &item.body,
624+ state: if item.state == State::Open { "open" } else { "closed" },
625+ status,
626+ author: &item.author.username,
627+ requested_by: item.requested_by.as_ref().map(|user| user.username.as_str()),
628+ labels: &item.labels,
629+ created_at: &item.created_at,
630+ updated_at: &item.updated_at,
631+ },
539632 )
540633 }
541634
542− fn pull_row(&self, repo_id: &str, pull: &Pull, labels: &[String]) -> Result<worker::D1PreparedStatement> {
635+ fn pull_row(&self, repo_id: &str, pull: &Pull) -> Result<worker::D1PreparedStatement> {
543636 self.item_row(
544− repo_id,
545− "pull",
546− pull.number,
547− &pull.title,
548− pull.body.as_deref().unwrap_or_default(),
549− if pull.status.is_active() { "open" } else { "closed" },
550− pull.status.as_str(),
551− (&pull.author.username, pull.requested_by.as_ref().map(|user| user.username.as_str())),
552− labels,
553− &pull.created_at,
554− &pull.updated_at,
637+ Row {
638+ kind: "pull",
639+ key: &pull_key(repo_id, pull.number),
640+ repo_id: Some(repo_id),
641+ workspace: None,
642+ number: pull.number,
643+ repos: "|",
644+ },
645+ Text {
646+ title: &pull.title,
647+ body: pull.body.as_deref().unwrap_or_default(),
648+ state: if pull.status.is_active() { "open" } else { "closed" },
649+ status: pull.status.as_str(),
650+ author: &pull.author.username,
651+ requested_by: pull.requested_by.as_ref().map(|user| user.username.as_str()),
652+ labels: &pull.labels,
653+ created_at: &pull.created_at,
654+ updated_at: &pull.updated_at,
655+ },
555656 )
556657 }
557658
558− #[allow(clippy::too_many_arguments)]
559− fn item_row(
560− &self,
561− repo_id: &str,
562− kind: &str,
563− number: u32,
564− title: &str,
565− body: &str,
566− state: &str,
567− status: &str,
568− // Who opened it, and for g1t's work, who asked for it.
569− (author, requested_by): (&str, Option<&str>),
570− labels: &[String],
571− created_at: &str,
572− updated_at: &str,
573− ) -> Result<worker::D1PreparedStatement> {
659+ fn item_row(&self, row: Row<'_>, text: Text<'_>) -> Result<worker::D1PreparedStatement> {
574660 store::prepare(
575661 &self.db,
576− "INSERT INTO items (repo_id, kind, number, title, body, state, status, author, requested_by, labels, created_at, updated_at)
577− VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
578− ON CONFLICT (repo_id, kind, number) DO UPDATE SET title = excluded.title, body = excluded.body,
579− state = excluded.state, status = excluded.status, author = excluded.author,
580− requested_by = excluded.requested_by, labels = excluded.labels, updated_at = excluded.updated_at",
662+ UPSERT_ITEM,
581663 vec![
582− p(repo_id),
583− p(kind),
584− Param::Int(i64::from(number)),
585− p(title),
586− p(&clip(body, BODY_CHARS)),
587− p(state),
588− p(status),
589− p(&author.to_lowercase()),
590− requested_by.map_or(Param::Null, |name| p(&name.to_lowercase())),
591− p(&labels_of(labels)),
592− p(created_at),
593− p(updated_at),
664+ p(row.kind),
665+ p(row.key),
666+ Param::opt(row.repo_id),
667+ Param::opt(row.workspace),
668+ Param::Int(i64::from(row.number)),
669+ p(row.repos),
670+ p(text.title),
671+ p(&clip(text.body, BODY_CHARS)),
672+ p(text.state),
673+ p(text.status),
674+ p(&text.author.to_lowercase()),
675+ text.requested_by.map_or(Param::Null, |name| p(&name.to_lowercase())),
676+ p(&labels_of(text.labels)),
677+ p(text.created_at),
678+ p(text.updated_at),
594679 ],
595680 )
596681 }
597682
598− /// Indexes one issue or pull request as it is now.
599− async fn index_item(&self, pull: bool, repo_id: &str, number: u32) -> Result<()> {
683+ async fn known(&self, repo_id: &str) -> Result<bool> {
684+ let found = store::count(
685+ &self.db,
686+ &Sql { text: "SELECT count(*) AS n FROM repos WHERE repo_id = ?".into(), params: vec![p(repo_id)] },
687+ )
688+ .await?;
689+ Ok(found > 0)
690+ }
691+
692+ /// Indexes one pull request as it is now.
693+ async fn index_pull(&self, repo_id: &str, number: u32) -> Result<()> {
600694 let Some(path) = self.path_of(repo_id).await? else {
601695 return Ok(());
602696 };
603697 let Some(actor) = self.actor(&path.namespace).await? else {
604698 return Ok(());
605699 };
606− // An issue on a repository not yet indexed brings the repository in.
607− let known = store::count(
608− &self.db,
609− &Sql { text: "SELECT count(*) AS n FROM repos WHERE repo_id = ?".into(), params: vec![p(repo_id)] },
610− )
611− .await?;
612− if known == 0 && self.refresh_repo(repo_id).await?.is_none() {
700+ // A pull request on a repository not yet indexed brings the repository in.
701+ if !self.known(repo_id).await? && self.refresh_repo(repo_id).await?.is_none() {
613702 return Ok(());
614703 }
615704 let view = ViewArgs { repo: path, number, viewer: Some(actor), after_seq: 0 };
616− let statement = if pull {
617− let found: Outcome<PullDetail> = g1t_kit::call(&self.work, "get_pull", &view).await?;
618− let Outcome::Ok(detail) = found else { return Ok(()) };
619− let labels = detail.issue.as_ref().map(|issue| issue.labels.clone()).unwrap_or_default();
620− self.pull_row(repo_id, &detail.pull, &labels)?
621− } else {
622− let found: Outcome<IssueDetail> = g1t_kit::call(&self.work, "get_issue", &view).await?;
623− let Outcome::Ok(detail) = found else { return Ok(()) };
624− self.issue_row(repo_id, &detail.issue)?
705+ let found: Outcome<PullDetail> = g1t_kit::call(&self.work, "get_pull", &view).await?;
706+ let Outcome::Ok(detail) = found else { return Ok(()) };
707+ store::run_all(&self.db, vec![self.pull_row(repo_id, &detail.pull)?]).await
708+ }
709+
710+ /// Brings in the repository a work item was filed in, if the index
711+ /// does not have it yet: who may read the item follows it.
712+ async fn ensure_origin(&self, item: &WorkItem) -> Result<()> {
713+ if let Some(origin) = item.origin()
714+ && !self.known(&origin.repo_id).await?
715+ {
716+ self.refresh_repo(&origin.repo_id).await?;
717+ }
718+ Ok(())
719+ }
720+
721+ /// Indexes one work item as it is now. One no longer found is
722+ /// forgotten, when its id is known.
723+ async fn index_work_item(&self, workspace: &str, number: u32, item_id: Option<&str>) -> Result<()> {
724+ if workspace.is_empty() || number == 0 {
725+ return Ok(());
726+ }
727+ let Some(actor) = self.actor(workspace).await? else {
728+ return Ok(());
625729 };
730+ let view = WorkItemViewArgs { workspace: workspace.to_lowercase(), number, viewer: Some(actor) };
731+ let found: Outcome<WorkItemDetail> = g1t_kit::call(&self.work, "get_work_item", &view).await?;
732+ let statement = match found {
733+ Outcome::Ok(detail) => {
734+ self.ensure_origin(&detail.item).await?;
735+ self.work_item_row(&detail.item)?
736+ }
737+ Outcome::Fail(_) => match item_id {
738+ Some(id) => store::prepare(&self.db, "DELETE FROM items WHERE kind = 'item' AND key = ?", vec![p(id)])?,
739+ None => return Ok(()),
740+ },
741+ };
626742 store::run_all(&self.db, vec![statement]).await
627743 }
628744
629− /// Indexes a repository's issues and pull requests, newest first, as
630− /// many as one list returns.
631− async fn index_items(&self, repo_id: &str, path: &RepoPath) -> Result<()> {
632− let Some(actor) = self.actor(&path.namespace).await? else {
745+ /// Indexes a workspace's work items, newest first, as many as one list
746+ /// returns.
747+ async fn index_work_items(&self, workspace: &str) -> Result<()> {
748+ let Some(actor) = self.actor(workspace).await? else {
633749 return Ok(());
634750 };
635− let issues: Outcome<Vec<Issue>> = g1t_kit::call(
751+ let items: Outcome<Vec<WorkItem>> = g1t_kit::call(
636752 &self.work,
637− "list_issues",
638− &ListIssuesArgs { repo: path.clone(), viewer: Some(actor.clone()), state: None, label: None, milestone: None },
753+ "list_work_items",
754+ &ListWorkItemsArgs {
755+ workspace: workspace.to_lowercase(),
756+ viewer: Some(actor),
757+ repo: None,
758+ role: None,
759+ state: None,
760+ label: None,
761+ milestone: None,
762+ assignee: None,
763+ },
639764 )
640765 .await?;
766+ let Outcome::Ok(items) = items else { return Ok(()) };
767+ let mut statements = Vec::new();
768+ for item in items.iter().filter(|item| item.number > 0) {
769+ self.ensure_origin(item).await?;
770+ statements.push(self.work_item_row(item)?);
771+ }
772+ store::run_all(&self.db, statements).await
773+ }
774+
775+ /// Indexes a repository's pull requests, newest first, as many as one
776+ /// list returns.
777+ async fn index_pulls(&self, repo_id: &str, path: &RepoPath) -> Result<()> {
778+ let Some(actor) = self.actor(&path.namespace).await? else {
779+ return Ok(());
780+ };
641781 let pulls: Outcome<Vec<Pull>> = g1t_kit::call(
642782 &self.work,
643783 "list_pulls",
644784 &ListPullsArgs { repo: path.clone(), viewer: Some(actor), state: None, label: None, milestone: None, base: None },
645785 )
646786 .await?;
647− let issues = match issues {
648− Outcome::Ok(issues) => issues,
649− Outcome::Fail(_) => Vec::new(),
650− };
651− let labels: HashMap<u32, &Vec<String>> = issues.iter().map(|issue| (issue.number, &issue.labels)).collect();
787+ let Outcome::Ok(pulls) = pulls else { return Ok(()) };
652788 let mut statements = Vec::new();
653− for issue in &issues {
654− statements.push(self.issue_row(repo_id, issue)?);
655− }
656− if let Outcome::Ok(pulls) = pulls {
657− for pull in &pulls {
658− let labels = pull.issue.and_then(|number| labels.get(&number)).map(|l| l.as_slice()).unwrap_or_default();
659− statements.push(self.pull_row(repo_id, pull, labels)?);
660− }
789+ for pull in &pulls {
790+ statements.push(self.pull_row(repo_id, pull)?);
661791 }
662792 store::run_all(&self.db, statements).await
663793 }
794924 }
795925 }
796926 // A transfer changes the repository's path; refreshing reads it
797− // again by id, issues and pull requests with it.
927+ // again by id.
798928 // Archiving changes nothing searched, but the details are read again.
799929 "repo.created"
800930 | "repo.updated"
824954 self.purge(&repo.repo_id).await?;
825955 }
826956 }
827− // A restored one is indexed again, whole: its code, issues and
828− // pull requests.
957+ // A restored one is indexed again, whole: its code and pull
958+ // requests.
829959 "repo.restored" => {
830960 if let Ok(repo) = serde_json::from_value::<RepoEvent>(data.clone()) {
831961 self.enqueue(vec![Job::Repo { repo_id: repo.repo_id }]).await?;
832962 }
833963 }
834− "issue.opened" | "issue.updated" | "issue.closed" | "issue.reopened" | "issue.assigned" => {
835− if let Ok(item) = serde_json::from_value::<NumberEvent>(data.clone()) {
836− self.index_item(false, &item.repo_id, item.number).await?;
964+ // Every change to a work item: its text, state, labels and the
965+ // repositories it is linked to are all indexed.
966+ kind if kind.starts_with("work_item.") => {
967+ if let Ok(item) = serde_json::from_value::<ItemEvent>(data.clone()) {
968+ self.index_work_item(&item.workspace, item.number, Some(&item.item_id)).await?;
969+ }
970+ }
971+ // Issues moved into work items keep their ids: each is read
972+ // again with its workspace and new number. The SQL migration
973+ // dropped what was kept of them as issues.
974+ "work.migrated" => {
975+ if let Ok(migrated) = serde_json::from_value::<WorkMigrated>(data.clone()) {
976+ self.enqueue(migrated_jobs(&migrated)).await?;
837977 }
838978 }
839979 "pull.opened" | "pull.ready" | "pull.updated" | "pull.closed" | "pull.reopened"
840− | "pull.converted_to_draft" | "pull.merged" => {
841− if let Ok(item) = serde_json::from_value::<NumberEvent>(data.clone()) {
842− self.index_item(true, &item.repo_id, item.number).await?;
980+ | "pull.converted_to_draft" | "pull.merged" | "pull.labeled" | "pull.unlabeled" => {
981+ if let Ok(pull) = serde_json::from_value::<NumberEvent>(data.clone()) {
982+ self.index_pull(&pull.repo_id, pull.number).await?;
843983 }
844984 }
845985 // A deleted account's profile is not found, so indexing it
8861026 event,
8871027 &[
8881028 "UPDATE repos SET namespace = ?1 WHERE namespace = ?2",
1029+ "UPDATE items SET workspace = ?1 WHERE workspace = ?2",
8891030 "UPDATE people SET slug = ?1 WHERE kind = 'workspace' AND slug = ?2",
8901031 ],
8911032 )
9201061 )
9211062 .await?;
9221063 }
923− self.index_items(&repo_id, &path).await?;
1064+ self.index_pulls(&repo_id, &path).await?;
9241065 self.reconcile(&repo_id).await
9251066 }
9261067 Job::Push { repo_id, before, after } => {
9761117 statements.push(self.person_row(&kind, &reference, entry)?);
9771118 }
9781119 store::run_all(&self.db, statements).await?;
1120+ // Each workspace's work items, a job each.
1121+ let mut jobs: Vec<Job> = if kind == "workspace" {
1122+ page.entries.iter().map(|entry| Job::Workspace { workspace: entry.slug.to_lowercase() }).collect()
1123+ } else {
1124+ Vec::new()
1125+ };
9791126 if let Some(next) = page.next {
980− self.enqueue(vec![Job::Directory { kind, after: Some(next) }]).await?;
1127+ jobs.push(Job::Directory { kind, after: Some(next) });
9811128 }
1129+ self.enqueue(jobs).await
1130+ }
1131+ Job::Workspace { workspace } => self.index_work_items(&workspace).await,
1132+ Job::Items { workspace, numbers } => {
1133+ for number in numbers {
1134+ self.index_work_item(&workspace, number, None).await?;
1135+ }
9821136 Ok(())
9831137 }
9841138 }
10231177 }
10241178
10251179 #[test]
1180+ fn migrated_items_are_read_again_by_workspace_a_few_at_a_time() {
1181+ use g1t_contracts::work::MigratedItem;
1182+ let moved = |workspace: &str, number: u32| MigratedItem {
1183+ repo_id: "rep_1".into(),
1184+ legacy_number: number,
1185+ workspace: workspace.into(),
1186+ number: number + 100,
1187+ item_id: format!("iss_{number}"),
1188+ };
1189+ let mut items: Vec<MigratedItem> = (1..=25).map(|n| moved("acme", n)).collect();
1190+ items.insert(3, moved("Beta", 7));
1191+ let jobs = migrated_jobs(&WorkMigrated { page: 1, items, done: false });
1192+ assert_eq!(jobs.len(), 3);
1193+ assert_eq!(jobs[0], Job::Items { workspace: "acme".into(), numbers: (101..=120).collect() });
1194+ assert_eq!(jobs[1], Job::Items { workspace: "acme".into(), numbers: (121..=125).collect() });
1195+ assert_eq!(jobs[2], Job::Items { workspace: "beta".into(), numbers: vec![107] });
1196+ let value = serde_json::to_value(&jobs[2]).unwrap();
1197+ assert_eq!(value, serde_json::json!({ "type": "items", "workspace": "beta", "numbers": [107] }));
1198+ }
1199+
1200+ #[test]
1201+ fn keys_and_linked_repositories() {
1202+ assert_eq!(pull_key("rep_1", 12), "rep_1/12");
1203+ assert_eq!(repos_of(&["rep_1", "rep_2"]), "|rep_1|rep_2|");
1204+ assert_eq!(repos_of(&[]), "|");
1205+ }
1206+
1207+ #[test]
10261208 fn pull_statuses_have_names() {
10271209 assert_eq!(PullStatus::Merged.as_str(), "merged");
10281210 }
+5−3
22 //!
33 //! - **What it searches.** Repositories (name, description, topics, the
44 //! opening of the README), code on default branches (paths and
5−//! contents), issues and pull requests (titles and bodies), people and
6−//! workspaces. Everything public, and private content the viewer is a
7−//! member of.
5+//! contents), work items and pull requests (titles and bodies), people
6+//! and workspaces. Everything public, and private content the viewer is
7+//! a member of.
88 //! - **The index.** D1 with FTS5, a database that costs nothing while no
99 //! one uses it: words for prose (`unicode61`), every run of three
1010 //! characters for code (`trigram`), each table kept in step with its
2020 //! `g1t_contracts::search`.
2121
2222 mod index;
23+#[cfg(test)]
24+mod migrated;
2325 mod query;
2426 mod read;
2527 mod rules;
+245−0
1+//! The search database as the migrations make it, for tests: SQLite (which
2+//! D1 is), each migration in a transaction of its own as D1 applies it.
3+//! What each search asks of it is run here too.
4+
5+use rusqlite::Connection;
6+use rusqlite::types::Value;
7+
8+use crate::sql::{Param, Sql};
9+
10+/// The migrations, in order: each file's name and SQL.
11+fn migrations() -> Vec<(String, String)> {
12+ let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("migrations");
13+ let mut names: Vec<String> = std::fs::read_dir(&dir)
14+ .unwrap()
15+ .map(|entry| entry.unwrap().file_name().into_string().unwrap())
16+ .filter(|name| name.ends_with(".sql"))
17+ .collect();
18+ names.sort();
19+ names.into_iter().map(|name| (name.clone(), std::fs::read_to_string(dir.join(&name)).unwrap())).collect()
20+}
21+
22+fn apply(db: &Connection, name: &str, sql: &str) {
23+ db.execute_batch(&format!("BEGIN;\n{sql}\nCOMMIT;")).unwrap_or_else(|error| panic!("{name}: {error}"));
24+}
25+
26+/// A database with the migrations before `name` applied.
27+fn database_before(name: &str) -> Connection {
28+ let db = Connection::open_in_memory().unwrap();
29+ for (file, sql) in migrations() {
30+ if file.as_str() >= name {
31+ break;
32+ }
33+ apply(&db, &file, &sql);
34+ }
35+ db
36+}
37+
38+fn migrate_from(db: &Connection, name: &str) {
39+ for (file, sql) in migrations() {
40+ if file.as_str() >= name {
41+ apply(db, &file, &sql);
42+ }
43+ }
44+}
45+
46+fn database() -> Connection {
47+ let db = Connection::open_in_memory().unwrap();
48+ for (file, sql) in migrations() {
49+ apply(&db, &file, &sql);
50+ }
51+ db
52+}
53+
54+fn values(params: &[Param]) -> Vec<Value> {
55+ params
56+ .iter()
57+ .map(|param| match param {
58+ Param::Text(text) => Value::Text(text.clone()),
59+ Param::Int(number) => Value::Integer(*number),
60+ Param::Null => Value::Null,
61+ })
62+ .collect()
63+}
64+
65+fn run(db: &Connection, text: &str, params: Vec<Param>) {
66+ db.execute(text, rusqlite::params_from_iter(values(&params))).unwrap_or_else(|error| panic!("{text}: {error}"));
67+}
68+
69+/// The `number` column of every row `sql` returns, in order.
70+fn numbers(db: &Connection, sql: &Sql) -> Vec<i64> {
71+ let mut statement = db.prepare(&sql.text).unwrap_or_else(|error| panic!("{}: {error}", sql.text));
72+ let index = statement.column_index("number").unwrap();
73+ statement
74+ .query_map(rusqlite::params_from_iter(values(&sql.params)), |row| row.get::<_, i64>(index))
75+ .unwrap()
76+ .map(Result::unwrap)
77+ .collect()
78+}
79+
80+#[cfg(test)]
81+mod tests {
82+ use g1t_contracts::access::{BasePermission, RepoGrant, RepoRole};
83+ use g1t_contracts::{Membership, User, Viewer};
84+
85+ use super::*;
86+ use crate::index::UPSERT_ITEM;
87+ use crate::query::{ItemKind, Query};
88+ use crate::sql::{Page, items_count, items_page};
89+ use crate::visibility::Reader;
90+
91+ const PAGE: Page = Page { limit: 20, offset: 0 };
92+
93+ fn repo(db: &Connection, id: &str, namespace: &str, name: &str, private: bool) {
94+ run(
95+ db,
96+ "INSERT INTO repos (repo_id, namespace, name, private, created_at, updated_at) VALUES (?, ?, ?, ?, '', '')",
97+ vec![Param::from(id), Param::from(namespace), Param::from(name), Param::Int(i64::from(private))],
98+ );
99+ }
100+
101+ /// A work item of acme numbered `number`, filed in `origin`, linked to `repos`.
102+ fn item(db: &Connection, number: i64, origin: Option<&str>, repos: &str, title: &str) {
103+ run(
104+ db,
105+ UPSERT_ITEM,
106+ vec![
107+ Param::from("item"),
108+ Param::Text(format!("wki_{number}")),
109+ Param::opt(origin),
110+ Param::from("acme"),
111+ Param::Int(number),
112+ Param::from(repos),
113+ Param::from(title),
114+ Param::from("the body"),
115+ Param::from("open"),
116+ Param::from("open"),
117+ Param::from("ana"),
118+ Param::Null,
119+ Param::from("|bug|"),
120+ Param::from("2026-10-01T00:00:00Z"),
121+ Param::Text(format!("2026-10-01T00:00:{number:02}Z")),
122+ ],
123+ );
124+ }
125+
126+ fn member(base: BasePermission) -> Viewer {
127+ Some(User {
128+ id: "usr_1".into(),
129+ username: "ana".into(),
130+ workspaces: vec![Membership { base_permission: Some(base), ..Membership::member("acme") }],
131+ ..User::default()
132+ })
133+ }
134+
135+ fn outsider(grants: &[&str]) -> Viewer {
136+ Some(User {
137+ id: "usr_2".into(),
138+ username: "bo".into(),
139+ grants: grants
140+ .iter()
141+ .map(|id| RepoGrant { repo_id: (*id).into(), workspace: "acme".into(), role: RepoRole::Read, team: None })
142+ .collect(),
143+ ..User::default()
144+ })
145+ }
146+
147+ /// acme/web is public, acme/api private; item 1 was filed in web, 2 in
148+ /// api (and targets web), 3 in no repository (and targets web).
149+ fn filled() -> Connection {
150+ let db = database();
151+ repo(&db, "rep_web", "acme", "web", false);
152+ repo(&db, "rep_api", "acme", "api", true);
153+ item(&db, 1, Some("rep_web"), "|rep_web|", "crash on login");
154+ item(&db, 2, Some("rep_api"), "|rep_api|rep_web|", "crash in the api");
155+ item(&db, 3, None, "|rep_web|", "crash everywhere");
156+ db
157+ }
158+
159+ fn found(db: &Connection, text: &str, viewer: &Viewer) -> Vec<i64> {
160+ let query = Query::parse(text);
161+ let reader = Reader::of(viewer);
162+ let mut page = numbers(db, &items_page(&query, &reader, ItemKind::Item, PAGE));
163+ page.sort();
164+ page
165+ }
166+
167+ #[test]
168+ fn issues_are_dropped_and_pull_requests_kept() {
169+ let db = database_before("0003");
170+ repo(&db, "rep_web", "acme", "web", false);
171+ for (kind, number, title) in [("issue", 1, "crash on login"), ("pull", 2, "fix the crash")] {
172+ run(
173+ &db,
174+ "INSERT INTO items (repo_id, kind, number, title, body, state, status, author, labels, created_at, updated_at, requested_by)
175+ VALUES ('rep_web', ?, ?, ?, 'body', 'open', 'open', 'ana', '|bug|', '', '', 'bo')",
176+ vec![Param::from(kind), Param::Int(number), Param::from(title)],
177+ );
178+ }
179+ migrate_from(&db, "0003");
180+ let rows: Vec<(String, String, Option<String>, String, String)> = db
181+ .prepare("SELECT kind, key, workspace, repos, requested_by FROM items")
182+ .unwrap()
183+ .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)))
184+ .unwrap()
185+ .map(Result::unwrap)
186+ .collect();
187+ assert_eq!(rows, vec![("pull".into(), "rep_web/2".into(), None, "|".into(), "bo".into())]);
188+ // The full-text index holds the pull request, and nothing of the issue.
189+ let pulls = numbers(&db, &items_page(&Query::parse("crash"), &Reader::default(), ItemKind::Pull, PAGE));
190+ assert_eq!(pulls, vec![2]);
191+ let matched: i64 = db.query_row("SELECT count(*) FROM items_fts WHERE items_fts MATCH 'login'", [], |row| row.get(0)).unwrap();
192+ assert_eq!(matched, 0);
193+ }
194+
195+ #[test]
196+ fn a_work_item_indexed_again_is_updated_in_place() {
197+ let db = filled();
198+ item(&db, 1, Some("rep_web"), "|rep_web|rep_api|", "a new title");
199+ let rows: i64 = db.query_row("SELECT count(*) FROM items WHERE key = 'wki_1'", [], |row| row.get(0)).unwrap();
200+ assert_eq!(rows, 1);
201+ assert_eq!(found(&db, "title", &None), vec![1]);
202+ assert!(found(&db, "login", &None).is_empty());
203+ }
204+
205+ #[test]
206+ fn members_see_every_work_item() {
207+ let db = filled();
208+ // Even a member whose base permission reads no repository.
209+ assert_eq!(found(&db, "crash", &member(BasePermission::None)), vec![1, 2, 3]);
210+ assert_eq!(found(&db, "crash", &member(BasePermission::Read)), vec![1, 2, 3]);
211+ }
212+
213+ #[test]
214+ fn others_see_what_was_filed_where_they_read() {
215+ let db = filled();
216+ // Signed out, or in no way part of acme: what was filed in its public repository.
217+ assert_eq!(found(&db, "crash", &None), vec![1]);
218+ assert_eq!(found(&db, "crash", &outsider(&[])), vec![1]);
219+ // Given a role on the private one: what was filed there too, never
220+ // what was filed in no repository.
221+ assert_eq!(found(&db, "crash", &outsider(&["rep_api"])), vec![1, 2]);
222+ let count = items_count(&Query::parse("crash"), &Reader::of(&None), ItemKind::Item);
223+ assert_eq!(numbers_count(&db, &count), 1);
224+ }
225+
226+ #[test]
227+ fn repo_matches_any_linked_repository_and_org_the_workspace() {
228+ let db = filled();
229+ let viewer = member(BasePermission::Read);
230+ assert_eq!(found(&db, "crash repo:acme/web", &viewer), vec![1, 2, 3]);
231+ assert_eq!(found(&db, "crash repo:acme/api", &viewer), vec![2]);
232+ assert_eq!(found(&db, "crash -repo:acme/api", &viewer), vec![1, 3]);
233+ assert_eq!(found(&db, "crash org:acme", &viewer), vec![1, 2, 3]);
234+ assert_eq!(found(&db, "workspace:other", &viewer), Vec::<i64>::new());
235+ // Public: filed in a public repository. Private: anything else.
236+ assert_eq!(found(&db, "crash is:public", &viewer), vec![1]);
237+ assert_eq!(found(&db, "crash is:private", &viewer), vec![2, 3]);
238+ // Without words, by qualifiers alone.
239+ assert_eq!(found(&db, "is:open label:bug", &viewer), vec![1, 2, 3]);
240+ }
241+
242+ fn numbers_count(db: &Connection, sql: &Sql) -> i64 {
243+ db.query_row(&sql.text, rusqlite::params_from_iter(values(&sql.params)), |row| row.get(0)).unwrap()
244+ }
245+}
+33−20
3838
3939 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
4040 pub enum ItemKind {
41− Issue,
41+ Item,
4242 Pull,
4343 }
4444
201201 return;
202202 }
203203 match lower {
204− "issue" | "issues" => self.kind = Some(ItemKind::Issue),
204+ "item" | "items" => self.kind = Some(ItemKind::Item),
205205 "pr" | "pull" | "pulls" | "pull-request" => self.kind = Some(ItemKind::Pull),
206206 "open" => self.state = Some(ItemState::Open),
207207 "closed" => self.state = Some(ItemState::Closed),
244244 }
245245
246246 /// The tab a query is about when none is named: `type:` if given, then
247− /// pull requests for `is:pr`, issues for `is:issue` or anything only
248− /// issues have, code for `path:`, else repositories.
247+ /// pull requests for `is:pr`, work items for `is:item` or anything only
248+ /// work items and pull requests have, code for `path:`, else
249+ /// repositories.
249250 pub fn default_type(&self) -> SearchType {
250251 if let Some(kind) = self.type_hint {
251252 return kind;
252253 }
253254 match self.kind {
254255 Some(ItemKind::Pull) => return SearchType::Pulls,
255− Some(ItemKind::Issue) => return SearchType::Issues,
256+ Some(ItemKind::Item) => return SearchType::Items,
256257 None => {}
257258 }
258259 if !self.paths.is_empty() {
259260 return SearchType::Code;
260261 }
261262 if self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty() {
262− return SearchType::Issues;
263+ return SearchType::Items;
263264 }
264265 SearchType::Repositories
265266 }
266267
267268 /// Whether the query can have results of `kind`: qualifiers only one
268269 /// kind has rule the others out (`path:` is only code, `label:` only
269− /// issues and pull requests).
270+ /// work items and pull requests).
270271 pub fn applies_to(&self, kind: SearchType) -> bool {
271272 let item_only = self.state.is_some() || !self.authors.is_empty() || !self.labels.is_empty();
272273 let repo_scoped = !self.repos.is_empty() || !self.owners.is_empty() || self.visibility.is_some();
273274 match kind {
274275 SearchType::Repositories => self.paths.is_empty() && self.kind.is_none() && !item_only,
275276 SearchType::Code => self.kind.is_none() && !item_only,
276− SearchType::Issues => {
277+ SearchType::Items => {
277278 self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Pull)
278279 && !matches!(self.state, Some(ItemState::Merged | ItemState::Draft))
279280 }
280− SearchType::Pulls => self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Issue),
281+ SearchType::Pulls => self.paths.is_empty() && self.languages.is_empty() && self.kind != Some(ItemKind::Item),
281282 SearchType::People => {
282283 self.paths.is_empty() && self.languages.is_empty() && self.kind.is_none() && !item_only && !repo_scoped
283284 }
284285 }
285286 }
286287
287− /// The full-text expression for prose (repositories, issues, people):
288+ /// The full-text expression for prose (repositories, work items, people):
288289 /// every word, each a prefix as people type it, phrases exact, `-`
289290 /// words left out. `None` when there are no words to match.
290291 pub fn prose_match(&self) -> Option<String> {
358359 qualifier("author", &self.authors);
359360 qualifier("label", &self.labels);
360361 match self.kind {
361− Some(ItemKind::Issue) => out.push("is:issue".into()),
362+ Some(ItemKind::Item) => out.push("is:item".into()),
362363 Some(ItemKind::Pull) if !matches!(self.state, Some(ItemState::Merged | ItemState::Draft)) => out.push("is:pr".into()),
363364 _ => {}
364365 }
463464 }
464465
465466 #[test]
467+ fn work_items_are_items() {
468+ assert_eq!(Query::parse("is:item").kind, Some(ItemKind::Item));
469+ // The old names are not kept.
470+ let old = Query::parse("is:issue type:issues");
471+ assert_eq!(old.kind, None);
472+ assert_eq!(old.type_hint, None);
473+ assert_eq!(old.notes.len(), 2);
474+ }
475+
476+ #[test]
466477 fn mistakes_are_said() {
467478 let query = Query::parse("repo:acme is:spaceship label:");
468479 assert_eq!(query.notes.len(), 3);
474485 assert_eq!(Query::parse("router").default_type(), SearchType::Repositories);
475486 assert_eq!(Query::parse("router path:src").default_type(), SearchType::Code);
476487 assert_eq!(Query::parse("crash is:pr").default_type(), SearchType::Pulls);
477− assert_eq!(Query::parse("crash is:open").default_type(), SearchType::Issues);
478− assert_eq!(Query::parse("crash label:bug").default_type(), SearchType::Issues);
488+ assert_eq!(Query::parse("crash is:open").default_type(), SearchType::Items);
489+ assert_eq!(Query::parse("crash is:item").default_type(), SearchType::Items);
490+ assert_eq!(Query::parse("crash type:items").default_type(), SearchType::Items);
491+ assert_eq!(Query::parse("crash label:bug").default_type(), SearchType::Items);
479492 assert_eq!(Query::parse("crash type:code").default_type(), SearchType::Code);
480493 }
481494
484497 let code = Query::parse("x path:src");
485498 assert!(code.applies_to(SearchType::Code));
486499 assert!(!code.applies_to(SearchType::Repositories));
487− assert!(!code.applies_to(SearchType::Issues));
488− let issues = Query::parse("x label:bug");
489− assert!(issues.applies_to(SearchType::Issues) && issues.applies_to(SearchType::Pulls));
490− assert!(!issues.applies_to(SearchType::Code) && !issues.applies_to(SearchType::People));
500+ assert!(!code.applies_to(SearchType::Items));
501+ let items = Query::parse("x label:bug");
502+ assert!(items.applies_to(SearchType::Items) && items.applies_to(SearchType::Pulls));
503+ assert!(!items.applies_to(SearchType::Code) && !items.applies_to(SearchType::People));
491504 let merged = Query::parse("x is:merged");
492− assert!(merged.applies_to(SearchType::Pulls) && !merged.applies_to(SearchType::Issues));
505+ assert!(merged.applies_to(SearchType::Pulls) && !merged.applies_to(SearchType::Items));
493506 let scoped = Query::parse("x repo:a/b");
494507 assert!(!scoped.applies_to(SearchType::People));
495508 assert!(Query::parse("ana").applies_to(SearchType::People));
528541
529542 #[test]
530543 fn the_query_is_written_back_tidied() {
531− let query = Query::parse(" Parse REPO:acme/web \"two words\" is:issue ");
532− assert_eq!(query.display(), "Parse \"two words\" repo:acme/web is:issue");
544+ let query = Query::parse(" Parse REPO:acme/web \"two words\" is:item ");
545+ assert_eq!(query.display(), "Parse \"two words\" repo:acme/web is:item");
533546 }
534547
535548 #[test]
+112−27
9494 #[derive(Deserialize)]
9595 struct ItemRow {
9696 kind: String,
97+ #[serde(default)]
98+ workspace: Option<String>,
9799 number: u32,
98100 title: String,
99101 state: String,
103105 requested_by: Option<String>,
104106 labels: String,
105107 updated_at: String,
106− repo_id: String,
107− namespace: String,
108− name: String,
109− private: u8,
108+ // A work item's origin, if it has one, as the index holds it.
109+ #[serde(default)]
110+ repo_id: Option<String>,
111+ #[serde(default)]
112+ namespace: Option<String>,
113+ #[serde(default)]
114+ name: Option<String>,
115+ #[serde(default)]
116+ private: Option<u8>,
110117 body: Option<String>,
111118 }
112119
119126 avatar: Option<String>,
120127 }
121128
122−/// A result, and the repository whose visibility decides whether it is shown.
129+/// A result, and what decides whether it is shown: the repository whose
130+/// visibility it has, and for a work item its workspace, whose members see
131+/// it whatever its repository (with none, only they do).
123132 struct Found {
124133 hit: Hit,
125134 repo: Option<Indexed>,
135+ workspace: Option<String>,
126136 }
127137
138+impl Found {
139+ fn shown(&self, reader: &Reader, keep: &HashSet<String>) -> bool {
140+ if let Some(workspace) = &self.workspace
141+ && reader.member_of(workspace)
142+ {
143+ return true;
144+ }
145+ match &self.repo {
146+ Some(repo) => keep.contains(&repo.repo_id),
147+ None => self.workspace.is_none(),
148+ }
149+ }
150+}
151+
128152 fn repo_hit(row: RepoRow, terms: &[String]) -> Found {
129153 let full = format!("{}/{}", row.namespace, row.name);
130154 let mut hit = Hit::new(HitKind::Repository, full.clone(), format!("/{}/{}", row.namespace, row.name));
146170 namespace: row.namespace,
147171 private: row.private != 0,
148172 }),
173+ workspace: None,
149174 }
150175 }
151176
152−fn item_hit(row: ItemRow, terms: &[String]) -> Found {
177+fn item_hit(row: ItemRow, terms: &[String]) -> Option<Found> {
153178 let pull = row.kind == "pull";
154− let full = format!("{}/{}", row.namespace, row.name);
155− let url = format!("/{full}/{}/{}", if pull { "pull" } else { "issues" }, row.number);
156− let mut hit = Hit::new(if pull { HitKind::Pull } else { HitKind::Issue }, row.title, url);
157− hit.repo = Some(full);
158− hit.private = row.private != 0;
179+ let repo = match (row.repo_id, row.namespace, row.name) {
180+ (Some(repo_id), Some(namespace), Some(name)) => {
181+ Some((Indexed { repo_id, namespace, private: row.private.unwrap_or(1) != 0 }, name))
182+ }
183+ _ => None,
184+ };
185+ let full = repo.as_ref().map(|(repo, name)| format!("{}/{name}", repo.namespace));
186+ let (kind, url, workspace) = if pull {
187+ (HitKind::Pull, format!("/{}/pull/{}", full.as_deref()?, row.number), None)
188+ } else {
189+ let workspace = row.workspace?;
190+ (HitKind::Item, format!("/{workspace}/-/work/{}", row.number), Some(workspace))
191+ };
192+ let mut hit = Hit::new(kind, row.title, url);
193+ hit.repo = full;
194+ hit.workspace = workspace.clone();
195+ // A work item filed in no repository is for its workspace's members.
196+ hit.private = repo.as_ref().is_none_or(|(repo, _)| repo.private);
159197 hit.snippet = snippet::prose(row.body.as_deref().unwrap_or_default(), terms);
160198 hit.number = Some(row.number);
161199 hit.state = Some(if pull { row.status } else { row.state });
163201 hit.requested_by = row.requested_by;
164202 hit.labels = split_labels(&row.labels);
165203 hit.updated_at = Some(row.updated_at);
166− Found {
167− hit,
168− repo: Some(Indexed {
169− repo_id: row.repo_id,
170− namespace: row.namespace,
171− private: row.private != 0,
172− }),
173− }
204+ Some(Found { hit, repo: repo.map(|(repo, _)| repo), workspace })
174205 }
175206
176207 fn person_hit(row: PersonRow, terms: &[String]) -> Found {
182213 hit.snippet = row.bio.as_deref().map(|bio| snippet::prose(bio, terms)).unwrap_or_default();
183214 hit.description = row.bio;
184215 hit.avatar = row.avatar;
185− Found { hit, repo: None }
216+ Found { hit, repo: None, workspace: None }
186217 }
187218
188219 impl Search {
240271 async fn admit(&self, viewer: &Viewer, found: Vec<Found>) -> Vec<Hit> {
241272 let indexed: Vec<Indexed> = found.iter().filter_map(|f| f.repo.clone()).collect();
242273 let (keep, _) = self.readable(viewer, &indexed).await;
274+ let reader = Reader::of(viewer);
243275 found
244276 .into_iter()
245− .filter(|f| f.repo.as_ref().is_none_or(|repo| keep.contains(&repo.repo_id)))
277+ .filter(|f| f.shown(&reader, &keep))
246278 .map(|f| f.hit)
247279 .collect()
248280 }
254286 let sql = match kind {
255287 SearchType::Repositories => Some(sql::repos_count(query, reader)),
256288 SearchType::Code => sql::code_plan(query).ok().map(|plan| sql::code_count(&plan, query, reader)),
257− SearchType::Issues => Some(sql::items_count(query, reader, ItemKind::Issue)),
289+ SearchType::Items => Some(sql::items_count(query, reader, ItemKind::Item)),
258290 SearchType::Pulls => Some(sql::items_count(query, reader, ItemKind::Pull)),
259291 SearchType::People => sql::people_select(query).map(|select| sql::count(&select)),
260292 };
316348 namespace: file.namespace.clone(),
317349 private: file.private != 0,
318350 }),
351+ workspace: None,
319352 });
320353 }
321354 Ok(found)
335368 Ok(plan) => self.code_hits(&plan, query, reader, page, terms).await?,
336369 Err(_) => Vec::new(),
337370 },
338− SearchType::Issues | SearchType::Pulls => {
339− let item = if kind == SearchType::Issues { ItemKind::Issue } else { ItemKind::Pull };
371+ SearchType::Items | SearchType::Pulls => {
372+ let item = if kind == SearchType::Items { ItemKind::Item } else { ItemKind::Pull };
340373 store::all::<ItemRow>(&self.db, &sql::items_page(query, reader, item, page))
341374 .await?
342375 .into_iter()
343− .map(|row| item_hit(row, terms))
376+ .filter_map(|row| item_hit(row, terms))
344377 .collect()
345378 }
346379 SearchType::People => match sql::people_page(query, page) {
412445 Ok(Outcome::Ok(results))
413446 }
414447
415− /// A few repositories, issues, pull requests and people, for the
448+ /// A few repositories, work items, pull requests and people, for the
416449 /// command palette. Never code: that is a full search away.
417450 pub async fn suggest(&self, a: SuggestArgs) -> Result<Vec<Hit>> {
418451 let query = Query::parse(&a.query);
423456 let page = Page { limit: SUGGEST_EACH, offset: 0 };
424457 let terms = query.highlights();
425458 let mut found = Vec::new();
426− for kind in [SearchType::Repositories, SearchType::Issues, SearchType::Pulls, SearchType::People] {
459+ for kind in [SearchType::Repositories, SearchType::Items, SearchType::Pulls, SearchType::People] {
427460 match self.page_of(kind, &query, &reader, page, &terms).await {
428461 Ok(more) => found.extend(more),
429462 Err(error) => worker::console_error!("search: suggesting {} failed: {error}", kind.as_str()),
485518 assert_eq!(encode_path("src/ünï.rs"), "src/%C3%BCn%C3%AF.rs");
486519 }
487520
521+ fn row(kind: &str, workspace: Option<&str>, origin: Option<(&str, &str, bool)>) -> ItemRow {
522+ ItemRow {
523+ kind: kind.into(),
524+ workspace: workspace.map(str::to_owned),
525+ number: 12,
526+ title: "Fix it".into(),
527+ state: "open".into(),
528+ status: "open".into(),
529+ author: "ana".into(),
530+ requested_by: None,
531+ labels: "|bug|".into(),
532+ updated_at: String::new(),
533+ repo_id: origin.map(|_| "rep_1".to_owned()),
534+ namespace: origin.map(|(namespace, _, _)| namespace.to_owned()),
535+ name: origin.map(|(_, name, _)| name.to_owned()),
536+ private: origin.map(|(_, _, private)| u8::from(private)),
537+ body: None,
538+ }
539+ }
540+
541+ #[test]
542+ fn work_items_are_at_their_workspace() {
543+ let found = item_hit(row("item", Some("acme"), Some(("acme", "web", false))), &[]).unwrap();
544+ assert_eq!(found.hit.kind, HitKind::Item);
545+ assert_eq!(found.hit.url, "/acme/-/work/12");
546+ assert_eq!(found.hit.repo.as_deref(), Some("acme/web"));
547+ assert_eq!(found.hit.workspace.as_deref(), Some("acme"));
548+ assert!(!found.hit.private);
549+ // Filed in no repository: for members only.
550+ let found = item_hit(row("item", Some("acme"), None), &[]).unwrap();
551+ assert_eq!(found.hit.repo, None);
552+ assert!(found.hit.private);
553+ let pull = item_hit(row("pull", None, Some(("acme", "web", false))), &[]).unwrap();
554+ assert_eq!(pull.hit.url, "/acme/web/pull/12");
555+ }
556+
557+ #[test]
558+ fn the_second_check_lets_members_see_their_work_items() {
559+ use g1t_contracts::{Membership, User};
560+ let member = Reader::of(&Some(User { workspaces: vec![Membership::member("acme")], ..User::default() }));
561+ let none = HashSet::new();
562+ let readable = HashSet::from(["rep_1".to_owned()]);
563+ let filed = item_hit(row("item", Some("acme"), Some(("acme", "web", true))), &[]).unwrap();
564+ let loose = item_hit(row("item", Some("acme"), None), &[]).unwrap();
565+ assert!(filed.shown(&member, &none) && loose.shown(&member, &none));
566+ // Others, only where the repos service says they read its origin.
567+ let outsider = Reader::default();
568+ assert!(!filed.shown(&outsider, &none));
569+ assert!(filed.shown(&outsider, &readable));
570+ assert!(!loose.shown(&outsider, &readable));
571+ }
572+
488573 #[test]
489574 fn labels_and_topics_are_split() {
490575 assert_eq!(split_labels("|bug|good first issue|"), vec!["bug", "good first issue"]);
+63−15
77 use g1t_contracts::search::COUNT_CAP;
88
99 use crate::query::{Filter, ItemKind, ItemState, Query, Visibility, path_pattern};
10−use crate::visibility::{Reader, clause};
10+use crate::visibility::{Reader, clause, item_clause};
1111
1212 /// A value bound to a `?`.
1313 #[derive(Clone, Debug, PartialEq, Serialize)]
321321 }
322322 }
323323
324−// ---- Issues and pull requests ----------------------------------------------
324+// ---- Work items and pull requests ------------------------------------------
325325
326−const ITEM_COLUMNS: &str = "i.kind, i.number, i.title, i.state, i.status, i.author, i.requested_by, i.labels, i.updated_at, \
327− r.repo_id, r.namespace, r.name, r.private";
326+const ITEM_COLUMNS: &str = "i.kind, i.workspace, i.number, i.title, i.state, i.status, i.author, i.requested_by, i.labels, i.updated_at, r.repo_id, r.namespace, r.name, r.private";
328327
328+/// What every query of work items starts with: the visibility check (a
329+/// member of the workspace, or a reader of the origin), then `repo:`
330+/// (any repository it is linked to), `org:` (its workspace) and
331+/// `is:public`/`is:private` (whether its origin is public).
332+fn item_scope(query: &Query, reader: &Reader) -> Where {
333+ let mut out = Where::default();
334+ out.add(
335+ item_clause("i", "r"),
336+ vec![Param::Text(reader.members_json()), Param::Text(reader.namespaces_json()), Param::Text(reader.granted_json())],
337+ );
338+ let linked = "SELECT 1 FROM repos lr WHERE (lr.namespace || '/' || lr.name) IN (SELECT value FROM json_each(?)) AND instr(i.repos, '|' || lr.repo_id || '|') > 0";
339+ if !query.repos.include.is_empty() {
340+ out.add(format!("EXISTS ({linked})"), vec![json(&query.repos.include)]);
341+ }
342+ if !query.repos.exclude.is_empty() {
343+ out.add(format!("NOT EXISTS ({linked})"), vec![json(&query.repos.exclude)]);
344+ }
345+ in_list(&query.owners, "i.workspace", &mut out);
346+ match query.visibility {
347+ Some(Visibility::Public) => out.add("r.private = 0", vec![]),
348+ Some(Visibility::Private) => out.add("COALESCE(r.private, 1) = 1", vec![]),
349+ None => {}
350+ }
351+ out
352+}
353+
329354 fn items_select(query: &Query, reader: &Reader, kind: ItemKind) -> (Sql, bool) {
330− let mut filters = scope(query, reader, "r");
331− filters.add("i.kind = ?", vec![Param::Text(if kind == ItemKind::Issue { "issue" } else { "pull" }.into())]);
355+ // A pull request belongs to its repository; a work item to its
356+ // workspace, with its origin, if any, joined for who may read it.
357+ let (mut filters, join) = match kind {
358+ ItemKind::Item => (item_scope(query, reader), "LEFT JOIN repos r ON r.repo_id = i.repo_id"),
359+ ItemKind::Pull => (scope(query, reader, "r"), "JOIN repos r ON r.repo_id = i.repo_id"),
360+ };
361+ filters.add("i.kind = ?", vec![Param::Text(if kind == ItemKind::Item { "item" } else { "pull" }.into())]);
332362 match query.state {
333363 Some(ItemState::Open) => filters.add("i.state = 'open'", vec![]),
334364 Some(ItemState::Closed) => filters.add("i.state = 'closed'", vec![]),
350380 (
351381 Sql {
352382 text: format!(
353− "SELECT {ITEM_COLUMNS}, snippet(items_fts, 1, '', '', '…', 40) AS body \
354− FROM items_fts JOIN items i ON i.iid = items_fts.rowid JOIN repos r ON r.repo_id = i.repo_id \
355− WHERE items_fts MATCH ? AND {}",
383+ "SELECT {ITEM_COLUMNS}, snippet(items_fts, 1, '', '', '…', 40) AS body FROM items_fts JOIN items i ON i.iid = items_fts.rowid {join} WHERE items_fts MATCH ? AND {}",
356384 filters.text()
357385 ),
358386 params,
363391 None => (
364392 Sql {
365393 text: format!(
366− "SELECT {ITEM_COLUMNS}, substr(i.body, 1, 400) AS body \
367− FROM items i JOIN repos r ON r.repo_id = i.repo_id WHERE {}",
394+ "SELECT {ITEM_COLUMNS}, substr(i.body, 1, 400) AS body FROM items i {join} WHERE {}",
368395 filters.text()
369396 ),
370397 params: filters.params,
465492 repos_count(&query, &reader),
466493 code_page(&plan, &query, &reader, PAGE),
467494 code_count(&plan, &query, &reader),
468− items_page(&query, &reader, ItemKind::Issue, PAGE),
495+ items_page(&query, &reader, ItemKind::Item, PAGE),
496+ items_count(&query, &reader, ItemKind::Item),
497+ items_page(&query, &reader, ItemKind::Pull, PAGE),
469498 items_count(&query, &reader, ItemKind::Pull),
470499 ] {
471500 assert!(sql.text.contains("r.private = 0 OR r.namespace IN"), "{}", sql.text);
496525 }
497526
498527 #[test]
499− fn issue_qualifiers_become_conditions() {
528+ fn work_items_are_for_members_and_readers_of_their_origin() {
529+ let query = Query::parse("crash repo:acme/web -repo:acme/old org:acme is:private");
530+ let sql = items_page(&query, &reader(&["acme"]), ItemKind::Item, PAGE);
531+ assert!(sql.text.contains("LEFT JOIN repos r"));
532+ assert!(sql.text.contains("i.workspace IN (SELECT value FROM json_each(?)) OR ((r.repo_id IS NOT NULL)"));
533+ // repo: is any linked repository, org: the workspace.
534+ assert!(sql.text.contains("EXISTS (SELECT 1 FROM repos lr"));
535+ assert!(sql.text.contains("NOT EXISTS (SELECT 1 FROM repos lr"));
536+ assert!(sql.text.contains("i.workspace IN (SELECT value FROM json_each(?))"));
537+ assert!(sql.text.contains("COALESCE(r.private, 1) = 1"));
538+ assert!(sql.params.contains(&Param::Text(r#"["acme/web"]"#.into())));
539+ assert!(sql.params.contains(&Param::Text("item".into())));
540+ assert_eq!(placeholders(&sql), sql.params.len(), "{}", sql.text);
541+ }
542+
543+ #[test]
544+ fn item_qualifiers_become_conditions() {
500545 let query = Query::parse("crash is:closed author:ana label:bug -label:wontfix");
501− let sql = items_page(&query, &reader(&[]), ItemKind::Issue, PAGE);
546+ let sql = items_page(&query, &reader(&[]), ItemKind::Item, PAGE);
502547 assert!(sql.text.contains("i.state = 'closed'"));
503548 assert!(sql.text.contains("i.author IN"));
504549 assert!(sql.params.contains(&Param::Text("|bug|".into())));
554599 "fn repo:acme/web",
555600 "repo:acme/web path:*.rs",
556601 "crash is:open label:bug author:ana",
602+ "crash repo:acme/web -repo:acme/old org:acme is:private",
557603 "language:rust",
558604 "ana",
559605 ] {
565611 out.push(code_count(&plan, &query, &reader));
566612 out.push(code_pieces(&plan, &[1, 2], 10));
567613 }
568− out.push(items_page(&query, &reader, ItemKind::Issue, PAGE));
614+ out.push(items_page(&query, &reader, ItemKind::Item, PAGE));
615+ out.push(items_count(&query, &reader, ItemKind::Item));
616+ out.push(items_page(&query, &reader, ItemKind::Pull, PAGE));
569617 out.push(items_count(&query, &reader, ItemKind::Pull));
570618 if let Some(sql) = people_page(&query, PAGE) {
571619 out.push(sql);
+50−1
44 //! row is read if its repository is public, as the index last heard,
55 //! in a workspace whose private repositories the viewer reads (an
66 //! owner, or a member whose base permission gives a role), or one they
7−//! were given a role on directly.
7+//! were given a role on directly. A work item is read by every member
8+//! of its workspace, and by anyone who reads the repository it was filed
9+//! in (`access::item_role`); one filed in no repository, by members only.
810 //! 2. On the page about to be returned, against the repos service, which
911 //! owns visibility: anything it does not say the viewer may read now is
1012 //! dropped, and the index is corrected. A repository made private a
3032 /// Repositories the reader was given a role on directly, by id: an
3133 /// outside collaborator's, or a member's beyond the base permission.
3234 pub granted: Vec<String>,
35+ /// Workspaces, by slug, the reader is a member of, so sees every work
36+ /// item of (`access::workspace_role`).
37+ pub members_of: Vec<String>,
3338 }
3439
3540 impl Reader {
4752 .filter(|slug| access::granted(user, RepoRef { id: "", namespace: slug, private: true }).is_some())
4853 .collect(),
4954 granted: user.grants.iter().map(|grant| grant.repo_id.clone()).collect(),
55+ members_of: user
56+ .workspaces
57+ .iter()
58+ .map(|membership| membership.slug.to_lowercase())
59+ .filter(|slug| access::workspace_role(user, slug).is_some())
60+ .collect(),
5061 }
5162 }
5263
7687 pub fn granted_json(&self) -> String {
7788 serde_json::to_string(&self.granted).unwrap_or_else(|_| "[]".into())
7889 }
90+
91+ /// Whether the reader sees every work item of `workspace`.
92+ pub fn member_of(&self, workspace: &str) -> bool {
93+ self.members_of.iter().any(|slug| slug.eq_ignore_ascii_case(workspace))
94+ }
95+
96+ /// [`Reader::members_of`] as JSON, likewise.
97+ pub fn members_json(&self) -> String {
98+ serde_json::to_string(&self.members_of).unwrap_or_else(|_| "[]".into())
99+ }
79100 }
80101
81102 /// The SQL the first check adds to every query, given the repository's
87108 )
88109 }
89110
111+/// The first check for work items, given the item's and its origin's table
112+/// aliases: a member of its workspace, or a reader of its origin, which is
113+/// joined with a `LEFT JOIN` (none: no origin, members only). Its three
114+/// parameters are [`Reader::members_json`], [`Reader::namespaces_json`] and
115+/// [`Reader::granted_json`], in that order.
116+pub fn item_clause(item: &str, origin: &str) -> String {
117+ format!(
118+ "({item}.workspace IN (SELECT value FROM json_each(?)) OR (({origin}.repo_id IS NOT NULL) AND {}))",
119+ clause(origin)
120+ )
121+}
122+
90123 /// A repository as the index holds it.
91124 #[derive(Clone, Debug, PartialEq, Eq)]
92125 pub struct Indexed {
347380 }
348381
349382 #[test]
383+ fn members_see_every_work_item_of_their_workspace() {
384+ // A member whose base permission gives no role on its repositories
385+ // still sees its work items.
386+ let reader = Reader::of(&member_with(BasePermission::None, &[]));
387+ assert!(reader.member_of("ACME"));
388+ assert_eq!(reader.members_json(), r#"["acme"]"#);
389+ assert_eq!(reader.namespaces_json(), "[]");
390+ assert!(!Reader::of(&None).member_of("acme"));
391+ assert!(!Reader::of(&viewer(&["other"])).member_of("acme"));
392+ assert_eq!(
393+ item_clause("i", "r"),
394+ "(i.workspace IN (SELECT value FROM json_each(?)) OR ((r.repo_id IS NOT NULL) AND (r.private = 0 OR r.namespace IN (SELECT value FROM json_each(?)) OR r.repo_id IN (SELECT value FROM json_each(?)))))"
395+ );
396+ }
397+
398+ #[test]
350399 fn a_moved_repository_is_corrected() {
351400 let reader = Reader::of(&None);
352401 let mut moved = repo("rep_4", "newname", false);

Comments

Sign in to comment.