| 1 | //! `get_work`: everything about a work item an agent needs to pick it up, |
| 2 | //! in one call. What it is for and what done means, the repositories it is |
| 3 | //! linked to with their pull requests for it and what landed where, what |
| 4 | //! is still missing, where else it may belong (suggested targets), who is |
| 5 | //! on it, what it waits on and what waits on it, the decisions made about |
| 6 | //! it, and its latest coordination events. |
| 7 | |
| 8 | use std::collections::HashMap; |
| 9 | |
| 10 | use futures_util::future::{try_join, try_join_all, try_join3}; |
| 11 | use g1t_contracts::access::Capability; |
| 12 | use g1t_contracts::decisions::{Decision, DecisionKind, ListDecisionsArgs}; |
| 13 | use g1t_contracts::events::{Event, ListArgs as ListEventsArgs}; |
| 14 | use g1t_contracts::repos::{ReadableArgs, Repo}; |
| 15 | use g1t_contracts::work::*; |
| 16 | use g1t_contracts::{Outcome, Viewer}; |
| 17 | use g1t_kit::now_ms; |
| 18 | use serde::Deserialize; |
| 19 | use worker::Result; |
| 20 | |
| 21 | use super::{SESSION_COLUMNS, SessionRow, unwrap_or_fail}; |
| 22 | use crate::Work; |
| 23 | use crate::items::{self, Held, PastLanding}; |
| 24 | use crate::rows::{PULL_COLUMNS, PullRow}; |
| 25 | |
| 26 | /// How many sessions, decisions, events and suggestions `get_work` gives |
| 27 | /// at most. |
| 28 | const SESSIONS: u32 = 50; |
| 29 | const DECISIONS: u32 = 20; |
| 30 | const EVENTS: u32 = 20; |
| 31 | const SUGGESTIONS: usize = 5; |
| 32 | /// How many earlier landings suggestions are drawn from. |
| 33 | const LANDINGS: u32 = 200; |
| 34 | |
| 35 | #[derive(Deserialize)] |
| 36 | struct PlaceRow { |
| 37 | state: QueueState, |
| 38 | position: u32, |
| 39 | } |
| 40 | |
| 41 | /// A repository of the workspace, and how many files sessions on the work |
| 42 | /// item touched in it. |
| 43 | #[derive(Clone, Debug, Deserialize)] |
| 44 | pub(crate) struct TouchedRow { |
| 45 | pub repo_id: String, |
| 46 | pub files: u32, |
| 47 | } |
| 48 | |
| 49 | /// The repositories where sessions on work item `?1` touched files. |
| 50 | const TOUCHED_FOR: &str = "SELECT s.repo_id AS repo_id, count(*) AS files |
| 51 | FROM session_touches t JOIN sessions s ON s.id = t.session_id |
| 52 | WHERE t.kind = 'file' |
| 53 | AND (s.item_id = ?1 OR s.id IN (SELECT session_id FROM session_links WHERE kind = 'item' AND item_id = ?1)) |
| 54 | GROUP BY s.repo_id"; |
| 55 | |
| 56 | /// The words of `text` that say something: lowercase, four letters or |
| 57 | /// more. |
| 58 | fn words(text: &str) -> Vec<String> { |
| 59 | let mut found: Vec<String> = Vec::new(); |
| 60 | for word in text.split(|c: char| !c.is_alphanumeric()).map(str::to_lowercase) { |
| 61 | if word.chars().count() >= 4 && !found.contains(&word) { |
| 62 | found.push(word); |
| 63 | } |
| 64 | } |
| 65 | found |
| 66 | } |
| 67 | |
| 68 | /// Where a work item may belong besides where it is meant to land already: |
| 69 | /// the workspace's repositories (`candidates`, by id and path), ranked by |
| 70 | /// where work items sharing its labels or title words landed before, by a |
| 71 | /// repository's name in its title or description, and by where sessions on |
| 72 | /// it touched files. Each says why; the likeliest come first. |
| 73 | pub(crate) fn suggest_targets( |
| 74 | item: &WorkItem, |
| 75 | candidates: &[(String, String)], |
| 76 | landed: &[PastLanding], |
| 77 | touched: &[TouchedRow], |
| 78 | ) -> Vec<SuggestedTarget> { |
| 79 | let title_words = words(&item.title); |
| 80 | let text = format!("{}\n{}", item.title, item.body).to_lowercase(); |
| 81 | let mut suggested: Vec<SuggestedTarget> = Vec::new(); |
| 82 | for (repo_id, path) in candidates { |
| 83 | if item.repos.iter().any(|repo| repo.repo_id == *repo_id && repo.role == LinkRole::Target) { |
| 84 | continue; |
| 85 | } |
| 86 | let mut score = 0; |
| 87 | let mut why: Vec<String> = Vec::new(); |
| 88 | let landings: Vec<&PastLanding> = landed.iter().filter(|landing| landing.repo_id == *repo_id).collect(); |
| 89 | let labelled = landings |
| 90 | .iter() |
| 91 | .filter(|landing| { |
| 92 | let labels: Vec<String> = serde_json::from_str(&landing.labels).unwrap_or_default(); |
| 93 | labels.iter().any(|label| item.labels.contains(label)) |
| 94 | }) |
| 95 | .count() as u32; |
| 96 | if labelled > 0 { |
| 97 | score += 3 * labelled; |
| 98 | why.push(format!("{labelled} work items with the same labels landed there")); |
| 99 | } |
| 100 | let alike = landings.iter().filter(|landing| words(&landing.title).iter().any(|word| title_words.contains(word))).count() as u32; |
| 101 | if alike > 0 { |
| 102 | score += 2 * alike; |
| 103 | why.push(format!("{alike} work items with like titles landed there")); |
| 104 | } |
| 105 | let name = path.rsplit('/').next().unwrap_or(path).to_lowercase(); |
| 106 | let names_it = text.split(|c: char| !(c.is_alphanumeric() || c == '-' || c == '_' || c == '.')).any(|word| word == name); |
| 107 | if names_it || text.contains(&path.to_lowercase()) { |
| 108 | score += 5; |
| 109 | why.push(format!("it names {name}")); |
| 110 | } |
| 111 | if let Some(found) = touched.iter().find(|found| found.repo_id == *repo_id) { |
| 112 | score += 4; |
| 113 | why.push(format!("sessions on it touched {} files there", found.files)); |
| 114 | } |
| 115 | if score > 0 { |
| 116 | let mut why = why.join("; "); |
| 117 | if let Some(first) = why.get_mut(0..1) { |
| 118 | first.make_ascii_uppercase(); |
| 119 | } |
| 120 | suggested.push(SuggestedTarget { repo_id: repo_id.clone(), path: path.clone(), why: format!("{why}."), score }); |
| 121 | } |
| 122 | } |
| 123 | suggested.sort_by(|a, b| b.score.cmp(&a.score).then_with(|| a.path.cmp(&b.path))); |
| 124 | suggested.truncate(SUGGESTIONS); |
| 125 | suggested |
| 126 | } |
| 127 | |
| 128 | impl Work { |
| 129 | pub(crate) async fn get_work(&self, a: WorkItemViewArgs) -> Result<Outcome<WorkContext>> { |
| 130 | let held = unwrap_or_fail!(items::for_viewer(self, &a.viewer, &a.workspace, a.number, Capability::Read).await?); |
| 131 | // What went quiet in its repositories is read as it is now. |
| 132 | for repo_id in held.repo_ids().iter().take(10) { |
| 133 | self.sweep(repo_id).await?; |
| 134 | } |
| 135 | let pulls: Vec<Pull> = self |
| 136 | .db |
| 137 | .prepare(format!("SELECT {PULL_COLUMNS} FROM pulls WHERE item_id = ? ORDER BY created_at")) |
| 138 | .bind(&[held.item.id.as_str().into()])? |
| 139 | .all() |
| 140 | .await? |
| 141 | .results::<PullRow>()? |
| 142 | .into_iter() |
| 143 | .map(Pull::from) |
| 144 | .collect(); |
| 145 | // The repositories it is linked to and has pull requests in, as far |
| 146 | // as the viewer may read them. |
| 147 | let mut ids = held.repo_ids(); |
| 148 | for pull in &pulls { |
| 149 | if !ids.contains(&pull.repo_id) { |
| 150 | ids.push(pull.repo_id.clone()); |
| 151 | } |
| 152 | } |
| 153 | let readable: Vec<Repo> = g1t_kit::call(&self.repos, "readable", &ReadableArgs { ids, viewer: a.viewer.clone() }).await?; |
| 154 | let pulls: Vec<Pull> = pulls.into_iter().filter(|pull| readable.iter().any(|repo| repo.id == pull.repo_id)).collect(); |
| 155 | let mut numbers = vec![held.item.number]; |
| 156 | numbers.extend(pulls.iter().map(|pull| pull.number)); |
| 157 | |
| 158 | let people = try_join3( |
| 159 | self.active_claims(&held.item.id), |
| 160 | self.work_sessions(&held, &pulls), |
| 161 | self.suggestions(&held), |
| 162 | ); |
| 163 | let dependencies = try_join(self.blockers(&held.item.workspace, &held.item.blocked_by), items::blocked_items(self, &held)); |
| 164 | let linked = held.repo_ids(); |
| 165 | let history = try_join( |
| 166 | self.coordination_events(&linked, &numbers, COORDINATION_EVENTS.to_vec(), EVENTS), |
| 167 | self.decisions_on(&held, &a.viewer), |
| 168 | ); |
| 169 | let repos = try_join_all(held.item.repos.iter().filter_map(|link| { |
| 170 | let repo = readable.iter().find(|repo| repo.id == link.repo_id)?; |
| 171 | let mine: Vec<&Pull> = pulls.iter().filter(|pull| pull.repo_id == repo.id).collect(); |
| 172 | Some(self.work_repo(link.clone(), repo, mine)) |
| 173 | })); |
| 174 | let (((claims, sessions, suggested_targets), (blocked_by, blocks)), ((recent, decisions), repos)) = |
| 175 | try_join(try_join(people, dependencies), try_join(history, repos)).await?; |
| 176 | Ok(Outcome::Ok(WorkContext { |
| 177 | definition_of_done: definition_of_done(&held.item.body), |
| 178 | missing: held.item.missing(), |
| 179 | item: held.item, |
| 180 | repos, |
| 181 | suggested_targets, |
| 182 | claims, |
| 183 | sessions, |
| 184 | blocked_by, |
| 185 | blocks, |
| 186 | last_integration_failure: decisions.iter().find(|decision| decision.kind == DecisionKind::IntegrationFailure).cloned(), |
| 187 | decisions, |
| 188 | recent, |
| 189 | })) |
| 190 | } |
| 191 | |
| 192 | /// One repository a work item is linked to: its required checks, and |
| 193 | /// its pull requests for the item. |
| 194 | async fn work_repo(&self, link: ItemRepo, repo: &Repo, pulls: Vec<&Pull>) -> Result<WorkRepo> { |
| 195 | let settings = self.default_branch_settings(repo).await?; |
| 196 | let pulls = try_join_all(pulls.into_iter().map(|pull| self.work_pull(repo, pull))).await?; |
| 197 | Ok(WorkRepo { link, required_checks: settings.required_checks, pulls }) |
| 198 | } |
| 199 | |
| 200 | /// Where else a work item may belong ([`suggest_targets`]). |
| 201 | async fn suggestions(&self, held: &Held) -> Result<Vec<SuggestedTarget>> { |
| 202 | #[derive(Deserialize)] |
| 203 | struct Candidate { |
| 204 | repo_id: String, |
| 205 | path: String, |
| 206 | } |
| 207 | let results = self |
| 208 | .db |
| 209 | .batch(vec![ |
| 210 | self.db |
| 211 | .prepare("SELECT repo_id, path FROM repo_workspaces WHERE workspace = ? ORDER BY path LIMIT 500") |
| 212 | .bind(&[held.item.workspace.as_str().into()])?, |
| 213 | self.db.prepare(TOUCHED_FOR).bind(&[held.item.id.as_str().into()])?, |
| 214 | ]) |
| 215 | .await?; |
| 216 | let mut results = results.into_iter(); |
| 217 | let candidates: Vec<(String, String)> = match results.next() { |
| 218 | Some(result) => result.results::<Candidate>()?.into_iter().map(|row| (row.repo_id, row.path)).collect(), |
| 219 | None => Vec::new(), |
| 220 | }; |
| 221 | let landed = items::landed_before(self, held, LANDINGS).await?; |
| 222 | let touched: Vec<TouchedRow> = match results.next() { |
| 223 | Some(result) => result.results()?, |
| 224 | None => Vec::new(), |
| 225 | }; |
| 226 | Ok(suggest_targets(&held.item, &candidates, &landed, &touched)) |
| 227 | } |
| 228 | |
| 229 | /// Sessions on a work item or its pull requests, in any of their |
| 230 | /// repositories, most recently heard from first. Subagents are listed |
| 231 | /// beside the sessions that started them, so sibling agents on one |
| 232 | /// work item see each other; so are sessions tied to it any other way |
| 233 | /// (a claim, a branch, a trailer, `#N`). |
| 234 | async fn work_sessions(&self, held: &Held, pulls: &[Pull]) -> Result<Vec<Session>> { |
| 235 | let now = now_ms(); |
| 236 | let mut pushed: Vec<String> = Vec::new(); |
| 237 | let mut by_repo: HashMap<&str, Vec<u32>> = HashMap::new(); |
| 238 | for pull in pulls { |
| 239 | by_repo.entry(pull.repo_id.as_str()).or_default().push(pull.number); |
| 240 | } |
| 241 | for (repo_id, numbers) in &by_repo { |
| 242 | pushed.extend(self.branch_sessions(repo_id, numbers).await?); |
| 243 | } |
| 244 | let pull_ids: Vec<&str> = pulls.iter().map(|pull| pull.id.as_str()).collect(); |
| 245 | let mut sessions: Vec<Session> = self |
| 246 | .db |
| 247 | .prepare(format!( |
| 248 | "SELECT {SESSION_COLUMNS} FROM sessions |
| 249 | WHERE item_id = ?1 |
| 250 | OR id IN (SELECT value FROM json_each(?3)) |
| 251 | OR EXISTS (SELECT 1 FROM pulls p WHERE p.id IN (SELECT value FROM json_each(?2)) |
| 252 | AND p.repo_id = sessions.repo_id AND p.number = sessions.pull) |
| 253 | OR id IN (SELECT session_id FROM session_links WHERE kind = 'item' AND item_id = ?1) |
| 254 | OR id IN (SELECT l.session_id FROM session_links l JOIN pulls p ON p.repo_id = l.repo_id AND p.number = l.number |
| 255 | WHERE l.kind = 'pull' AND p.id IN (SELECT value FROM json_each(?2))) |
| 256 | ORDER BY state = 'ended', last_seen_at DESC LIMIT ?4" |
| 257 | )) |
| 258 | .bind(&[ |
| 259 | held.item.id.as_str().into(), |
| 260 | serde_json::to_string(&pull_ids)?.into(), |
| 261 | serde_json::to_string(&pushed)?.into(), |
| 262 | SESSIONS.into(), |
| 263 | ])? |
| 264 | .all() |
| 265 | .await? |
| 266 | .results::<SessionRow>()? |
| 267 | .into_iter() |
| 268 | .map(|row| row.at(now)) |
| 269 | .collect(); |
| 270 | self.fill_subagents(&mut sessions).await?; |
| 271 | Ok(sessions) |
| 272 | } |
| 273 | |
| 274 | /// The latest events of `types` about `numbers` in `repos`, newest |
| 275 | /// first. |
| 276 | async fn coordination_events(&self, repos: &[String], numbers: &[u32], types: Vec<&str>, limit: u32) -> Result<Vec<Event>> { |
| 277 | if repos.is_empty() { |
| 278 | return Ok(Vec::new()); |
| 279 | } |
| 280 | g1t_kit::call( |
| 281 | &self.events, |
| 282 | "list", |
| 283 | &ListEventsArgs { |
| 284 | repo_ids: repos.to_vec(), |
| 285 | types: types.into_iter().map(str::to_owned).collect(), |
| 286 | numbers: numbers.to_vec(), |
| 287 | limit: Some(limit), |
| 288 | ..ListEventsArgs::default() |
| 289 | }, |
| 290 | ) |
| 291 | .await |
| 292 | } |
| 293 | |
| 294 | /// Decision records on a work item and its pull requests, newest |
| 295 | /// first, up to [`DECISIONS`]. |
| 296 | async fn decisions_on(&self, held: &Held, viewer: &Viewer) -> Result<Vec<Decision>> { |
| 297 | let args = ListDecisionsArgs { |
| 298 | repo: None, |
| 299 | workspace: held.item.workspace.clone(), |
| 300 | viewer: viewer.clone(), |
| 301 | number: held.item.number, |
| 302 | }; |
| 303 | let mut decisions = match self.list_decisions(args).await? { |
| 304 | Outcome::Ok(decisions) => decisions, |
| 305 | Outcome::Fail(_) => Vec::new(), |
| 306 | }; |
| 307 | decisions.reverse(); |
| 308 | decisions.truncate(DECISIONS as usize); |
| 309 | Ok(decisions) |
| 310 | } |
| 311 | |
| 312 | /// One of the work item's pull requests and where it stands: its |
| 313 | /// required checks, whether it merges, what it overlaps and its place |
| 314 | /// in the merge queue, for those still in progress. |
| 315 | async fn work_pull(&self, repo: &Repo, pull: &Pull) -> Result<WorkPull> { |
| 316 | let mut pull = pull.clone(); |
| 317 | crate::fill_base(&mut pull, repo); |
| 318 | let brief = |pull: &Pull| WorkPull { |
| 319 | repo: items::path_of(repo), |
| 320 | number: pull.number, |
| 321 | title: pull.title.clone(), |
| 322 | status: pull.status, |
| 323 | branch: pull.branch.clone(), |
| 324 | head_commit: pull.head_commit.clone(), |
| 325 | author: pull.owner().into(), |
| 326 | check_status: pull.check_status, |
| 327 | required_checks: Vec::new(), |
| 328 | mergeable: Mergeable::Unknown, |
| 329 | overlaps: Vec::new(), |
| 330 | queue: None, |
| 331 | }; |
| 332 | if !pull.status.is_active() { |
| 333 | return Ok(brief(&pull)); |
| 334 | } |
| 335 | let required = async { |
| 336 | let settings = self.settings_on(repo, &pull).await?; |
| 337 | if settings.required_checks.is_empty() { |
| 338 | return Ok(Vec::new()); |
| 339 | } |
| 340 | let statuses = self.statuses(&repo.id, pull.head_commit.as_deref()).await?; |
| 341 | let checked = self.with_job_checks(&repo.id, pull.head_commit.as_deref(), statuses).await?; |
| 342 | Ok::<_, worker::Error>(required_checks(&settings.required_checks, &checked)) |
| 343 | }; |
| 344 | let place_query = self |
| 345 | .db |
| 346 | .prepare( |
| 347 | "SELECT mine.state AS state, |
| 348 | (SELECT count(*) FROM queue_entries ahead |
| 349 | WHERE ahead.repo_id = mine.repo_id AND ahead.state IN ('waiting', 'testing', 'passed') |
| 350 | AND (ahead.created_at < mine.created_at |
| 351 | OR (ahead.created_at = mine.created_at AND ahead.id < mine.id))) + 1 AS position |
| 352 | FROM queue_entries mine |
| 353 | WHERE mine.pull_id = ? AND mine.state IN ('waiting', 'testing', 'passed') LIMIT 1", |
| 354 | ) |
| 355 | .bind(&[pull.id.as_str().into()])?; |
| 356 | let place = place_query.first::<PlaceRow>(None); |
| 357 | let ((required_checks, (mergeable, _)), (overlaps, place)) = |
| 358 | try_join(try_join(required, self.mergeability(&pull)), try_join(self.overlaps(&pull), place)).await?; |
| 359 | Ok(WorkPull { |
| 360 | required_checks, |
| 361 | mergeable, |
| 362 | overlaps, |
| 363 | queue: place.map(|place| QueuePlace { state: place.state, position: place.position }), |
| 364 | ..brief(&pull) |
| 365 | }) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | #[cfg(test)] |
| 370 | mod tests { |
| 371 | use super::*; |
| 372 | |
| 373 | fn item(title: &str, body: &str, labels: &[&str], repos: Vec<ItemRepo>) -> WorkItem { |
| 374 | serde_json::from_value(serde_json::json!({ |
| 375 | "id": "wki_1", "workspace": "acme", "number": 7, "title": title, "body": body, |
| 376 | "labels": labels, "state": "open", "reason": null, "author": { "id": "usr_1", "username": "ana" }, |
| 377 | "createdAt": "", "updatedAt": "", "closedAt": null, "pullCount": 0, "commentCount": 0, |
| 378 | "repos": repos |
| 379 | })) |
| 380 | .unwrap() |
| 381 | } |
| 382 | |
| 383 | fn landing(repo: &str, title: &str, labels: &str) -> PastLanding { |
| 384 | PastLanding { repo_id: repo.into(), title: title.into(), labels: labels.into() } |
| 385 | } |
| 386 | |
| 387 | #[test] |
| 388 | fn suggestions_say_why_and_come_likeliest_first() { |
| 389 | let candidates = vec![ |
| 390 | ("rep_api".to_owned(), "acme/api".to_owned()), |
| 391 | ("rep_web".to_owned(), "acme/web".to_owned()), |
| 392 | ("rep_docs".to_owned(), "acme/docs".to_owned()), |
| 393 | ]; |
| 394 | let item = item("Login page shows a stale avatar", "The web client caches it.", &["bug"], Vec::new()); |
| 395 | let landed = vec![ |
| 396 | landing("rep_api", "Avatar upload fails", r#"["bug"]"#), |
| 397 | landing("rep_api", "Rate limits", r#"["enhancement"]"#), |
| 398 | landing("rep_docs", "Typo", "[]"), |
| 399 | ]; |
| 400 | let touched = vec![TouchedRow { repo_id: "rep_web".into(), files: 3 }]; |
| 401 | let suggested = suggest_targets(&item, &candidates, &landed, &touched); |
| 402 | let paths: Vec<&str> = suggested.iter().map(|s| s.path.as_str()).collect(); |
| 403 | assert_eq!(paths, ["acme/web", "acme/api"]); |
| 404 | assert_eq!(suggested[0].why, "It names web; sessions on it touched 3 files there."); |
| 405 | assert_eq!(suggested[1].why, "1 work items with the same labels landed there; 1 work items with like titles landed there."); |
| 406 | assert!(suggested[0].score > suggested[1].score); |
| 407 | } |
| 408 | |
| 409 | #[test] |
| 410 | fn a_target_already_linked_is_not_suggested() { |
| 411 | let candidates = vec![("rep_web".to_owned(), "acme/web".to_owned())]; |
| 412 | let target = ItemRepo { repo_id: "rep_web".into(), path: "acme/web".into(), role: LinkRole::Target, legacy_number: None, landed: false }; |
| 413 | let item = item("Fix web", "", &[], vec![target]); |
| 414 | assert!(suggest_targets(&item, &candidates, &[], &[]).is_empty()); |
| 415 | } |
| 416 | } |