| 1 | //! Coordinating work between agents and people |
| 2 | //! (`g1t_contracts::work::coordination`): sessions of their own harnesses |
| 3 | //! on a repository (sessions.rs), advisory claims on work items |
| 4 | //! (claims.rs), what a work item is blocked by (dependencies.rs), pull |
| 5 | //! requests that start changing the same files (overlaps.rs), and the one |
| 6 | //! call that gathers it all for a work item (context.rs). |
| 7 | //! |
| 8 | //! Nothing here is a lock and nothing waits on a harness: sessions and |
| 9 | //! claims that went quiet are found when the repository's coordination is |
| 10 | //! next touched (`sweep`), and announced then as ended or expired. |
| 11 | |
| 12 | pub(crate) mod updates; |
| 13 | mod claims; |
| 14 | mod context; |
| 15 | mod dependencies; |
| 16 | mod links; |
| 17 | mod overlaps; |
| 18 | mod sessions; |
| 19 | pub(crate) mod shown; |
| 20 | pub(crate) mod titles; |
| 21 | mod workspace; |
| 22 | #[cfg(test)] |
| 23 | mod tests; |
| 24 | |
| 25 | use g1t_contracts::credentials::Principal; |
| 26 | use g1t_contracts::repos::{Repo, RepoPath}; |
| 27 | use g1t_contracts::time::{parse_rfc3339, rfc3339}; |
| 28 | use g1t_contracts::work::*; |
| 29 | use g1t_contracts::{FailureCode, Outcome, User, Viewer}; |
| 30 | use g1t_kit::now_ms; |
| 31 | use serde::Deserialize; |
| 32 | use worker::Result; |
| 33 | |
| 34 | use crate::Work; |
| 35 | |
| 36 | /// Unwraps an `Outcome`, returning its failure from the enclosing method. |
| 37 | macro_rules! unwrap_or_fail { |
| 38 | ($outcome:expr) => { |
| 39 | match $outcome { |
| 40 | g1t_contracts::Outcome::Ok(value) => value, |
| 41 | g1t_contracts::Outcome::Fail(failure) => return Ok(g1t_contracts::Outcome::Fail(failure)), |
| 42 | } |
| 43 | }; |
| 44 | } |
| 45 | pub(crate) use unwrap_or_fail; |
| 46 | |
| 47 | /// A session's columns, with its work item's number beside its id. |
| 48 | const SESSION_COLUMNS: &str = concat!( |
| 49 | "id, repo_id, item_id, |
| 50 | ", |
| 51 | crate::items::sql::item_number!("sessions.item_id"), |
| 52 | " AS item, |
| 53 | pull, branch, actor_id, actor_name, harness, harness_session_id, |
| 54 | title, state, head_commit, started_at, last_seen_at, ended_at, end_reason, |
| 55 | parent_id, agent_type, task, prompt, result, model, title_source, title_settled_at, |
| 56 | usage, cost_usd" |
| 57 | ); |
| 58 | |
| 59 | /// A claim's columns, with its work item's workspace and number. |
| 60 | const CLAIM_COLUMNS: &str = concat!( |
| 61 | "id, item_id, repo_id, |
| 62 | ", |
| 63 | crate::items::sql::item_workspace!("claims.item_id"), |
| 64 | " AS workspace, |
| 65 | ", |
| 66 | crate::items::sql::item_number!("claims.item_id"), |
| 67 | " AS item, |
| 68 | session_id, actor_id, actor_name, harness, note, state, |
| 69 | ttl_minutes, created_at, expires_at, ended_at, end_reason, handoff_to" |
| 70 | ); |
| 71 | |
| 72 | #[derive(Clone, Deserialize)] |
| 73 | pub(crate) struct SessionRow { |
| 74 | id: String, |
| 75 | repo_id: String, |
| 76 | #[serde(default)] |
| 77 | item_id: Option<String>, |
| 78 | #[serde(default)] |
| 79 | item: Option<u32>, |
| 80 | pull: Option<u32>, |
| 81 | branch: Option<String>, |
| 82 | actor_id: String, |
| 83 | actor_name: String, |
| 84 | harness: String, |
| 85 | harness_session_id: Option<String>, |
| 86 | title: Option<String>, |
| 87 | state: String, |
| 88 | head_commit: Option<String>, |
| 89 | started_at: String, |
| 90 | last_seen_at: String, |
| 91 | ended_at: Option<String>, |
| 92 | end_reason: Option<String>, |
| 93 | #[serde(default)] |
| 94 | parent_id: Option<String>, |
| 95 | #[serde(default)] |
| 96 | agent_type: Option<String>, |
| 97 | #[serde(default)] |
| 98 | task: Option<String>, |
| 99 | #[serde(default)] |
| 100 | prompt: Option<String>, |
| 101 | #[serde(default)] |
| 102 | result: Option<String>, |
| 103 | #[serde(default)] |
| 104 | model: Option<String>, |
| 105 | #[serde(default)] |
| 106 | title_source: Option<String>, |
| 107 | #[serde(default)] |
| 108 | title_settled_at: Option<String>, |
| 109 | /// Its token usage as last reported (JSON). |
| 110 | #[serde(default)] |
| 111 | usage: Option<String>, |
| 112 | #[serde(default)] |
| 113 | cost_usd: Option<f64>, |
| 114 | } |
| 115 | |
| 116 | /// Whether some of a session's usage, as kept, was for models priced as unknown. |
| 117 | fn cost_partial(usage: Option<&str>) -> bool { |
| 118 | usage |
| 119 | .and_then(|usage| serde_json::from_str::<Vec<g1t_contracts::model_prices::ModelUsage>>(usage).ok()) |
| 120 | .is_some_and(|usage| g1t_contracts::model_prices::estimate(&usage).partial) |
| 121 | } |
| 122 | |
| 123 | impl SessionRow { |
| 124 | /// The session as it reads at `now`. |
| 125 | pub(crate) fn at(self, now: u64) -> Session { |
| 126 | let last_seen = parse_rfc3339(&self.last_seen_at).unwrap_or(now); |
| 127 | Session { |
| 128 | state: SessionState::at(self.state == "ended", last_seen, now), |
| 129 | id: self.id, |
| 130 | repo_id: self.repo_id, |
| 131 | item: self.item, |
| 132 | pull: self.pull, |
| 133 | branch: self.branch, |
| 134 | actor: Principal { id: self.actor_id, username: self.actor_name }, |
| 135 | harness: Harness::parse(&self.harness).unwrap_or(Harness::Other), |
| 136 | harness_session_id: self.harness_session_id, |
| 137 | title: self.title, |
| 138 | head_commit: self.head_commit, |
| 139 | started_at: self.started_at, |
| 140 | last_seen_at: self.last_seen_at, |
| 141 | ended_at: self.ended_at, |
| 142 | end_reason: self.end_reason, |
| 143 | parent_id: self.parent_id, |
| 144 | agent_type: self.agent_type, |
| 145 | task: self.task, |
| 146 | prompt: self.prompt, |
| 147 | result: self.result, |
| 148 | model: self.model, |
| 149 | subagents: 0, |
| 150 | first_prompt: None, |
| 151 | title_source: self.title_source.as_deref().and_then(TitleSource::parse), |
| 152 | title_settled_at: self.title_settled_at, |
| 153 | cost_partial: cost_partial(self.usage.as_deref()), |
| 154 | cost_usd: self.cost_usd, |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | #[derive(Deserialize)] |
| 160 | pub(crate) struct ClaimRow { |
| 161 | id: String, |
| 162 | pub(crate) item_id: String, |
| 163 | #[serde(default)] |
| 164 | repo_id: Option<String>, |
| 165 | #[serde(default)] |
| 166 | workspace: Option<String>, |
| 167 | #[serde(default)] |
| 168 | item: Option<u32>, |
| 169 | session_id: Option<String>, |
| 170 | actor_id: String, |
| 171 | actor_name: String, |
| 172 | harness: Option<String>, |
| 173 | note: Option<String>, |
| 174 | state: String, |
| 175 | #[allow(dead_code)] |
| 176 | ttl_minutes: u32, |
| 177 | created_at: String, |
| 178 | expires_at: String, |
| 179 | ended_at: Option<String>, |
| 180 | end_reason: Option<String>, |
| 181 | handoff_to: Option<String>, |
| 182 | } |
| 183 | |
| 184 | impl From<ClaimRow> for Claim { |
| 185 | fn from(row: ClaimRow) -> Self { |
| 186 | Claim { |
| 187 | id: row.id, |
| 188 | repo_id: row.repo_id, |
| 189 | workspace: row.workspace.unwrap_or_default(), |
| 190 | item: row.item.unwrap_or_default(), |
| 191 | session_id: row.session_id, |
| 192 | actor: Principal { id: row.actor_id, username: row.actor_name }, |
| 193 | harness: row.harness.as_deref().and_then(Harness::parse), |
| 194 | note: row.note, |
| 195 | state: ClaimState::parse(&row.state).unwrap_or(ClaimState::Expired), |
| 196 | created_at: row.created_at, |
| 197 | expires_at: row.expires_at, |
| 198 | ended_at: row.ended_at, |
| 199 | end_reason: row.end_reason, |
| 200 | handoff_to: row.handoff_to, |
| 201 | } |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | /// Text as given, trimmed and cut to `limit` characters; none when empty. |
| 206 | pub(crate) fn tidy(text: Option<&str>, limit: usize) -> Option<String> { |
| 207 | let text = text?.trim(); |
| 208 | (!text.is_empty()).then(|| text.chars().take(limit).collect()) |
| 209 | } |
| 210 | |
| 211 | /// The most sessions, and claims, one sweep ends; the rest wait for the next. |
| 212 | const SWEEP_LIMIT: u32 = 50; |
| 213 | |
| 214 | /// Ends a repository's (`?1`) session `?3` if last seen at or before `?2`, |
| 215 | /// as `timed_out`, as of when it was last seen; its title is settled a |
| 216 | /// last time (titles.rs). |
| 217 | pub(crate) fn time_out_sessions_sql() -> String { |
| 218 | format!( |
| 219 | "UPDATE sessions SET state = 'ended', ended_at = last_seen_at, end_reason = 'timed_out', {final_due} |
| 220 | WHERE repo_id = ?1 AND state = 'active' AND last_seen_at <= ?2 AND id = ?3 |
| 221 | RETURNING {SESSION_COLUMNS}", |
| 222 | final_due = titles::final_due("last_seen_at") |
| 223 | ) |
| 224 | } |
| 225 | |
| 226 | /// Expires active claim `?3` if due at or before `?2` (`?1` unused, so |
| 227 | /// the binds are the sessions'). |
| 228 | pub(crate) fn expire_claims_sql() -> String { |
| 229 | format!( |
| 230 | "UPDATE claims SET state = 'expired', ended_at = expires_at, end_reason = 'expired' |
| 231 | WHERE ?1 IS NOT NULL AND state = 'active' AND expires_at <= ?2 AND id = ?3 |
| 232 | RETURNING {CLAIM_COLUMNS}" |
| 233 | ) |
| 234 | } |
| 235 | |
| 236 | /// RFC 3339 for `minutes` after `now`. |
| 237 | pub(crate) fn after_minutes(now: u64, minutes: u32) -> String { |
| 238 | rfc3339(now + u64::from(minutes) * 60_000) |
| 239 | } |
| 240 | |
| 241 | impl Work { |
| 242 | /// The repository as `viewer` sees it, for reading its coordination. |
| 243 | pub(crate) async fn coordination_repo(&self, path: &RepoPath, viewer: &Viewer) -> Result<Outcome<Repo>> { |
| 244 | let repo = unwrap_or_fail!(self.repo(path, viewer).await?); |
| 245 | self.sweep(&repo.id).await?; |
| 246 | Ok(Outcome::Ok(repo)) |
| 247 | } |
| 248 | |
| 249 | /// The repository, if `actor` may take part in it: a confirmed email |
| 250 | /// address, a repository that is not archived, and a role that may |
| 251 | /// open work items and pull requests. |
| 252 | pub(crate) async fn participating_repo(&self, path: &RepoPath, actor: &User) -> Result<Outcome<Repo>> { |
| 253 | if !actor.verified { |
| 254 | return Ok(Outcome::fail(FailureCode::Forbidden, crate::UNVERIFIED)); |
| 255 | } |
| 256 | let repo = unwrap_or_fail!(self.repo(path, &Some(actor.clone())).await?); |
| 257 | unwrap_or_fail!(crate::retired::writable(&repo)); |
| 258 | unwrap_or_fail!(crate::allowed(Some(actor), &repo, g1t_contracts::access::Capability::Participate)); |
| 259 | self.sweep(&repo.id).await?; |
| 260 | Ok(Outcome::Ok(repo)) |
| 261 | } |
| 262 | |
| 263 | /// Ends the repository's sessions that have gone 30 minutes without a |
| 264 | /// heartbeat (`timed_out`) and expires the claims made from it or on |
| 265 | /// work items linked to it past their time, and announces each. Run |
| 266 | /// before the repository's coordination is read or changed, so that |
| 267 | /// nobody sees a stale one as live. |
| 268 | pub(crate) async fn sweep(&self, repo_id: &str) -> Result<()> { |
| 269 | let now = now_ms(); |
| 270 | let quiet_since = rfc3339(now.saturating_sub(SESSION_TIMEOUT_MS)); |
| 271 | let at = rfc3339(now); |
| 272 | // Read first: most of the time nothing is due, and nothing is written. |
| 273 | let due = self |
| 274 | .db |
| 275 | .batch(vec![ |
| 276 | self.db |
| 277 | .prepare(format!( |
| 278 | "SELECT {SESSION_COLUMNS} FROM sessions |
| 279 | WHERE repo_id = ?1 AND state = 'active' AND last_seen_at <= ?2 LIMIT ?3" |
| 280 | )) |
| 281 | .bind(&[repo_id.into(), quiet_since.as_str().into(), SWEEP_LIMIT.into()])?, |
| 282 | self.db |
| 283 | .prepare(format!( |
| 284 | "SELECT {CLAIM_COLUMNS} FROM claims |
| 285 | WHERE state = 'active' AND expires_at <= ?2 AND (repo_id = ?1 OR item_id IN {linked}) |
| 286 | LIMIT ?3", |
| 287 | linked = crate::items::sql::items_linked_to!("?1"), |
| 288 | )) |
| 289 | .bind(&[repo_id.into(), at.as_str().into(), SWEEP_LIMIT.into()])?, |
| 290 | ]) |
| 291 | .await?; |
| 292 | let mut due = due.into_iter(); |
| 293 | let quiet: Vec<SessionRow> = match due.next() { |
| 294 | Some(result) => result.results()?, |
| 295 | None => Vec::new(), |
| 296 | }; |
| 297 | let lapsed: Vec<ClaimRow> = match due.next() { |
| 298 | Some(result) => result.results()?, |
| 299 | None => Vec::new(), |
| 300 | }; |
| 301 | if quiet.is_empty() && lapsed.is_empty() { |
| 302 | return Ok(()); |
| 303 | } |
| 304 | // Each one's end and its updates are one change: a sweep running at |
| 305 | // the same moment ends it, and delivers it, once. |
| 306 | let mut statements = Vec::new(); |
| 307 | let mut sessions = Vec::new(); |
| 308 | for row in quiet { |
| 309 | statements.push( |
| 310 | self.db |
| 311 | .prepare(time_out_sessions_sql()) |
| 312 | .bind(&[repo_id.into(), quiet_since.as_str().into(), row.id.as_str().into()])?, |
| 313 | ); |
| 314 | let index = statements.len() - 1; |
| 315 | let ended_at = row.last_seen_at.clone(); |
| 316 | let expected = SessionRow { |
| 317 | state: "ended".to_owned(), |
| 318 | ended_at: Some(ended_at.clone()), |
| 319 | end_reason: Some("timed_out".to_owned()), |
| 320 | ..row |
| 321 | } |
| 322 | .at(now); |
| 323 | let delivery = self.delivery( |
| 324 | "session.ended", |
| 325 | repo_id, |
| 326 | serde_json::to_value(SessionEvent::of(&expected))?, |
| 327 | &format!("{}:ended:{ended_at}", expected.id), |
| 328 | "EXISTS (SELECT 1 FROM sessions WHERE id = ?7 AND state = 'ended' AND ended_at = ?8)", |
| 329 | vec![expected.id.as_str().into(), ended_at.as_str().into()], |
| 330 | )?; |
| 331 | sessions.push((index, updates::push_delivery(&mut statements, delivery))); |
| 332 | } |
| 333 | let mut claims = Vec::new(); |
| 334 | for row in lapsed { |
| 335 | statements.push( |
| 336 | self.db.prepare(expire_claims_sql()).bind(&[repo_id.into(), at.as_str().into(), row.id.as_str().into()])?, |
| 337 | ); |
| 338 | let index = statements.len() - 1; |
| 339 | let held = Claim::from(row); |
| 340 | let expected = Claim { |
| 341 | state: ClaimState::Expired, |
| 342 | ended_at: Some(held.expires_at.clone()), |
| 343 | end_reason: Some("expired".to_owned()), |
| 344 | ..held |
| 345 | }; |
| 346 | let delivery = self.delivery( |
| 347 | "work.claim_expired", |
| 348 | repo_id, |
| 349 | serde_json::to_value(ClaimEvent::of(&expected, &[repo_id.to_owned()]))?, |
| 350 | &format!("{}:{}", expected.id, ClaimState::Expired.as_str()), |
| 351 | "EXISTS (SELECT 1 FROM claims WHERE id = ?7 AND state = 'expired')", |
| 352 | vec![expected.id.as_str().into()], |
| 353 | )?; |
| 354 | claims.push((index, updates::push_delivery(&mut statements, delivery))); |
| 355 | } |
| 356 | let results = self.db.batch(statements).await?; |
| 357 | let returned = |index: usize| results.get(index); |
| 358 | for (index, sent) in sessions { |
| 359 | let Some(row) = returned(index).map(|result| result.results::<SessionRow>()).transpose()?.and_then(|rows| rows.into_iter().next()) |
| 360 | else { |
| 361 | continue; |
| 362 | }; |
| 363 | let session = row.at(now); |
| 364 | self.delivered(sent, "session.ended", repo_id, None, serde_json::to_value(SessionEvent::of(&session))?).await?; |
| 365 | } |
| 366 | for (index, sent) in claims { |
| 367 | let Some(row) = returned(index).map(|result| result.results::<ClaimRow>()).transpose()?.and_then(|rows| rows.into_iter().next()) |
| 368 | else { |
| 369 | continue; |
| 370 | }; |
| 371 | let claim = Claim::from(row); |
| 372 | let data = serde_json::to_value(ClaimEvent::of(&claim, &[repo_id.to_owned()]))?; |
| 373 | self.delivered(sent, "work.claim_expired", repo_id, None, data).await?; |
| 374 | } |
| 375 | Ok(()) |
| 376 | } |
| 377 | |
| 378 | pub(crate) async fn session_row(&self, repo_id: &str, id: &str) -> Result<Option<SessionRow>> { |
| 379 | self.db |
| 380 | .prepare(format!("SELECT {SESSION_COLUMNS} FROM sessions WHERE repo_id = ? AND id = ?")) |
| 381 | .bind(&[repo_id.into(), id.into()])? |
| 382 | .first::<SessionRow>(None) |
| 383 | .await |
| 384 | } |
| 385 | |
| 386 | /// Active claims on a work item, oldest first. |
| 387 | pub(crate) async fn active_claims(&self, item_id: &str) -> Result<Vec<Claim>> { |
| 388 | Ok(self |
| 389 | .db |
| 390 | .prepare(format!( |
| 391 | "SELECT {CLAIM_COLUMNS} FROM claims |
| 392 | WHERE item_id = ? AND state = 'active' ORDER BY created_at, id" |
| 393 | )) |
| 394 | .bind(&[item_id.into()])? |
| 395 | .all() |
| 396 | .await? |
| 397 | .results::<ClaimRow>()? |
| 398 | .into_iter() |
| 399 | .map(Claim::from) |
| 400 | .collect()) |
| 401 | } |
| 402 | } |