Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/work/src/confidence.rs

991 lines39,885 bytesCodeBlame
1//! How sure g1t is of a change an agent made.
2//!
3//! Worked out from what g1t can observe, never from how the agent sounds:
4//! whether the checks the default branch requires pass on it (and whether
5//! the branch requires any), whether it failed in the merge queue, how
6//! many times the agent was sent back, the reviewer agent's verdict and how
7//! much it had to say, whether tests were added or changed, how large the
8//! change is and whether it reached outside the files its plan expected,
9//! whether it touched paths that run or configure things (CI, secrets,
10//! infrastructure), how close its runs came to their guardrails, and what
11//! it asked other agents without an answer.
12//!
13//! The agent can say how sure it is too, at the end of its run
14//! (`report_confidence`). That is combined with the signals by taking the
15//! lower of the two: what g1t observes can lower what the agent says, never
16//! raise it.
17//!
18//! [`score`] is pure and tested on its own; [`Work::assess_confidence`]
19//! gathers the signals for a pull request, and records the result on it
20//! and on the run that left it so. Where a repository asks for it
21//! (`hold_low_confidence`), a low-confidence change waits for a person
22//! instead of merging by itself (lifecycle.rs).
23
24use futures_util::future::try_join;
25use g1t_contracts::time::rfc3339;
26use g1t_contracts::work::{
27 ChangedFile, Confidence, ConfidenceLevel, Pull, ReportConfidenceArgs, RequiredCheck, RequiredState, Verdict,
28};
29use g1t_contracts::{FailureCode, Outcome};
30use g1t_kit::now_ms;
31use serde::Deserialize;
32use worker::Result;
33use worker::wasm_bindgen::JsValue;
34
35use crate::Work;
36use crate::checks::hash;
37use crate::reviews::AGENT_ID;
38use crate::prefetch::Slot;
39use crate::rows::NumberRow;
40
41/// At this many points, low; at none, high; medium between.
42const LOW_AT: u32 = 3;
43/// The most reasons a confidence gives.
44const MAX_REASONS: usize = 4;
45/// The most a self-report's list of doubts keeps, and of each.
46const MAX_UNCERTAIN: usize = 5;
47const MAX_UNCERTAIN_CHARS: usize = 160;
48/// A share of a run's cap past which it was close to it.
49const NEAR_CAP: f64 = 0.8;
50
51/// Where the checks the default branch requires stand on a change's head,
52/// taken together.
53#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
54pub(crate) enum RequiredSignal {
55 /// The branch requires no checks: nothing has to pass.
56 #[default]
57 NoneRequired,
58 Passing,
59 Failing,
60 Running,
61 /// Required, and nothing has reported them on the head.
62 NotRun,
63}
64
65impl RequiredSignal {
66 pub(crate) fn of(required: &[RequiredCheck]) -> RequiredSignal {
67 let any = |state: RequiredState| required.iter().any(|check| check.state == state);
68 if required.is_empty() {
69 RequiredSignal::NoneRequired
70 } else if any(RequiredState::Failure) {
71 RequiredSignal::Failing
72 } else if any(RequiredState::Pending) {
73 RequiredSignal::Running
74 } else if any(RequiredState::Expected) {
75 RequiredSignal::NotRun
76 } else {
77 RequiredSignal::Passing
78 }
79 }
80}
81
82/// Everything confidence is worked out from.
83#[derive(Clone, Debug, Default)]
84pub(crate) struct Signals {
85 /// Where the required checks stand on its head.
86 pub required: RequiredSignal,
87 /// The merge queue took it out: it failed together with what was ahead.
88 pub queue_failed: bool,
89 /// A recorded run errored, or failed and then passed on the same
90 /// commit: they pass, but not reliably.
91 pub flaky_checks: bool,
92 /// How many times the agent was sent back.
93 pub revisions: u32,
94 /// Whether a second agent reviews changes here.
95 pub agent_review: bool,
96 /// The verdict of the latest review of the change as it is now.
97 pub review: Option<Verdict>,
98 /// How many comments on lines that review left.
99 pub review_comments: u32,
100 pub files: Vec<ChangedFile>,
101 /// The files the issue's plan expected it to change; empty when it was
102 /// not planned.
103 pub expected: Vec<String>,
104 /// `budget` or `time` when a run was stopped at a cap.
105 pub halted: Option<String>,
106 /// The latest run's cost as a share of its cost cap.
107 pub budget_share: Option<f64>,
108 /// The latest run's time as a share of its time cap.
109 pub time_share: Option<f64>,
110 /// Commands and tools the guardrails refused while it worked.
111 pub denials: u32,
112 /// Questions and handoffs it sent other agents that have no answer.
113 pub unanswered: u32,
114 /// What the agent said of its own change.
115 pub self_reported: Option<ConfidenceLevel>,
116 pub uncertain_about: Vec<String>,
117}
118
119/// Test files, by the names test runners look for.
120pub(crate) fn is_test(path: &str) -> bool {
121 let lower = path.to_ascii_lowercase();
122 let file = lower.rsplit('/').next().unwrap_or(&lower);
123 lower.split('/').any(|dir| matches!(dir, "test" | "tests" | "__tests__" | "spec" | "specs" | "testdata"))
124 || [".test.", ".spec.", "_test.", "-test.", "_spec."].iter().any(|mark| file.contains(mark))
125 || file.starts_with("test_")
126}
127
128/// Files that change nothing that runs: prose and pictures.
129fn is_prose(path: &str) -> bool {
130 let lower = path.to_ascii_lowercase();
131 [".md", ".mdx", ".txt", ".rst", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"]
132 .iter()
133 .any(|extension| lower.ends_with(extension))
134 || lower.starts_with("docs/")
135 || lower.rsplit('/').next().is_some_and(|file| file == "license" || file == "changelog")
136}
137
138/// Paths that run, configure or guard things rather than being the code
139/// itself: CI, repository automation, secrets, infrastructure, ownership.
140/// A change that reaches them deserves a person's eyes.
141pub(crate) fn sensitive(path: &str) -> Option<&'static str> {
142 let lower = path.to_ascii_lowercase();
143 let file = lower.rsplit('/').next().unwrap_or(&lower);
144 if lower.starts_with(".github/workflows/") || lower.starts_with(".gitlab-ci") || lower.starts_with(".circleci/") {
145 return Some("CI workflows");
146 }
147 if lower.starts_with(".g1t/") || lower.starts_with(".github/") {
148 return Some("repository automation");
149 }
150 if file == "codeowners" {
151 return Some("CODEOWNERS");
152 }
153 if file.starts_with(".env") || file.ends_with(".pem") || file.ends_with(".key") || file.contains("secret") {
154 return Some("secrets");
155 }
156 if file.ends_with(".tf")
157 || file.ends_with(".tfvars")
158 || file == "dockerfile"
159 || file.starts_with("docker-compose")
160 || file.starts_with("wrangler.")
161 {
162 return Some("infrastructure");
163 }
164 None
165}
166
167/// Whether `path` is where the plan said the work would be: one of its
168/// files, or beside one, in the same directory or below it.
169fn expected(path: &str, planned: &[String]) -> bool {
170 planned.iter().any(|file| {
171 let file = file.trim().trim_start_matches("./");
172 if path == file {
173 return true;
174 }
175 let dir = file.rsplit_once('/').map_or("", |(dir, _)| dir);
176 // A plan that names a directory, or a file at the root, covers less.
177 let dir = if file.ends_with('/') { file.trim_end_matches('/') } else { dir };
178 !dir.is_empty() && path.starts_with(&format!("{dir}/"))
179 })
180}
181
182fn plural(n: u32, one: &str, many: &str) -> String {
183 format!("{n} {}", if n == 1 { one } else { many })
184}
185
186/// What lowered confidence: how much, and in a few words. `None` points
187/// makes it low on its own.
188struct Mark {
189 points: Option<u32>,
190 reason: String,
191}
192
193fn sink(reason: impl Into<String>) -> Mark {
194 Mark { points: None, reason: reason.into() }
195}
196
197fn points(points: u32, reason: impl Into<String>) -> Mark {
198 Mark { points: Some(points), reason: reason.into() }
199}
200
201/// How sure g1t is of a change, from `signals`. Each signal that tells
202/// against it adds points, or makes it low outright; no points is high,
203/// one or two medium, three or more low. The agent's own word, when it
204/// gave one, can only make it lower.
205pub(crate) fn score(signals: &Signals) -> (ConfidenceLevel, Vec<String>) {
206 let mut marks: Vec<Mark> = Vec::new();
207
208 if signals.queue_failed {
209 marks.push(sink("failed in the merge queue"));
210 }
211 match signals.required {
212 RequiredSignal::Failing => marks.push(sink("required checks failing")),
213 RequiredSignal::Running => marks.push(points(1, "required checks not finished")),
214 RequiredSignal::NotRun => marks.push(points(1, "required checks not run")),
215 RequiredSignal::NoneRequired => marks.push(points(1, "branch has no required checks")),
216 RequiredSignal::Passing => {}
217 }
218 if signals.flaky_checks && signals.required == RequiredSignal::Passing {
219 marks.push(points(1, "checks passed only on a retry"));
220 }
221
222 match signals.revisions {
223 0 => {}
224 n @ (1 | 2) => marks.push(points(n, plural(n, "revision", "revisions"))),
225 n => marks.push(points(3, plural(n, "revision", "revisions"))),
226 }
227
228 match signals.review {
229 Some(Verdict::RequestChanges) => marks.push(sink("reviewer asked for changes")),
230 Some(Verdict::Approve) if signals.review_comments >= 3 => {
231 marks.push(points(1, format!("reviewer left {} comments", signals.review_comments)));
232 }
233 Some(Verdict::Approve) => {}
234 None if signals.agent_review => marks.push(points(1, "not reviewed yet")),
235 None => marks.push(points(1, "no review")),
236 }
237
238 let code: Vec<&ChangedFile> = signals
239 .files
240 .iter()
241 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
242 .collect();
243 let tests = signals.files.iter().filter(|file| is_test(&file.path)).count();
244 if !code.is_empty() && tests == 0 {
245 marks.push(points(1, "tests not added"));
246 }
247
248 let lines: u32 = signals.files.iter().map(|file| file.additions + file.deletions).sum();
249 if lines > 1000 {
250 marks.push(points(2, format!("large change ({lines} lines)")));
251 } else if lines > 400 {
252 marks.push(points(1, format!("{lines} lines changed")));
253 }
254 let files = signals.files.len() as u32;
255 if files > 30 {
256 marks.push(points(1, format!("{files} files changed")));
257 }
258 if !signals.expected.is_empty() {
259 let outside = signals
260 .files
261 .iter()
262 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
263 .filter(|file| !expected(&file.path, &signals.expected))
264 .count() as u32;
265 if outside > 0 {
266 marks.push(points(
267 if outside >= 4 { 2 } else { 1 },
268 format!("{} outside the planned area", plural(outside, "file", "files")),
269 ));
270 }
271 }
272 let mut touched: Vec<&str> = signals.files.iter().filter_map(|file| sensitive(&file.path)).collect();
273 touched.dedup();
274 if let Some(first) = touched.first() {
275 marks.push(points(2, format!("touches {first}")));
276 }
277
278 match signals.halted.as_deref() {
279 Some("budget") => marks.push(sink("stopped at its cost cap")),
280 Some("time") => marks.push(sink("stopped at its time cap")),
281 _ => {
282 if signals.budget_share.is_some_and(|share| share >= NEAR_CAP) {
283 let share = (signals.budget_share.unwrap_or_default() * 100.0).round() as u32;
284 marks.push(points(1, format!("used {}% of its cost cap", share.min(100))));
285 }
286 if signals.time_share.is_some_and(|share| share >= NEAR_CAP) {
287 let share = (signals.time_share.unwrap_or_default() * 100.0).round() as u32;
288 marks.push(points(1, format!("used {}% of its time cap", share.min(100))));
289 }
290 }
291 }
292 if signals.denials > 0 {
293 marks.push(points(
294 if signals.denials >= 3 { 2 } else { 1 },
295 format!("{} by guardrails", plural(signals.denials, "step refused", "steps refused")),
296 ));
297 }
298 if signals.unanswered > 0 {
299 marks.push(points(
300 2,
301 plural(signals.unanswered, "question unanswered", "questions unanswered"),
302 ));
303 }
304 if !signals.uncertain_about.is_empty() {
305 marks.push(points(1, format!("agent unsure about {}", signals.uncertain_about[0])));
306 }
307
308 let sunk = marks.iter().any(|mark| mark.points.is_none());
309 let total: u32 = marks.iter().filter_map(|mark| mark.points).sum();
310 let observed = if sunk || total >= LOW_AT {
311 ConfidenceLevel::Low
312 } else if total > 0 {
313 ConfidenceLevel::Medium
314 } else {
315 ConfidenceLevel::High
316 };
317 let level = signals.self_reported.map_or(observed, |said| said.min(observed));
318
319 // Most telling first: what makes it low on its own, then by weight.
320 marks.sort_by_key(|mark| std::cmp::Reverse(mark.points.unwrap_or(u32::MAX)));
321 let mut reasons: Vec<String> = Vec::new();
322 if let Some(said) = signals.self_reported.filter(|said| *said < observed) {
323 reasons.push(format!("agent says {}", said.as_str()));
324 }
325 reasons.extend(marks.into_iter().map(|mark| mark.reason));
326 if reasons.is_empty() {
327 // High: what it rests on.
328 if signals.required == RequiredSignal::Passing {
329 reasons.push("required checks pass".to_owned());
330 }
331 if signals.review == Some(Verdict::Approve) {
332 reasons.push(if signals.revisions == 0 { "approved on first review" } else { "review approved" }.to_owned());
333 }
334 if tests > 0 {
335 reasons.push("tests added".to_owned());
336 }
337 if lines > 0 && lines <= 100 {
338 reasons.push("small change".to_owned());
339 }
340 }
341 reasons.truncate(MAX_REASONS);
342 (level, reasons)
343}
344
345/// A self-report's doubts, tidied: short, distinct, a few.
346fn tidy(uncertain: &[String]) -> Vec<String> {
347 let mut out: Vec<String> = Vec::new();
348 for item in uncertain {
349 let line = item.split_whitespace().collect::<Vec<_>>().join(" ");
350 let line: String = line.chars().take(MAX_UNCERTAIN_CHARS).collect();
351 if !line.is_empty() && !out.contains(&line) {
352 out.push(line);
353 }
354 if out.len() == MAX_UNCERTAIN {
355 break;
356 }
357 }
358 out
359}
360
361#[derive(Deserialize)]
362struct CheckRow {
363 head_commit: String,
364 status: String,
365}
366
367#[derive(Deserialize)]
368struct LatestRun {
369 id: String,
370 cost_usd: Option<f64>,
371 budget_usd: Option<f64>,
372 time_cap_minutes: Option<u32>,
373 minutes: Option<f64>,
374 self_level: Option<String>,
375 uncertain_about: Option<String>,
376}
377
378#[derive(Deserialize)]
379struct Halted {
380 halted: Option<String>,
381}
382
383#[derive(Deserialize)]
384struct PlannedFiles {
385 files: Option<String>,
386}
387
388#[derive(Deserialize)]
389struct RunTicket {
390 repo_id: String,
391 pull_id: Option<String>,
392 token_hash: String,
393}
394
395/// Whether a list of check runs, oldest first, shows checks that pass but
396/// not reliably: one errored, or failed and later passed on the same commit.
397pub(crate) fn flaky(runs: &[(String, String)]) -> bool {
398 runs.iter().any(|(_, status)| status == "errored")
399 || runs.iter().enumerate().any(|(index, (commit, status))| {
400 status == "failed" && runs[index + 1..].iter().any(|(later, status)| later == commit && status == "passed")
401 })
402}
403
404impl Work {
405 /// Whether the repository asks a person before merging a low-confidence
406 /// change. On unless someone turned it off.
407 pub(crate) async fn holds_low_confidence(&self, repo_id: &str) -> Result<bool> {
408 Ok(self
409 .db
410 .prepare("SELECT hold_low AS n FROM confidence_rules WHERE repo_id = ?")
411 .bind(&[repo_id.into()])?
412 .first::<NumberRow>(None)
413 .await?
414 .is_none_or(|row| row.n != 0))
415 }
416
417 /// Records whether the repository holds low-confidence changes.
418 pub(crate) async fn set_hold_low_confidence(&self, repo_id: &str, hold: bool, by: &str, at: &str) -> Result<()> {
419 self.db
420 .prepare(
421 "INSERT INTO confidence_rules (repo_id, hold_low, updated_by, updated_at) VALUES (?, ?, ?, ?)
422 ON CONFLICT (repo_id) DO UPDATE SET
423 hold_low = excluded.hold_low, updated_by = excluded.updated_by, updated_at = excluded.updated_at",
424 )
425 .bind(&[repo_id.into(), u32::from(hold).into(), by.into(), at.into()])?
426 .run()
427 .await?;
428 Ok(())
429 }
430
431 /// The signals for a pull request a g1t agent has finished, besides the
432 /// ones its lifecycle already knows (`known`).
433 async fn signals(&self, pull: &Pull, known: Signals) -> Result<(Signals, Option<String>)> {
434 let (runs, ((latest, halted), (denials, unanswered, expected))) = match self.prefetched_pull(&pull.id) {
435 Some(found) => (
436 found
437 .rows::<CheckRow>(Slot::RunHistory)?
438 .into_iter()
439 .map(|row| (row.head_commit, row.status))
440 .collect::<Vec<_>>(),
441 (
442 (
443 found.first::<LatestRun>(Slot::LatestRun)?,
444 found.first::<Halted>(Slot::Halted)?.and_then(|row| row.halted),
445 ),
446 (
447 found.first::<NumberRow>(Slot::Denials)?.map_or(0, |row| row.n),
448 found.first::<NumberRow>(Slot::Unanswered)?.map_or(0, |row| row.n),
449 found
450 .first::<PlannedFiles>(Slot::Planned)?
451 .and_then(|row| row.files)
452 .and_then(|files| serde_json::from_str::<Vec<String>>(&files).ok())
453 .unwrap_or_default(),
454 ),
455 ),
456 ),
457 None => self.read_signals(pull).await?,
458 };
459 Ok(Self::signals_from(pull, known, runs, latest, halted, denials, unanswered, expected))
460 }
461
462 /// The rows [`Self::signals`] works from, read one query at a time.
463 #[allow(clippy::type_complexity)]
464 async fn read_signals(
465 &self,
466 pull: &Pull,
467 ) -> Result<(Vec<(String, String)>, ((Option<LatestRun>, Option<String>), (u32, u32, Vec<String>)))> {
468 let checks = async {
469 let rows = self
470 .db
471 .prepare("SELECT head_commit, status FROM check_runs WHERE pull_id = ? ORDER BY id LIMIT 50")
472 .bind(&[pull.id.as_str().into()])?
473 .all()
474 .await?
475 .results::<CheckRow>()?;
476 Ok::<_, worker::Error>(rows.into_iter().map(|row| (row.head_commit, row.status)).collect::<Vec<_>>())
477 };
478 let run = async {
479 // The latest run that worked on the change, and what its agent
480 // said of it.
481 let latest = self
482 .db
483 .prepare(
484 "SELECT r.id, r.cost_usd, r.budget_usd, r.time_cap_minutes,
485 (julianday(COALESCE(r.finished_at, r.updated_at)) - julianday(COALESCE(r.started_at, r.created_at))) * 1440 AS minutes,
486 c.self_level, c.uncertain_about
487 FROM agent_runs r LEFT JOIN run_confidence c ON c.run_id = r.id
488 WHERE r.pull_id = ? AND r.kind IN ('implement', 'revise')
489 ORDER BY r.created_at DESC LIMIT 1",
490 )
491 .bind(&[pull.id.as_str().into()])?
492 .first::<LatestRun>(None)
493 .await?;
494 // Any run on it stopped at a cap.
495 let halted = self
496 .db
497 .prepare(
498 "SELECT halted FROM agent_runs WHERE pull_id = ? AND halted IS NOT NULL
499 ORDER BY created_at DESC LIMIT 1",
500 )
501 .bind(&[pull.id.as_str().into()])?
502 .first::<Halted>(None)
503 .await?
504 .and_then(|row| row.halted);
505 Ok::<_, worker::Error>((latest, halted))
506 };
507 let counts = async {
508 let denials = self
509 .db
510 .prepare(
511 "SELECT count(*) AS n FROM session_entries
512 WHERE pull_id = ? AND kind = 'note' AND text LIKE 'Denied:%'",
513 )
514 .bind(&[pull.id.as_str().into()])?
515 .first::<NumberRow>(None)
516 .await?
517 .map_or(0, |row| row.n);
518 let unanswered = self
519 .db
520 .prepare(
521 "SELECT count(*) AS n FROM agent_messages
522 WHERE repo_id = ? AND from_number = ? AND kind IN ('question', 'handoff')
523 AND answered_at IS NULL",
524 )
525 .bind(&[pull.repo_id.as_str().into(), pull.number.into()])?
526 .first::<NumberRow>(None)
527 .await?
528 .map_or(0, |row| row.n);
529 let planned = match pull.issue {
530 Some(number) => self
531 .db
532 .prepare(
533 "SELECT json_extract(planned.value, '$.files') AS files
534 FROM plans, json_each(plans.issues) AS planned
535 WHERE plans.repo_id = ? AND plans.status = 'applied'
536 AND json_extract(planned.value, '$.number') = ?
537 LIMIT 1",
538 )
539 .bind(&[pull.repo_id.as_str().into(), number.into()])?
540 .first::<PlannedFiles>(None)
541 .await?
542 .and_then(|row| row.files)
543 .and_then(|files| serde_json::from_str::<Vec<String>>(&files).ok())
544 .unwrap_or_default(),
545 None => Vec::new(),
546 };
547 Ok::<_, worker::Error>((denials, unanswered, planned))
548 };
549 try_join(checks, try_join(run, counts)).await
550 }
551
552 #[allow(clippy::too_many_arguments)]
553 fn signals_from(
554 pull: &Pull,
555 known: Signals,
556 runs: Vec<(String, String)>,
557 latest: Option<LatestRun>,
558 halted: Option<String>,
559 denials: u32,
560 unanswered: u32,
561 expected: Vec<String>,
562 ) -> (Signals, Option<String>) {
563 let run_id = latest.as_ref().map(|run| run.id.clone());
564 let share = |used: Option<f64>, cap: Option<f64>| match (used, cap) {
565 (Some(used), Some(cap)) if cap > 0.0 => Some(used / cap),
566 _ => None,
567 };
568 let signals = Signals {
569 flaky_checks: flaky(&runs),
570 files: pull.files.clone(),
571 expected,
572 halted,
573 budget_share: latest.as_ref().and_then(|run| share(run.cost_usd, run.budget_usd)),
574 time_share: latest
575 .as_ref()
576 .and_then(|run| share(run.minutes, run.time_cap_minutes.map(f64::from))),
577 denials,
578 unanswered,
579 self_reported: latest
580 .as_ref()
581 .and_then(|run| run.self_level.as_deref())
582 .and_then(ConfidenceLevel::parse),
583 uncertain_about: latest
584 .as_ref()
585 .and_then(|run| run.uncertain_about.as_deref())
586 .and_then(|items| serde_json::from_str::<Vec<String>>(items).ok())
587 .unwrap_or_default(),
588 ..known
589 };
590 (signals, run_id)
591 }
592
593 /// Works out how sure g1t is of a pull request a g1t agent has finished
594 /// (`known` holds what its lifecycle already read), and records it on
595 /// the pull request and on the run that left it so when it changed.
596 pub(crate) async fn assess_confidence(&self, pull: &Pull, known: Signals) -> Result<Confidence> {
597 let (signals, run_id) = self.signals(pull, known).await?;
598 let (level, reasons) = score(&signals);
599 let unchanged = pull.confidence.as_ref().filter(|was| {
600 was.level == level
601 && was.reasons == reasons
602 && was.self_reported == signals.self_reported
603 && was.uncertain_about == signals.uncertain_about
604 && was.run_id == run_id
605 });
606 if let Some(was) = unchanged {
607 return Ok(was.clone());
608 }
609 let now = rfc3339(now_ms());
610 let confidence = Confidence {
611 level,
612 reasons,
613 self_reported: signals.self_reported,
614 uncertain_about: signals.uncertain_about,
615 run_id: run_id.clone(),
616 assessed_at: now.clone(),
617 };
618 let detail = serde_json::to_string(&confidence)?;
619 self.db
620 .prepare(
621 "INSERT INTO pull_confidence (pull_id, repo_id, level, detail, updated_at) VALUES (?, ?, ?, ?, ?)
622 ON CONFLICT (pull_id) DO UPDATE SET
623 level = excluded.level, detail = excluded.detail, updated_at = excluded.updated_at",
624 )
625 .bind(&[
626 pull.id.as_str().into(),
627 pull.repo_id.as_str().into(),
628 level.as_str().into(),
629 detail.as_str().into(),
630 now.as_str().into(),
631 ])?
632 .run()
633 .await?;
634 if let Some(run_id) = &run_id {
635 self.db
636 .prepare(
637 "INSERT INTO run_confidence (run_id, pull_id, repo_id, detail, updated_at) VALUES (?, ?, ?, ?, ?)
638 ON CONFLICT (run_id) DO UPDATE SET detail = excluded.detail, updated_at = excluded.updated_at",
639 )
640 .bind(&[
641 run_id.as_str().into(),
642 pull.id.as_str().into(),
643 pull.repo_id.as_str().into(),
644 detail.as_str().into(),
645 now.as_str().into(),
646 ])?
647 .run()
648 .await?;
649 }
650 Ok(confidence)
651 }
652
653 /// What the agent of a run said of its own change, with the run's token.
654 pub(crate) async fn report_confidence(&self, a: ReportConfidenceArgs) -> Result<Outcome<bool>> {
655 let run = self
656 .db
657 .prepare("SELECT repo_id, pull_id, token_hash FROM agent_runs WHERE id = ?")
658 .bind(&[a.run_id.as_str().into()])?
659 .first::<RunTicket>(None)
660 .await?
661 .filter(|run| !a.token.is_empty() && run.token_hash == hash(&a.token));
662 let Some(run) = run else {
663 return Ok(Outcome::fail(FailureCode::NotFound, "Run not found."));
664 };
665 let Some(level) = ConfidenceLevel::parse(&a.confidence) else {
666 return Ok(Outcome::fail(FailureCode::Invalid, "confidence is high, medium or low."));
667 };
668 let uncertain = serde_json::to_string(&tidy(&a.uncertain_about))?;
669 self.db
670 .prepare(
671 "INSERT INTO run_confidence (run_id, pull_id, repo_id, self_level, uncertain_about, updated_at)
672 VALUES (?, ?, ?, ?, ?, ?)
673 ON CONFLICT (run_id) DO UPDATE SET
674 self_level = excluded.self_level, uncertain_about = excluded.uncertain_about,
675 updated_at = excluded.updated_at",
676 )
677 .bind(&[
678 a.run_id.as_str().into(),
679 run.pull_id.as_deref().map_or(JsValue::NULL, JsValue::from),
680 run.repo_id.as_str().into(),
681 level.as_str().into(),
682 uncertain.into(),
683 rfc3339(now_ms()).into(),
684 ])?
685 .run()
686 .await?;
687 Ok(Outcome::Ok(true))
688 }
689
690 /// How many comments on lines a review by g1t's agent left, which it
691 /// records at the moment it finished.
692 pub(crate) async fn review_comments(&self, pull: &Pull, finished_at: &str) -> Result<u32> {
693 // Read for this request against the latest finished review, which
694 // is the one asked about whenever it is asked.
695 if let Some(found) = self.prefetched_pull(&pull.id) {
696 return Ok(found.first::<NumberRow>(Slot::ReviewComments)?.map_or(0, |row| row.n));
697 }
698 Ok(self
699 .db
700 .prepare(
701 "SELECT count(*) AS n FROM comments
702 WHERE repo_id = ? AND number = ? AND author_id = ? AND created_at = ? AND path IS NOT NULL",
703 )
704 .bind(&[
705 pull.repo_id.as_str().into(),
706 pull.number.into(),
707 AGENT_ID.into(),
708 finished_at.into(),
709 ])?
710 .first::<NumberRow>(None)
711 .await?
712 .map_or(0, |row| row.n))
713 }
714
715 /// Whether a person other than the author approved the change since the
716 /// agent last revised it: someone has looked, so a hold is lifted.
717 pub(crate) async fn person_approved(&self, pull: &Pull, revised_at: Option<&str>) -> Result<bool> {
718 #[derive(Deserialize)]
719 struct Latest {
720 verdict: String,
721 created_at: String,
722 }
723 #[derive(Deserialize)]
724 struct ByAuthor {
725 author_id: String,
726 verdict: String,
727 created_at: String,
728 }
729 let rows = match self.prefetched_pull(&pull.id) {
730 Some(found) => found
731 .rows::<ByAuthor>(Slot::Verdicts)?
732 .into_iter()
733 .rev()
734 .filter(|row| {
735 row.author_id != pull.author.id
736 && row.author_id != AGENT_ID
737 && row.author_id != crate::lifecycle::POLICY_ACTOR_ID
738 })
739 .take(20)
740 .map(|row| Latest { verdict: row.verdict, created_at: row.created_at })
741 .collect::<Vec<_>>(),
742 None => self
743 .db
744 .prepare(
745 "SELECT verdict, created_at FROM comments
746 WHERE repo_id = ? AND number = ? AND verdict IS NOT NULL
747 AND author_id != ? AND author_id != ? AND author_id != ?
748 ORDER BY id DESC LIMIT 20",
749 )
750 .bind(&[
751 pull.repo_id.as_str().into(),
752 pull.number.into(),
753 pull.author.id.as_str().into(),
754 AGENT_ID.into(),
755 crate::lifecycle::POLICY_ACTOR_ID.into(),
756 ])?
757 .all()
758 .await?
759 .results::<Latest>()?,
760 };
761 Ok(rows
762 .first()
763 .is_some_and(|latest| latest.verdict == "approve" && revised_at.is_none_or(|revised| latest.created_at.as_str() > revised)))
764 }
765}
766
767#[cfg(test)]
768mod tests {
769 use super::*;
770
771 fn file(path: &str, lines: u32) -> ChangedFile {
772 ChangedFile { path: path.to_owned(), additions: lines, deletions: 0 }
773 }
774
775 /// A clean change: required checks pass, approved on the first
776 /// review, a test beside the code, small.
777 fn clean() -> Signals {
778 Signals {
779 required: RequiredSignal::Passing,
780 agent_review: true,
781 review: Some(Verdict::Approve),
782 files: vec![file("src/retry.ts", 40), file("src/retry.test.ts", 30)],
783 ..Signals::default()
784 }
785 }
786
787 #[test]
788 fn signals_score_as_the_table_says() {
789 use ConfidenceLevel::*;
790 let cases: Vec<(&str, Signals, ConfidenceLevel, &[&str])> = vec![
791 ("clean", clean(), High, &["required checks pass", "approved on first review", "tests added", "small change"]),
792 (
793 "failing required checks",
794 Signals { required: RequiredSignal::Failing, ..clean() },
795 Low,
796 &["required checks failing"],
797 ),
798 (
799 "required checks not run",
800 Signals { required: RequiredSignal::NotRun, ..clean() },
801 Medium,
802 &["required checks not run"],
803 ),
804 (
805 "required checks still running",
806 Signals { required: RequiredSignal::Running, ..clean() },
807 Medium,
808 &["required checks not finished"],
809 ),
810 (
811 "failed in the merge queue",
812 Signals { queue_failed: true, ..clean() },
813 Low,
814 &["failed in the merge queue"],
815 ),
816 ("one revision", Signals { revisions: 1, ..clean() }, Medium, &["1 revision"]),
817 ("three revisions", Signals { revisions: 3, ..clean() }, Low, &["3 revisions"]),
818 (
819 "no tests and three revisions",
820 Signals { revisions: 3, files: vec![file("src/retry.ts", 40)], ..clean() },
821 Low,
822 &["3 revisions", "tests not added"],
823 ),
824 (
825 "no tests",
826 Signals { files: vec![file("src/retry.ts", 40)], ..clean() },
827 Medium,
828 &["tests not added"],
829 ),
830 (
831 "docs need no tests",
832 Signals { files: vec![file("README.md", 40), file("docs/guide.md", 10)], ..clean() },
833 High,
834 &["required checks pass", "approved on first review", "small change"],
835 ),
836 (
837 "the reviewer asks for changes",
838 Signals { review: Some(Verdict::RequestChanges), ..clean() },
839 Low,
840 &["reviewer asked for changes"],
841 ),
842 (
843 "a review with much to say",
844 Signals { review_comments: 4, ..clean() },
845 Medium,
846 &["reviewer left 4 comments"],
847 ),
848 ("no review yet", Signals { review: None, ..clean() }, Medium, &["not reviewed yet"]),
849 (
850 "no reviewer agent and no required checks",
851 Signals { review: None, agent_review: false, required: RequiredSignal::NoneRequired, ..clean() },
852 Medium,
853 &["branch has no required checks", "no review"],
854 ),
855 (
856 "a large change",
857 Signals { files: vec![file("src/a.ts", 900), file("src/a.test.ts", 300)], ..clean() },
858 Medium,
859 &["large change (1200 lines)"],
860 ),
861 (
862 "outside the planned area",
863 Signals {
864 expected: vec!["src/retry.ts".to_owned()],
865 files: vec![file("src/retry.ts", 10), file("lib/billing/charge.ts", 10), file("src/retry.test.ts", 5)],
866 ..clean()
867 },
868 Medium,
869 &["1 file outside the planned area"],
870 ),
871 (
872 "beside the planned files is inside",
873 Signals {
874 expected: vec!["src/webhooks/retry.ts".to_owned()],
875 files: vec![file("src/webhooks/backoff.ts", 10), file("src/webhooks/retry.test.ts", 5)],
876 ..clean()
877 },
878 High,
879 &["required checks pass", "approved on first review", "tests added", "small change"],
880 ),
881 (
882 "a CI workflow",
883 Signals { files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
884 Medium,
885 &["touches CI workflows"],
886 ),
887 (
888 "a CI workflow and a revision",
889 Signals { revisions: 1, files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
890 Low,
891 &["touches CI workflows", "1 revision"],
892 ),
893 ("stopped at a cap", Signals { halted: Some("budget".to_owned()), ..clean() }, Low, &["stopped at its cost cap"]),
894 (
895 "near its cost cap",
896 Signals { budget_share: Some(0.92), ..clean() },
897 Medium,
898 &["used 92% of its cost cap"],
899 ),
900 (
901 "flaky checks",
902 Signals { flaky_checks: true, ..clean() },
903 Medium,
904 &["checks passed only on a retry"],
905 ),
906 (
907 "unanswered questions",
908 Signals { unanswered: 2, ..clean() },
909 Medium,
910 &["2 questions unanswered"],
911 ),
912 (
913 "guardrails refused a lot",
914 Signals { denials: 3, revisions: 1, ..clean() },
915 Low,
916 &["3 steps refused by guardrails", "1 revision"],
917 ),
918 (
919 "the agent says low",
920 Signals { self_reported: Some(Low), ..clean() },
921 Low,
922 &["agent says low"],
923 ),
924 (
925 "the agent cannot raise it",
926 Signals { self_reported: Some(High), revisions: 1, ..clean() },
927 Medium,
928 &["1 revision"],
929 ),
930 (
931 "the agent's doubts count",
932 Signals { self_reported: Some(High), uncertain_about: vec!["the retry limit".to_owned()], ..clean() },
933 Medium,
934 &["agent unsure about the retry limit"],
935 ),
936 ];
937 for (name, signals, level, reasons) in cases {
938 let (got, why) = score(&signals);
939 assert_eq!(got, level, "{name}: {why:?}");
940 assert_eq!(why, reasons.iter().map(|r| (*r).to_owned()).collect::<Vec<_>>(), "{name}");
941 }
942 }
943
944 #[test]
945 fn reasons_are_few_and_the_worst_come_first() {
946 let signals = Signals {
947 required: RequiredSignal::Failing,
948 revisions: 2,
949 files: vec![file("src/a.ts", 600), file(".env.example", 1)],
950 unanswered: 1,
951 ..clean()
952 };
953 let (level, reasons) = score(&signals);
954 assert_eq!(level, ConfidenceLevel::Low);
955 assert_eq!(reasons.len(), MAX_REASONS);
956 assert_eq!(reasons[0], "required checks failing");
957 }
958
959 #[test]
960 fn checks_that_pass_on_a_retry_are_flaky() {
961 let runs = |list: &[(&str, &str)]| list.iter().map(|(c, s)| ((*c).to_owned(), (*s).to_owned())).collect::<Vec<_>>();
962 assert!(!flaky(&runs(&[("a", "failed"), ("b", "passed")])));
963 assert!(flaky(&runs(&[("a", "failed"), ("a", "passed")])));
964 assert!(flaky(&runs(&[("a", "errored"), ("b", "passed")])));
965 assert!(!flaky(&runs(&[("a", "passed")])));
966 }
967
968 #[test]
969 fn tests_prose_and_sensitive_paths_are_told_apart() {
970 for path in ["src/__tests__/a.ts", "tests/test_api.py", "pkg/api_test.go", "src/a.spec.tsx", "crates/x/tests/it.rs"] {
971 assert!(is_test(path), "{path}");
972 }
973 for path in ["src/testing.ts", "src/contest.rs", "attest/a.rs"] {
974 assert!(!is_test(path), "{path}");
975 }
976 assert!(is_prose("docs/setup.md") && is_prose("README.md") && !is_prose("src/readme.ts"));
977 assert_eq!(sensitive(".github/workflows/ci.yml"), Some("CI workflows"));
978 assert_eq!(sensitive("apps/web/wrangler.jsonc"), Some("infrastructure"));
979 assert_eq!(sensitive("config/.env.production"), Some("secrets"));
980 assert_eq!(sensitive("src/env.ts"), None);
981 }
982
983 #[test]
984 fn doubts_are_tidied() {
985 let items = vec![" the retry limit ".to_owned(), "the retry limit".to_owned(), String::new(), "x".repeat(400)];
986 let tidied = tidy(&items);
987 assert_eq!(tidied.len(), 2);
988 assert_eq!(tidied[0], "the retry limit");
989 assert_eq!(tidied[1].chars().count(), MAX_UNCERTAIN_CHARS);
990 }
991}