Sign in

flagon-io/g1t

public

g1t/services/work/src/confidence.rs

919 lines36,908 bytesCodeBlame
1//! How sure g1t is of a change an agent made.
2//!
3//! Worked out from what g1t can observe, never from how the agent sounds:
4//! whether the checks the default branch requires pass on it (and whether
5//! the branch requires any), whether it failed in the merge queue, how
6//! many times the agent was sent back, the reviewer agent's verdict and how
7//! much it had to say, whether tests were added or changed, how large the
8//! change is and whether it reached outside the files its plan expected,
9//! whether it touched paths that run or configure things (CI, secrets,
10//! infrastructure), how close its runs came to their guardrails, and what
11//! it asked other agents without an answer.
12//!
13//! The agent can say how sure it is too, at the end of its run
14//! (`report_confidence`). That is combined with the signals by taking the
15//! lower of the two: what g1t observes can lower what the agent says, never
16//! raise it.
17//!
18//! [`score`] is pure and tested on its own; [`Work::assess_confidence`]
19//! gathers the signals for a pull request, and records the result on it
20//! and on the run that left it so. Where a repository asks for it
21//! (`hold_low_confidence`), a low-confidence change waits for a person
22//! instead of merging by itself (lifecycle.rs).
23
24use futures_util::future::try_join;
25use g1t_contracts::time::rfc3339;
26use g1t_contracts::work::{
27 ChangedFile, Confidence, ConfidenceLevel, Pull, ReportConfidenceArgs, RequiredCheck, RequiredState, Verdict,
28};
29use g1t_contracts::{FailureCode, Outcome};
30use g1t_kit::now_ms;
31use serde::Deserialize;
32use worker::Result;
33use worker::wasm_bindgen::JsValue;
34
35use crate::Work;
36use crate::checks::hash;
37use crate::reviews::AGENT_ID;
38use crate::rows::NumberRow;
39
40/// At this many points, low; at none, high; medium between.
41const LOW_AT: u32 = 3;
42/// The most reasons a confidence gives.
43const MAX_REASONS: usize = 4;
44/// The most a self-report's list of doubts keeps, and of each.
45const MAX_UNCERTAIN: usize = 5;
46const MAX_UNCERTAIN_CHARS: usize = 160;
47/// A share of a run's cap past which it was close to it.
48const NEAR_CAP: f64 = 0.8;
49
50/// Where the checks the default branch requires stand on a change's head,
51/// taken together.
52#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
53pub(crate) enum RequiredSignal {
54 /// The branch requires no checks: nothing has to pass.
55 #[default]
56 NoneRequired,
57 Passing,
58 Failing,
59 Running,
60 /// Required, and nothing has reported them on the head.
61 NotRun,
62}
63
64impl RequiredSignal {
65 pub(crate) fn of(required: &[RequiredCheck]) -> RequiredSignal {
66 let any = |state: RequiredState| required.iter().any(|check| check.state == state);
67 if required.is_empty() {
68 RequiredSignal::NoneRequired
69 } else if any(RequiredState::Failure) {
70 RequiredSignal::Failing
71 } else if any(RequiredState::Pending) {
72 RequiredSignal::Running
73 } else if any(RequiredState::Expected) {
74 RequiredSignal::NotRun
75 } else {
76 RequiredSignal::Passing
77 }
78 }
79}
80
81/// Everything confidence is worked out from.
82#[derive(Clone, Debug, Default)]
83pub(crate) struct Signals {
84 /// Where the required checks stand on its head.
85 pub required: RequiredSignal,
86 /// The merge queue took it out: it failed together with what was ahead.
87 pub queue_failed: bool,
88 /// A recorded run errored, or failed and then passed on the same
89 /// commit: they pass, but not reliably.
90 pub flaky_checks: bool,
91 /// How many times the agent was sent back.
92 pub revisions: u32,
93 /// Whether a second agent reviews changes here.
94 pub agent_review: bool,
95 /// The verdict of the latest review of the change as it is now.
96 pub review: Option<Verdict>,
97 /// How many comments on lines that review left.
98 pub review_comments: u32,
99 pub files: Vec<ChangedFile>,
100 /// The files the issue's plan expected it to change; empty when it was
101 /// not planned.
102 pub expected: Vec<String>,
103 /// `budget` or `time` when a run was stopped at a cap.
104 pub halted: Option<String>,
105 /// The latest run's cost as a share of its cost cap.
106 pub budget_share: Option<f64>,
107 /// The latest run's time as a share of its time cap.
108 pub time_share: Option<f64>,
109 /// Commands and tools the guardrails refused while it worked.
110 pub denials: u32,
111 /// Questions and handoffs it sent other agents that have no answer.
112 pub unanswered: u32,
113 /// What the agent said of its own change.
114 pub self_reported: Option<ConfidenceLevel>,
115 pub uncertain_about: Vec<String>,
116}
117
118/// Test files, by the names test runners look for.
119pub(crate) fn is_test(path: &str) -> bool {
120 let lower = path.to_ascii_lowercase();
121 let file = lower.rsplit('/').next().unwrap_or(&lower);
122 lower.split('/').any(|dir| matches!(dir, "test" | "tests" | "__tests__" | "spec" | "specs" | "testdata"))
123 || [".test.", ".spec.", "_test.", "-test.", "_spec."].iter().any(|mark| file.contains(mark))
124 || file.starts_with("test_")
125}
126
127/// Files that change nothing that runs: prose and pictures.
128fn is_prose(path: &str) -> bool {
129 let lower = path.to_ascii_lowercase();
130 [".md", ".mdx", ".txt", ".rst", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"]
131 .iter()
132 .any(|extension| lower.ends_with(extension))
133 || lower.starts_with("docs/")
134 || lower.rsplit('/').next().is_some_and(|file| file == "license" || file == "changelog")
135}
136
137/// Paths that run, configure or guard things rather than being the code
138/// itself: CI, repository automation, secrets, infrastructure, ownership.
139/// A change that reaches them deserves a person's eyes.
140pub(crate) fn sensitive(path: &str) -> Option<&'static str> {
141 let lower = path.to_ascii_lowercase();
142 let file = lower.rsplit('/').next().unwrap_or(&lower);
143 if lower.starts_with(".github/workflows/") || lower.starts_with(".gitlab-ci") || lower.starts_with(".circleci/") {
144 return Some("CI workflows");
145 }
146 if lower.starts_with(".g1t/") || lower.starts_with(".github/") {
147 return Some("repository automation");
148 }
149 if file == "codeowners" {
150 return Some("CODEOWNERS");
151 }
152 if file.starts_with(".env") || file.ends_with(".pem") || file.ends_with(".key") || file.contains("secret") {
153 return Some("secrets");
154 }
155 if file.ends_with(".tf")
156 || file.ends_with(".tfvars")
157 || file == "dockerfile"
158 || file.starts_with("docker-compose")
159 || file.starts_with("wrangler.")
160 {
161 return Some("infrastructure");
162 }
163 None
164}
165
166/// Whether `path` is where the plan said the work would be: one of its
167/// files, or beside one, in the same directory or below it.
168fn expected(path: &str, planned: &[String]) -> bool {
169 planned.iter().any(|file| {
170 let file = file.trim().trim_start_matches("./");
171 if path == file {
172 return true;
173 }
174 let dir = file.rsplit_once('/').map_or("", |(dir, _)| dir);
175 // A plan that names a directory, or a file at the root, covers less.
176 let dir = if file.ends_with('/') { file.trim_end_matches('/') } else { dir };
177 !dir.is_empty() && path.starts_with(&format!("{dir}/"))
178 })
179}
180
181fn plural(n: u32, one: &str, many: &str) -> String {
182 format!("{n} {}", if n == 1 { one } else { many })
183}
184
185/// What lowered confidence: how much, and in a few words. `None` points
186/// makes it low on its own.
187struct Mark {
188 points: Option<u32>,
189 reason: String,
190}
191
192fn sink(reason: impl Into<String>) -> Mark {
193 Mark { points: None, reason: reason.into() }
194}
195
196fn points(points: u32, reason: impl Into<String>) -> Mark {
197 Mark { points: Some(points), reason: reason.into() }
198}
199
200/// How sure g1t is of a change, from `signals`. Each signal that tells
201/// against it adds points, or makes it low outright; no points is high,
202/// one or two medium, three or more low. The agent's own word, when it
203/// gave one, can only make it lower.
204pub(crate) fn score(signals: &Signals) -> (ConfidenceLevel, Vec<String>) {
205 let mut marks: Vec<Mark> = Vec::new();
206
207 if signals.queue_failed {
208 marks.push(sink("failed in the merge queue"));
209 }
210 match signals.required {
211 RequiredSignal::Failing => marks.push(sink("required checks failing")),
212 RequiredSignal::Running => marks.push(points(1, "required checks not finished")),
213 RequiredSignal::NotRun => marks.push(points(1, "required checks not run")),
214 RequiredSignal::NoneRequired => marks.push(points(1, "branch has no required checks")),
215 RequiredSignal::Passing => {}
216 }
217 if signals.flaky_checks && signals.required == RequiredSignal::Passing {
218 marks.push(points(1, "checks passed only on a retry"));
219 }
220
221 match signals.revisions {
222 0 => {}
223 n @ (1 | 2) => marks.push(points(n, plural(n, "revision", "revisions"))),
224 n => marks.push(points(3, plural(n, "revision", "revisions"))),
225 }
226
227 match signals.review {
228 Some(Verdict::RequestChanges) => marks.push(sink("reviewer asked for changes")),
229 Some(Verdict::Approve) if signals.review_comments >= 3 => {
230 marks.push(points(1, format!("reviewer left {} comments", signals.review_comments)));
231 }
232 Some(Verdict::Approve) => {}
233 None if signals.agent_review => marks.push(points(1, "not reviewed yet")),
234 None => marks.push(points(1, "no review")),
235 }
236
237 let code: Vec<&ChangedFile> = signals
238 .files
239 .iter()
240 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
241 .collect();
242 let tests = signals.files.iter().filter(|file| is_test(&file.path)).count();
243 if !code.is_empty() && tests == 0 {
244 marks.push(points(1, "tests not added"));
245 }
246
247 let lines: u32 = signals.files.iter().map(|file| file.additions + file.deletions).sum();
248 if lines > 1000 {
249 marks.push(points(2, format!("large change ({lines} lines)")));
250 } else if lines > 400 {
251 marks.push(points(1, format!("{lines} lines changed")));
252 }
253 let files = signals.files.len() as u32;
254 if files > 30 {
255 marks.push(points(1, format!("{files} files changed")));
256 }
257 if !signals.expected.is_empty() {
258 let outside = signals
259 .files
260 .iter()
261 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
262 .filter(|file| !expected(&file.path, &signals.expected))
263 .count() as u32;
264 if outside > 0 {
265 marks.push(points(
266 if outside >= 4 { 2 } else { 1 },
267 format!("{} outside the planned area", plural(outside, "file", "files")),
268 ));
269 }
270 }
271 let mut touched: Vec<&str> = signals.files.iter().filter_map(|file| sensitive(&file.path)).collect();
272 touched.dedup();
273 if let Some(first) = touched.first() {
274 marks.push(points(2, format!("touches {first}")));
275 }
276
277 match signals.halted.as_deref() {
278 Some("budget") => marks.push(sink("stopped at its cost cap")),
279 Some("time") => marks.push(sink("stopped at its time cap")),
280 _ => {
281 if signals.budget_share.is_some_and(|share| share >= NEAR_CAP) {
282 let share = (signals.budget_share.unwrap_or_default() * 100.0).round() as u32;
283 marks.push(points(1, format!("used {}% of its cost cap", share.min(100))));
284 }
285 if signals.time_share.is_some_and(|share| share >= NEAR_CAP) {
286 let share = (signals.time_share.unwrap_or_default() * 100.0).round() as u32;
287 marks.push(points(1, format!("used {}% of its time cap", share.min(100))));
288 }
289 }
290 }
291 if signals.denials > 0 {
292 marks.push(points(
293 if signals.denials >= 3 { 2 } else { 1 },
294 format!("{} by guardrails", plural(signals.denials, "step refused", "steps refused")),
295 ));
296 }
297 if signals.unanswered > 0 {
298 marks.push(points(
299 2,
300 plural(signals.unanswered, "question unanswered", "questions unanswered"),
301 ));
302 }
303 if !signals.uncertain_about.is_empty() {
304 marks.push(points(1, format!("agent unsure about {}", signals.uncertain_about[0])));
305 }
306
307 let sunk = marks.iter().any(|mark| mark.points.is_none());
308 let total: u32 = marks.iter().filter_map(|mark| mark.points).sum();
309 let observed = if sunk || total >= LOW_AT {
310 ConfidenceLevel::Low
311 } else if total > 0 {
312 ConfidenceLevel::Medium
313 } else {
314 ConfidenceLevel::High
315 };
316 let level = signals.self_reported.map_or(observed, |said| said.min(observed));
317
318 // Most telling first: what makes it low on its own, then by weight.
319 marks.sort_by_key(|mark| std::cmp::Reverse(mark.points.unwrap_or(u32::MAX)));
320 let mut reasons: Vec<String> = Vec::new();
321 if let Some(said) = signals.self_reported.filter(|said| *said < observed) {
322 reasons.push(format!("agent says {}", said.as_str()));
323 }
324 reasons.extend(marks.into_iter().map(|mark| mark.reason));
325 if reasons.is_empty() {
326 // High: what it rests on.
327 if signals.required == RequiredSignal::Passing {
328 reasons.push("required checks pass".to_owned());
329 }
330 if signals.review == Some(Verdict::Approve) {
331 reasons.push(if signals.revisions == 0 { "approved on first review" } else { "review approved" }.to_owned());
332 }
333 if tests > 0 {
334 reasons.push("tests added".to_owned());
335 }
336 if lines > 0 && lines <= 100 {
337 reasons.push("small change".to_owned());
338 }
339 }
340 reasons.truncate(MAX_REASONS);
341 (level, reasons)
342}
343
344/// A self-report's doubts, tidied: short, distinct, a few.
345fn tidy(uncertain: &[String]) -> Vec<String> {
346 let mut out: Vec<String> = Vec::new();
347 for item in uncertain {
348 let line = item.split_whitespace().collect::<Vec<_>>().join(" ");
349 let line: String = line.chars().take(MAX_UNCERTAIN_CHARS).collect();
350 if !line.is_empty() && !out.contains(&line) {
351 out.push(line);
352 }
353 if out.len() == MAX_UNCERTAIN {
354 break;
355 }
356 }
357 out
358}
359
360#[derive(Deserialize)]
361struct CheckRow {
362 head_commit: String,
363 status: String,
364}
365
366#[derive(Deserialize)]
367struct LatestRun {
368 id: String,
369 cost_usd: Option<f64>,
370 budget_usd: Option<f64>,
371 time_cap_minutes: Option<u32>,
372 minutes: Option<f64>,
373 self_level: Option<String>,
374 uncertain_about: Option<String>,
375}
376
377#[derive(Deserialize)]
378struct Halted {
379 halted: Option<String>,
380}
381
382#[derive(Deserialize)]
383struct PlannedFiles {
384 files: Option<String>,
385}
386
387#[derive(Deserialize)]
388struct RunTicket {
389 repo_id: String,
390 pull_id: Option<String>,
391 token_hash: String,
392}
393
394/// Whether a list of check runs, oldest first, shows checks that pass but
395/// not reliably: one errored, or failed and later passed on the same commit.
396pub(crate) fn flaky(runs: &[(String, String)]) -> bool {
397 runs.iter().any(|(_, status)| status == "errored")
398 || runs.iter().enumerate().any(|(index, (commit, status))| {
399 status == "failed" && runs[index + 1..].iter().any(|(later, status)| later == commit && status == "passed")
400 })
401}
402
403impl Work {
404 /// Whether the repository asks a person before merging a low-confidence
405 /// change. On unless someone turned it off.
406 pub(crate) async fn holds_low_confidence(&self, repo_id: &str) -> Result<bool> {
407 Ok(self
408 .db
409 .prepare("SELECT hold_low AS n FROM confidence_rules WHERE repo_id = ?")
410 .bind(&[repo_id.into()])?
411 .first::<NumberRow>(None)
412 .await?
413 .is_none_or(|row| row.n != 0))
414 }
415
416 /// Records whether the repository holds low-confidence changes.
417 pub(crate) async fn set_hold_low_confidence(&self, repo_id: &str, hold: bool, by: &str, at: &str) -> Result<()> {
418 self.db
419 .prepare(
420 "INSERT INTO confidence_rules (repo_id, hold_low, updated_by, updated_at) VALUES (?, ?, ?, ?)
421 ON CONFLICT (repo_id) DO UPDATE SET
422 hold_low = excluded.hold_low, updated_by = excluded.updated_by, updated_at = excluded.updated_at",
423 )
424 .bind(&[repo_id.into(), u32::from(hold).into(), by.into(), at.into()])?
425 .run()
426 .await?;
427 Ok(())
428 }
429
430 /// The signals for a pull request a g1t agent has finished, besides the
431 /// ones its lifecycle already knows (`known`).
432 async fn signals(&self, pull: &Pull, known: Signals) -> Result<(Signals, Option<String>)> {
433 let checks = async {
434 let rows = self
435 .db
436 .prepare("SELECT head_commit, status FROM check_runs WHERE pull_id = ? ORDER BY id LIMIT 50")
437 .bind(&[pull.id.as_str().into()])?
438 .all()
439 .await?
440 .results::<CheckRow>()?;
441 Ok::<_, worker::Error>(rows.into_iter().map(|row| (row.head_commit, row.status)).collect::<Vec<_>>())
442 };
443 let run = async {
444 // The latest run that worked on the change, and what its agent
445 // said of it.
446 let latest = self
447 .db
448 .prepare(
449 "SELECT r.id, r.cost_usd, r.budget_usd, r.time_cap_minutes,
450 (julianday(COALESCE(r.finished_at, r.updated_at)) - julianday(COALESCE(r.started_at, r.created_at))) * 1440 AS minutes,
451 c.self_level, c.uncertain_about
452 FROM agent_runs r LEFT JOIN run_confidence c ON c.run_id = r.id
453 WHERE r.pull_id = ? AND r.kind IN ('implement', 'revise')
454 ORDER BY r.created_at DESC LIMIT 1",
455 )
456 .bind(&[pull.id.as_str().into()])?
457 .first::<LatestRun>(None)
458 .await?;
459 // Any run on it stopped at a cap.
460 let halted = self
461 .db
462 .prepare(
463 "SELECT halted FROM agent_runs WHERE pull_id = ? AND halted IS NOT NULL
464 ORDER BY created_at DESC LIMIT 1",
465 )
466 .bind(&[pull.id.as_str().into()])?
467 .first::<Halted>(None)
468 .await?
469 .and_then(|row| row.halted);
470 Ok::<_, worker::Error>((latest, halted))
471 };
472 let counts = async {
473 let denials = self
474 .db
475 .prepare(
476 "SELECT count(*) AS n FROM session_entries
477 WHERE pull_id = ? AND kind = 'note' AND text LIKE 'Denied:%'",
478 )
479 .bind(&[pull.id.as_str().into()])?
480 .first::<NumberRow>(None)
481 .await?
482 .map_or(0, |row| row.n);
483 let unanswered = self
484 .db
485 .prepare(
486 "SELECT count(*) AS n FROM agent_messages
487 WHERE repo_id = ? AND from_number = ? AND kind IN ('question', 'handoff')
488 AND answered_at IS NULL",
489 )
490 .bind(&[pull.repo_id.as_str().into(), pull.number.into()])?
491 .first::<NumberRow>(None)
492 .await?
493 .map_or(0, |row| row.n);
494 let planned = match pull.issue {
495 Some(number) => self
496 .db
497 .prepare(
498 "SELECT json_extract(planned.value, '$.files') AS files
499 FROM plans, json_each(plans.issues) AS planned
500 WHERE plans.repo_id = ? AND plans.status = 'applied'
501 AND json_extract(planned.value, '$.number') = ?
502 LIMIT 1",
503 )
504 .bind(&[pull.repo_id.as_str().into(), number.into()])?
505 .first::<PlannedFiles>(None)
506 .await?
507 .and_then(|row| row.files)
508 .and_then(|files| serde_json::from_str::<Vec<String>>(&files).ok())
509 .unwrap_or_default(),
510 None => Vec::new(),
511 };
512 Ok::<_, worker::Error>((denials, unanswered, planned))
513 };
514 let (runs, ((latest, halted), (denials, unanswered, expected))) =
515 try_join(checks, try_join(run, counts)).await?;
516 let run_id = latest.as_ref().map(|run| run.id.clone());
517 let share = |used: Option<f64>, cap: Option<f64>| match (used, cap) {
518 (Some(used), Some(cap)) if cap > 0.0 => Some(used / cap),
519 _ => None,
520 };
521 let signals = Signals {
522 flaky_checks: flaky(&runs),
523 files: pull.files.clone(),
524 expected,
525 halted,
526 budget_share: latest.as_ref().and_then(|run| share(run.cost_usd, run.budget_usd)),
527 time_share: latest
528 .as_ref()
529 .and_then(|run| share(run.minutes, run.time_cap_minutes.map(f64::from))),
530 denials,
531 unanswered,
532 self_reported: latest
533 .as_ref()
534 .and_then(|run| run.self_level.as_deref())
535 .and_then(ConfidenceLevel::parse),
536 uncertain_about: latest
537 .as_ref()
538 .and_then(|run| run.uncertain_about.as_deref())
539 .and_then(|items| serde_json::from_str::<Vec<String>>(items).ok())
540 .unwrap_or_default(),
541 ..known
542 };
543 Ok((signals, run_id))
544 }
545
546 /// Works out how sure g1t is of a pull request a g1t agent has finished
547 /// (`known` holds what its lifecycle already read), and records it on
548 /// the pull request and on the run that left it so when it changed.
549 pub(crate) async fn assess_confidence(&self, pull: &Pull, known: Signals) -> Result<Confidence> {
550 let (signals, run_id) = self.signals(pull, known).await?;
551 let (level, reasons) = score(&signals);
552 let unchanged = pull.confidence.as_ref().filter(|was| {
553 was.level == level
554 && was.reasons == reasons
555 && was.self_reported == signals.self_reported
556 && was.uncertain_about == signals.uncertain_about
557 && was.run_id == run_id
558 });
559 if let Some(was) = unchanged {
560 return Ok(was.clone());
561 }
562 let now = rfc3339(now_ms());
563 let confidence = Confidence {
564 level,
565 reasons,
566 self_reported: signals.self_reported,
567 uncertain_about: signals.uncertain_about,
568 run_id: run_id.clone(),
569 assessed_at: now.clone(),
570 };
571 let detail = serde_json::to_string(&confidence)?;
572 self.db
573 .prepare(
574 "INSERT INTO pull_confidence (pull_id, repo_id, level, detail, updated_at) VALUES (?, ?, ?, ?, ?)
575 ON CONFLICT (pull_id) DO UPDATE SET
576 level = excluded.level, detail = excluded.detail, updated_at = excluded.updated_at",
577 )
578 .bind(&[
579 pull.id.as_str().into(),
580 pull.repo_id.as_str().into(),
581 level.as_str().into(),
582 detail.as_str().into(),
583 now.as_str().into(),
584 ])?
585 .run()
586 .await?;
587 if let Some(run_id) = &run_id {
588 self.db
589 .prepare(
590 "INSERT INTO run_confidence (run_id, pull_id, repo_id, detail, updated_at) VALUES (?, ?, ?, ?, ?)
591 ON CONFLICT (run_id) DO UPDATE SET detail = excluded.detail, updated_at = excluded.updated_at",
592 )
593 .bind(&[
594 run_id.as_str().into(),
595 pull.id.as_str().into(),
596 pull.repo_id.as_str().into(),
597 detail.as_str().into(),
598 now.as_str().into(),
599 ])?
600 .run()
601 .await?;
602 }
603 Ok(confidence)
604 }
605
606 /// What the agent of a run said of its own change, with the run's token.
607 pub(crate) async fn report_confidence(&self, a: ReportConfidenceArgs) -> Result<Outcome<bool>> {
608 let run = self
609 .db
610 .prepare("SELECT repo_id, pull_id, token_hash FROM agent_runs WHERE id = ?")
611 .bind(&[a.run_id.as_str().into()])?
612 .first::<RunTicket>(None)
613 .await?
614 .filter(|run| !a.token.is_empty() && run.token_hash == hash(&a.token));
615 let Some(run) = run else {
616 return Ok(Outcome::fail(FailureCode::NotFound, "Run not found."));
617 };
618 let Some(level) = ConfidenceLevel::parse(&a.confidence) else {
619 return Ok(Outcome::fail(FailureCode::Invalid, "confidence is high, medium or low."));
620 };
621 let uncertain = serde_json::to_string(&tidy(&a.uncertain_about))?;
622 self.db
623 .prepare(
624 "INSERT INTO run_confidence (run_id, pull_id, repo_id, self_level, uncertain_about, updated_at)
625 VALUES (?, ?, ?, ?, ?, ?)
626 ON CONFLICT (run_id) DO UPDATE SET
627 self_level = excluded.self_level, uncertain_about = excluded.uncertain_about,
628 updated_at = excluded.updated_at",
629 )
630 .bind(&[
631 a.run_id.as_str().into(),
632 run.pull_id.as_deref().map_or(JsValue::NULL, JsValue::from),
633 run.repo_id.as_str().into(),
634 level.as_str().into(),
635 uncertain.into(),
636 rfc3339(now_ms()).into(),
637 ])?
638 .run()
639 .await?;
640 Ok(Outcome::Ok(true))
641 }
642
643 /// How many comments on lines a review by g1t's agent left, which it
644 /// records at the moment it finished.
645 pub(crate) async fn review_comments(&self, pull: &Pull, finished_at: &str) -> Result<u32> {
646 Ok(self
647 .db
648 .prepare(
649 "SELECT count(*) AS n FROM comments
650 WHERE repo_id = ? AND number = ? AND author_id = ? AND created_at = ? AND path IS NOT NULL",
651 )
652 .bind(&[
653 pull.repo_id.as_str().into(),
654 pull.number.into(),
655 AGENT_ID.into(),
656 finished_at.into(),
657 ])?
658 .first::<NumberRow>(None)
659 .await?
660 .map_or(0, |row| row.n))
661 }
662
663 /// Whether a person other than the author approved the change since the
664 /// agent last revised it: someone has looked, so a hold is lifted.
665 pub(crate) async fn person_approved(&self, pull: &Pull, revised_at: Option<&str>) -> Result<bool> {
666 #[derive(Deserialize)]
667 struct Latest {
668 verdict: String,
669 created_at: String,
670 }
671 let rows = self
672 .db
673 .prepare(
674 "SELECT verdict, created_at FROM comments
675 WHERE repo_id = ? AND number = ? AND verdict IS NOT NULL
676 AND author_id != ? AND author_id != ? AND author_id != ?
677 ORDER BY id DESC LIMIT 20",
678 )
679 .bind(&[
680 pull.repo_id.as_str().into(),
681 pull.number.into(),
682 pull.author.id.as_str().into(),
683 AGENT_ID.into(),
684 crate::lifecycle::POLICY_ACTOR_ID.into(),
685 ])?
686 .all()
687 .await?
688 .results::<Latest>()?;
689 Ok(rows
690 .first()
691 .is_some_and(|latest| latest.verdict == "approve" && revised_at.is_none_or(|revised| latest.created_at.as_str() > revised)))
692 }
693}
694
695#[cfg(test)]
696mod tests {
697 use super::*;
698
699 fn file(path: &str, lines: u32) -> ChangedFile {
700 ChangedFile { path: path.to_owned(), additions: lines, deletions: 0 }
701 }
702
703 /// A clean change: required checks pass, approved on the first
704 /// review, a test beside the code, small.
705 fn clean() -> Signals {
706 Signals {
707 required: RequiredSignal::Passing,
708 agent_review: true,
709 review: Some(Verdict::Approve),
710 files: vec![file("src/retry.ts", 40), file("src/retry.test.ts", 30)],
711 ..Signals::default()
712 }
713 }
714
715 #[test]
716 fn signals_score_as_the_table_says() {
717 use ConfidenceLevel::*;
718 let cases: Vec<(&str, Signals, ConfidenceLevel, &[&str])> = vec![
719 ("clean", clean(), High, &["required checks pass", "approved on first review", "tests added", "small change"]),
720 (
721 "failing required checks",
722 Signals { required: RequiredSignal::Failing, ..clean() },
723 Low,
724 &["required checks failing"],
725 ),
726 (
727 "required checks not run",
728 Signals { required: RequiredSignal::NotRun, ..clean() },
729 Medium,
730 &["required checks not run"],
731 ),
732 (
733 "required checks still running",
734 Signals { required: RequiredSignal::Running, ..clean() },
735 Medium,
736 &["required checks not finished"],
737 ),
738 (
739 "failed in the merge queue",
740 Signals { queue_failed: true, ..clean() },
741 Low,
742 &["failed in the merge queue"],
743 ),
744 ("one revision", Signals { revisions: 1, ..clean() }, Medium, &["1 revision"]),
745 ("three revisions", Signals { revisions: 3, ..clean() }, Low, &["3 revisions"]),
746 (
747 "no tests and three revisions",
748 Signals { revisions: 3, files: vec![file("src/retry.ts", 40)], ..clean() },
749 Low,
750 &["3 revisions", "tests not added"],
751 ),
752 (
753 "no tests",
754 Signals { files: vec![file("src/retry.ts", 40)], ..clean() },
755 Medium,
756 &["tests not added"],
757 ),
758 (
759 "docs need no tests",
760 Signals { files: vec![file("README.md", 40), file("docs/guide.md", 10)], ..clean() },
761 High,
762 &["required checks pass", "approved on first review", "small change"],
763 ),
764 (
765 "the reviewer asks for changes",
766 Signals { review: Some(Verdict::RequestChanges), ..clean() },
767 Low,
768 &["reviewer asked for changes"],
769 ),
770 (
771 "a review with much to say",
772 Signals { review_comments: 4, ..clean() },
773 Medium,
774 &["reviewer left 4 comments"],
775 ),
776 ("no review yet", Signals { review: None, ..clean() }, Medium, &["not reviewed yet"]),
777 (
778 "no reviewer agent and no required checks",
779 Signals { review: None, agent_review: false, required: RequiredSignal::NoneRequired, ..clean() },
780 Medium,
781 &["branch has no required checks", "no review"],
782 ),
783 (
784 "a large change",
785 Signals { files: vec![file("src/a.ts", 900), file("src/a.test.ts", 300)], ..clean() },
786 Medium,
787 &["large change (1200 lines)"],
788 ),
789 (
790 "outside the planned area",
791 Signals {
792 expected: vec!["src/retry.ts".to_owned()],
793 files: vec![file("src/retry.ts", 10), file("lib/billing/charge.ts", 10), file("src/retry.test.ts", 5)],
794 ..clean()
795 },
796 Medium,
797 &["1 file outside the planned area"],
798 ),
799 (
800 "beside the planned files is inside",
801 Signals {
802 expected: vec!["src/webhooks/retry.ts".to_owned()],
803 files: vec![file("src/webhooks/backoff.ts", 10), file("src/webhooks/retry.test.ts", 5)],
804 ..clean()
805 },
806 High,
807 &["required checks pass", "approved on first review", "tests added", "small change"],
808 ),
809 (
810 "a CI workflow",
811 Signals { files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
812 Medium,
813 &["touches CI workflows"],
814 ),
815 (
816 "a CI workflow and a revision",
817 Signals { revisions: 1, files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
818 Low,
819 &["touches CI workflows", "1 revision"],
820 ),
821 ("stopped at a cap", Signals { halted: Some("budget".to_owned()), ..clean() }, Low, &["stopped at its cost cap"]),
822 (
823 "near its cost cap",
824 Signals { budget_share: Some(0.92), ..clean() },
825 Medium,
826 &["used 92% of its cost cap"],
827 ),
828 (
829 "flaky checks",
830 Signals { flaky_checks: true, ..clean() },
831 Medium,
832 &["checks passed only on a retry"],
833 ),
834 (
835 "unanswered questions",
836 Signals { unanswered: 2, ..clean() },
837 Medium,
838 &["2 questions unanswered"],
839 ),
840 (
841 "guardrails refused a lot",
842 Signals { denials: 3, revisions: 1, ..clean() },
843 Low,
844 &["3 steps refused by guardrails", "1 revision"],
845 ),
846 (
847 "the agent says low",
848 Signals { self_reported: Some(Low), ..clean() },
849 Low,
850 &["agent says low"],
851 ),
852 (
853 "the agent cannot raise it",
854 Signals { self_reported: Some(High), revisions: 1, ..clean() },
855 Medium,
856 &["1 revision"],
857 ),
858 (
859 "the agent's doubts count",
860 Signals { self_reported: Some(High), uncertain_about: vec!["the retry limit".to_owned()], ..clean() },
861 Medium,
862 &["agent unsure about the retry limit"],
863 ),
864 ];
865 for (name, signals, level, reasons) in cases {
866 let (got, why) = score(&signals);
867 assert_eq!(got, level, "{name}: {why:?}");
868 assert_eq!(why, reasons.iter().map(|r| (*r).to_owned()).collect::<Vec<_>>(), "{name}");
869 }
870 }
871
872 #[test]
873 fn reasons_are_few_and_the_worst_come_first() {
874 let signals = Signals {
875 required: RequiredSignal::Failing,
876 revisions: 2,
877 files: vec![file("src/a.ts", 600), file(".env.example", 1)],
878 unanswered: 1,
879 ..clean()
880 };
881 let (level, reasons) = score(&signals);
882 assert_eq!(level, ConfidenceLevel::Low);
883 assert_eq!(reasons.len(), MAX_REASONS);
884 assert_eq!(reasons[0], "required checks failing");
885 }
886
887 #[test]
888 fn checks_that_pass_on_a_retry_are_flaky() {
889 let runs = |list: &[(&str, &str)]| list.iter().map(|(c, s)| ((*c).to_owned(), (*s).to_owned())).collect::<Vec<_>>();
890 assert!(!flaky(&runs(&[("a", "failed"), ("b", "passed")])));
891 assert!(flaky(&runs(&[("a", "failed"), ("a", "passed")])));
892 assert!(flaky(&runs(&[("a", "errored"), ("b", "passed")])));
893 assert!(!flaky(&runs(&[("a", "passed")])));
894 }
895
896 #[test]
897 fn tests_prose_and_sensitive_paths_are_told_apart() {
898 for path in ["src/__tests__/a.ts", "tests/test_api.py", "pkg/api_test.go", "src/a.spec.tsx", "crates/x/tests/it.rs"] {
899 assert!(is_test(path), "{path}");
900 }
901 for path in ["src/testing.ts", "src/contest.rs", "attest/a.rs"] {
902 assert!(!is_test(path), "{path}");
903 }
904 assert!(is_prose("docs/setup.md") && is_prose("README.md") && !is_prose("src/readme.ts"));
905 assert_eq!(sensitive(".github/workflows/ci.yml"), Some("CI workflows"));
906 assert_eq!(sensitive("apps/web/wrangler.jsonc"), Some("infrastructure"));
907 assert_eq!(sensitive("config/.env.production"), Some("secrets"));
908 assert_eq!(sensitive("src/env.ts"), None);
909 }
910
911 #[test]
912 fn doubts_are_tidied() {
913 let items = vec![" the retry limit ".to_owned(), "the retry limit".to_owned(), String::new(), "x".repeat(400)];
914 let tidied = tidy(&items);
915 assert_eq!(tidied.len(), 2);
916 assert_eq!(tidied[0], "the retry limit");
917 assert_eq!(tidied[1].chars().count(), MAX_UNCERTAIN_CHARS);
918 }
919}