Sign in

flagon-io/g1t

public

g1t/services/work/src/confidence.rs

993 lines40,111 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step1//! How sure g1t is of a change an agent made.
2//!
3//! Worked out from what g1t can observe, never from how the agent sounds:
Fast pages, required checks on the branch, self-hosted runners, honest incidents4//! whether the checks the default branch requires pass on it (and whether
5//! the branch requires any), whether it failed in the merge queue, how
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step6//! many times the agent was sent back, the reviewer agent's verdict and how
7//! much it had to say, whether tests were added or changed, how large the
8//! change is and whether it reached outside the files its plan expected,
9//! whether it touched paths that run or configure things (CI, secrets,
10//! infrastructure), how close its runs came to their guardrails, and what
11//! it asked other agents without an answer.
12//!
13//! The agent can say how sure it is too, at the end of its run
14//! (`report_confidence`). That is combined with the signals by taking the
15//! lower of the two: what g1t observes can lower what the agent says, never
16//! raise it.
17//!
18//! [`score`] is pure and tested on its own; [`Work::assess_confidence`]
19//! gathers the signals for a pull request, and records the result on it
20//! and on the run that left it so. Where a repository asks for it
21//! (`hold_low_confidence`), a low-confidence change waits for a person
22//! instead of merging by itself (lifecycle.rs).
23
24use futures_util::future::try_join;
25use g1t_contracts::time::rfc3339;
26use g1t_contracts::work::{
Fast pages, required checks on the branch, self-hosted runners, honest incidents27 ChangedFile, Confidence, ConfidenceLevel, Pull, ReportConfidenceArgs, RequiredCheck, RequiredState, Verdict,
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step28};
29use g1t_contracts::{FailureCode, Outcome};
30use g1t_kit::now_ms;
31use serde::Deserialize;
32use worker::Result;
33use worker::wasm_bindgen::JsValue;
34
35use crate::Work;
36use crate::checks::hash;
37use crate::reviews::AGENT_ID;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily38use crate::prefetch::Slot;
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step39use crate::rows::NumberRow;
40
41/// At this many points, low; at none, high; medium between.
42const LOW_AT: u32 = 3;
43/// The most reasons a confidence gives.
44const MAX_REASONS: usize = 4;
45/// The most a self-report's list of doubts keeps, and of each.
46const MAX_UNCERTAIN: usize = 5;
47const MAX_UNCERTAIN_CHARS: usize = 160;
48/// A share of a run's cap past which it was close to it.
49const NEAR_CAP: f64 = 0.8;
50
Fast pages, required checks on the branch, self-hosted runners, honest incidents51/// Where the checks the default branch requires stand on a change's head,
52/// taken together.
53#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
54pub(crate) enum RequiredSignal {
55 /// The branch requires no checks: nothing has to pass.
56 #[default]
57 NoneRequired,
58 Passing,
59 Failing,
60 Running,
61 /// Required, and nothing has reported them on the head.
62 NotRun,
63}
64
65impl RequiredSignal {
66 pub(crate) fn of(required: &[RequiredCheck]) -> RequiredSignal {
67 let any = |state: RequiredState| required.iter().any(|check| check.state == state);
68 if required.is_empty() {
69 RequiredSignal::NoneRequired
70 } else if any(RequiredState::Failure) {
71 RequiredSignal::Failing
72 } else if any(RequiredState::Pending) {
73 RequiredSignal::Running
74 } else if any(RequiredState::Expected) {
75 RequiredSignal::NotRun
76 } else {
77 RequiredSignal::Passing
78 }
79 }
80}
81
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step82/// Everything confidence is worked out from.
83#[derive(Clone, Debug, Default)]
84pub(crate) struct Signals {
Fast pages, required checks on the branch, self-hosted runners, honest incidents85 /// Where the required checks stand on its head.
86 pub required: RequiredSignal,
87 /// The merge queue took it out: it failed together with what was ahead.
88 pub queue_failed: bool,
89 /// A recorded run errored, or failed and then passed on the same
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step90 /// commit: they pass, but not reliably.
91 pub flaky_checks: bool,
92 /// How many times the agent was sent back.
93 pub revisions: u32,
94 /// Whether a second agent reviews changes here.
95 pub agent_review: bool,
96 /// The verdict of the latest review of the change as it is now.
97 pub review: Option<Verdict>,
98 /// How many comments on lines that review left.
99 pub review_comments: u32,
100 pub files: Vec<ChangedFile>,
101 /// The files the issue's plan expected it to change; empty when it was
102 /// not planned.
103 pub expected: Vec<String>,
104 /// `budget` or `time` when a run was stopped at a cap.
105 pub halted: Option<String>,
106 /// The latest run's cost as a share of its cost cap.
107 pub budget_share: Option<f64>,
108 /// The latest run's time as a share of its time cap.
109 pub time_share: Option<f64>,
110 /// Commands and tools the guardrails refused while it worked.
111 pub denials: u32,
112 /// Questions and handoffs it sent other agents that have no answer.
113 pub unanswered: u32,
114 /// What the agent said of its own change.
115 pub self_reported: Option<ConfidenceLevel>,
116 pub uncertain_about: Vec<String>,
117}
118
119/// Test files, by the names test runners look for.
120pub(crate) fn is_test(path: &str) -> bool {
121 let lower = path.to_ascii_lowercase();
122 let file = lower.rsplit('/').next().unwrap_or(&lower);
123 lower.split('/').any(|dir| matches!(dir, "test" | "tests" | "__tests__" | "spec" | "specs" | "testdata"))
124 || [".test.", ".spec.", "_test.", "-test.", "_spec."].iter().any(|mark| file.contains(mark))
125 || file.starts_with("test_")
A file named test.js or spec.rb counts as a test: confidence no longer says tests not added when they were126 // A file named for what it is: test.js, tests.py, spec.rb.
127 || matches!(file.split('.').next(), Some("test" | "tests" | "spec" | "specs")) && file.contains('.')
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step128}
129
130/// Files that change nothing that runs: prose and pictures.
131fn is_prose(path: &str) -> bool {
132 let lower = path.to_ascii_lowercase();
133 [".md", ".mdx", ".txt", ".rst", ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"]
134 .iter()
135 .any(|extension| lower.ends_with(extension))
136 || lower.starts_with("docs/")
137 || lower.rsplit('/').next().is_some_and(|file| file == "license" || file == "changelog")
138}
139
140/// Paths that run, configure or guard things rather than being the code
141/// itself: CI, repository automation, secrets, infrastructure, ownership.
142/// A change that reaches them deserves a person's eyes.
143pub(crate) fn sensitive(path: &str) -> Option<&'static str> {
144 let lower = path.to_ascii_lowercase();
145 let file = lower.rsplit('/').next().unwrap_or(&lower);
146 if lower.starts_with(".github/workflows/") || lower.starts_with(".gitlab-ci") || lower.starts_with(".circleci/") {
147 return Some("CI workflows");
148 }
149 if lower.starts_with(".g1t/") || lower.starts_with(".github/") {
150 return Some("repository automation");
151 }
152 if file == "codeowners" {
153 return Some("CODEOWNERS");
154 }
155 if file.starts_with(".env") || file.ends_with(".pem") || file.ends_with(".key") || file.contains("secret") {
156 return Some("secrets");
157 }
158 if file.ends_with(".tf")
159 || file.ends_with(".tfvars")
160 || file == "dockerfile"
161 || file.starts_with("docker-compose")
162 || file.starts_with("wrangler.")
163 {
164 return Some("infrastructure");
165 }
166 None
167}
168
169/// Whether `path` is where the plan said the work would be: one of its
170/// files, or beside one, in the same directory or below it.
171fn expected(path: &str, planned: &[String]) -> bool {
172 planned.iter().any(|file| {
173 let file = file.trim().trim_start_matches("./");
174 if path == file {
175 return true;
176 }
177 let dir = file.rsplit_once('/').map_or("", |(dir, _)| dir);
178 // A plan that names a directory, or a file at the root, covers less.
179 let dir = if file.ends_with('/') { file.trim_end_matches('/') } else { dir };
180 !dir.is_empty() && path.starts_with(&format!("{dir}/"))
181 })
182}
183
184fn plural(n: u32, one: &str, many: &str) -> String {
185 format!("{n} {}", if n == 1 { one } else { many })
186}
187
188/// What lowered confidence: how much, and in a few words. `None` points
189/// makes it low on its own.
190struct Mark {
191 points: Option<u32>,
192 reason: String,
193}
194
195fn sink(reason: impl Into<String>) -> Mark {
196 Mark { points: None, reason: reason.into() }
197}
198
199fn points(points: u32, reason: impl Into<String>) -> Mark {
200 Mark { points: Some(points), reason: reason.into() }
201}
202
203/// How sure g1t is of a change, from `signals`. Each signal that tells
204/// against it adds points, or makes it low outright; no points is high,
205/// one or two medium, three or more low. The agent's own word, when it
206/// gave one, can only make it lower.
207pub(crate) fn score(signals: &Signals) -> (ConfidenceLevel, Vec<String>) {
208 let mut marks: Vec<Mark> = Vec::new();
209
Fast pages, required checks on the branch, self-hosted runners, honest incidents210 if signals.queue_failed {
211 marks.push(sink("failed in the merge queue"));
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step212 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents213 match signals.required {
214 RequiredSignal::Failing => marks.push(sink("required checks failing")),
215 RequiredSignal::Running => marks.push(points(1, "required checks not finished")),
216 RequiredSignal::NotRun => marks.push(points(1, "required checks not run")),
217 RequiredSignal::NoneRequired => marks.push(points(1, "branch has no required checks")),
218 RequiredSignal::Passing => {}
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step219 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents220 if signals.flaky_checks && signals.required == RequiredSignal::Passing {
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step221 marks.push(points(1, "checks passed only on a retry"));
222 }
223
224 match signals.revisions {
225 0 => {}
226 n @ (1 | 2) => marks.push(points(n, plural(n, "revision", "revisions"))),
227 n => marks.push(points(3, plural(n, "revision", "revisions"))),
228 }
229
230 match signals.review {
231 Some(Verdict::RequestChanges) => marks.push(sink("reviewer asked for changes")),
232 Some(Verdict::Approve) if signals.review_comments >= 3 => {
233 marks.push(points(1, format!("reviewer left {} comments", signals.review_comments)));
234 }
235 Some(Verdict::Approve) => {}
236 None if signals.agent_review => marks.push(points(1, "not reviewed yet")),
237 None => marks.push(points(1, "no review")),
238 }
239
240 let code: Vec<&ChangedFile> = signals
241 .files
242 .iter()
243 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
244 .collect();
245 let tests = signals.files.iter().filter(|file| is_test(&file.path)).count();
246 if !code.is_empty() && tests == 0 {
247 marks.push(points(1, "tests not added"));
248 }
249
250 let lines: u32 = signals.files.iter().map(|file| file.additions + file.deletions).sum();
251 if lines > 1000 {
252 marks.push(points(2, format!("large change ({lines} lines)")));
253 } else if lines > 400 {
254 marks.push(points(1, format!("{lines} lines changed")));
255 }
256 let files = signals.files.len() as u32;
257 if files > 30 {
258 marks.push(points(1, format!("{files} files changed")));
259 }
260 if !signals.expected.is_empty() {
261 let outside = signals
262 .files
263 .iter()
264 .filter(|file| !is_test(&file.path) && !is_prose(&file.path))
265 .filter(|file| !expected(&file.path, &signals.expected))
266 .count() as u32;
267 if outside > 0 {
268 marks.push(points(
269 if outside >= 4 { 2 } else { 1 },
270 format!("{} outside the planned area", plural(outside, "file", "files")),
271 ));
272 }
273 }
274 let mut touched: Vec<&str> = signals.files.iter().filter_map(|file| sensitive(&file.path)).collect();
275 touched.dedup();
276 if let Some(first) = touched.first() {
277 marks.push(points(2, format!("touches {first}")));
278 }
279
280 match signals.halted.as_deref() {
281 Some("budget") => marks.push(sink("stopped at its cost cap")),
282 Some("time") => marks.push(sink("stopped at its time cap")),
283 _ => {
284 if signals.budget_share.is_some_and(|share| share >= NEAR_CAP) {
285 let share = (signals.budget_share.unwrap_or_default() * 100.0).round() as u32;
286 marks.push(points(1, format!("used {}% of its cost cap", share.min(100))));
287 }
288 if signals.time_share.is_some_and(|share| share >= NEAR_CAP) {
289 let share = (signals.time_share.unwrap_or_default() * 100.0).round() as u32;
290 marks.push(points(1, format!("used {}% of its time cap", share.min(100))));
291 }
292 }
293 }
294 if signals.denials > 0 {
295 marks.push(points(
296 if signals.denials >= 3 { 2 } else { 1 },
297 format!("{} by guardrails", plural(signals.denials, "step refused", "steps refused")),
298 ));
299 }
300 if signals.unanswered > 0 {
301 marks.push(points(
302 2,
303 plural(signals.unanswered, "question unanswered", "questions unanswered"),
304 ));
305 }
306 if !signals.uncertain_about.is_empty() {
307 marks.push(points(1, format!("agent unsure about {}", signals.uncertain_about[0])));
308 }
309
310 let sunk = marks.iter().any(|mark| mark.points.is_none());
311 let total: u32 = marks.iter().filter_map(|mark| mark.points).sum();
312 let observed = if sunk || total >= LOW_AT {
313 ConfidenceLevel::Low
314 } else if total > 0 {
315 ConfidenceLevel::Medium
316 } else {
317 ConfidenceLevel::High
318 };
319 let level = signals.self_reported.map_or(observed, |said| said.min(observed));
320
321 // Most telling first: what makes it low on its own, then by weight.
322 marks.sort_by_key(|mark| std::cmp::Reverse(mark.points.unwrap_or(u32::MAX)));
323 let mut reasons: Vec<String> = Vec::new();
324 if let Some(said) = signals.self_reported.filter(|said| *said < observed) {
325 reasons.push(format!("agent says {}", said.as_str()));
326 }
327 reasons.extend(marks.into_iter().map(|mark| mark.reason));
328 if reasons.is_empty() {
329 // High: what it rests on.
Fast pages, required checks on the branch, self-hosted runners, honest incidents330 if signals.required == RequiredSignal::Passing {
331 reasons.push("required checks pass".to_owned());
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step332 }
333 if signals.review == Some(Verdict::Approve) {
334 reasons.push(if signals.revisions == 0 { "approved on first review" } else { "review approved" }.to_owned());
335 }
336 if tests > 0 {
337 reasons.push("tests added".to_owned());
338 }
339 if lines > 0 && lines <= 100 {
340 reasons.push("small change".to_owned());
341 }
342 }
343 reasons.truncate(MAX_REASONS);
344 (level, reasons)
345}
346
347/// A self-report's doubts, tidied: short, distinct, a few.
348fn tidy(uncertain: &[String]) -> Vec<String> {
349 let mut out: Vec<String> = Vec::new();
350 for item in uncertain {
351 let line = item.split_whitespace().collect::<Vec<_>>().join(" ");
352 let line: String = line.chars().take(MAX_UNCERTAIN_CHARS).collect();
353 if !line.is_empty() && !out.contains(&line) {
354 out.push(line);
355 }
356 if out.len() == MAX_UNCERTAIN {
357 break;
358 }
359 }
360 out
361}
362
363#[derive(Deserialize)]
364struct CheckRow {
365 head_commit: String,
366 status: String,
367}
368
369#[derive(Deserialize)]
370struct LatestRun {
371 id: String,
372 cost_usd: Option<f64>,
373 budget_usd: Option<f64>,
374 time_cap_minutes: Option<u32>,
375 minutes: Option<f64>,
376 self_level: Option<String>,
377 uncertain_about: Option<String>,
378}
379
380#[derive(Deserialize)]
381struct Halted {
382 halted: Option<String>,
383}
384
385#[derive(Deserialize)]
386struct PlannedFiles {
387 files: Option<String>,
388}
389
390#[derive(Deserialize)]
391struct RunTicket {
392 repo_id: String,
393 pull_id: Option<String>,
394 token_hash: String,
395}
396
397/// Whether a list of check runs, oldest first, shows checks that pass but
398/// not reliably: one errored, or failed and later passed on the same commit.
399pub(crate) fn flaky(runs: &[(String, String)]) -> bool {
400 runs.iter().any(|(_, status)| status == "errored")
401 || runs.iter().enumerate().any(|(index, (commit, status))| {
402 status == "failed" && runs[index + 1..].iter().any(|(later, status)| later == commit && status == "passed")
403 })
404}
405
406impl Work {
407 /// Whether the repository asks a person before merging a low-confidence
408 /// change. On unless someone turned it off.
409 pub(crate) async fn holds_low_confidence(&self, repo_id: &str) -> Result<bool> {
410 Ok(self
411 .db
412 .prepare("SELECT hold_low AS n FROM confidence_rules WHERE repo_id = ?")
413 .bind(&[repo_id.into()])?
414 .first::<NumberRow>(None)
415 .await?
416 .is_none_or(|row| row.n != 0))
417 }
418
419 /// Records whether the repository holds low-confidence changes.
420 pub(crate) async fn set_hold_low_confidence(&self, repo_id: &str, hold: bool, by: &str, at: &str) -> Result<()> {
421 self.db
422 .prepare(
423 "INSERT INTO confidence_rules (repo_id, hold_low, updated_by, updated_at) VALUES (?, ?, ?, ?)
424 ON CONFLICT (repo_id) DO UPDATE SET
425 hold_low = excluded.hold_low, updated_by = excluded.updated_by, updated_at = excluded.updated_at",
426 )
427 .bind(&[repo_id.into(), u32::from(hold).into(), by.into(), at.into()])?
428 .run()
429 .await?;
430 Ok(())
431 }
432
433 /// The signals for a pull request a g1t agent has finished, besides the
434 /// ones its lifecycle already knows (`known`).
435 async fn signals(&self, pull: &Pull, known: Signals) -> Result<(Signals, Option<String>)> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily436 let (runs, ((latest, halted), (denials, unanswered, expected))) = match self.prefetched_pull(&pull.id) {
437 Some(found) => (
438 found
439 .rows::<CheckRow>(Slot::RunHistory)?
440 .into_iter()
441 .map(|row| (row.head_commit, row.status))
442 .collect::<Vec<_>>(),
443 (
444 (
445 found.first::<LatestRun>(Slot::LatestRun)?,
446 found.first::<Halted>(Slot::Halted)?.and_then(|row| row.halted),
447 ),
448 (
449 found.first::<NumberRow>(Slot::Denials)?.map_or(0, |row| row.n),
450 found.first::<NumberRow>(Slot::Unanswered)?.map_or(0, |row| row.n),
451 found
452 .first::<PlannedFiles>(Slot::Planned)?
453 .and_then(|row| row.files)
454 .and_then(|files| serde_json::from_str::<Vec<String>>(&files).ok())
455 .unwrap_or_default(),
456 ),
457 ),
458 ),
459 None => self.read_signals(pull).await?,
460 };
461 Ok(Self::signals_from(pull, known, runs, latest, halted, denials, unanswered, expected))
462 }
463
464 /// The rows [`Self::signals`] works from, read one query at a time.
465 #[allow(clippy::type_complexity)]
466 async fn read_signals(
467 &self,
468 pull: &Pull,
469 ) -> Result<(Vec<(String, String)>, ((Option<LatestRun>, Option<String>), (u32, u32, Vec<String>)))> {
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step470 let checks = async {
471 let rows = self
472 .db
473 .prepare("SELECT head_commit, status FROM check_runs WHERE pull_id = ? ORDER BY id LIMIT 50")
474 .bind(&[pull.id.as_str().into()])?
475 .all()
476 .await?
477 .results::<CheckRow>()?;
478 Ok::<_, worker::Error>(rows.into_iter().map(|row| (row.head_commit, row.status)).collect::<Vec<_>>())
479 };
480 let run = async {
481 // The latest run that worked on the change, and what its agent
482 // said of it.
483 let latest = self
484 .db
485 .prepare(
486 "SELECT r.id, r.cost_usd, r.budget_usd, r.time_cap_minutes,
487 (julianday(COALESCE(r.finished_at, r.updated_at)) - julianday(COALESCE(r.started_at, r.created_at))) * 1440 AS minutes,
488 c.self_level, c.uncertain_about
489 FROM agent_runs r LEFT JOIN run_confidence c ON c.run_id = r.id
490 WHERE r.pull_id = ? AND r.kind IN ('implement', 'revise')
491 ORDER BY r.created_at DESC LIMIT 1",
492 )
493 .bind(&[pull.id.as_str().into()])?
494 .first::<LatestRun>(None)
495 .await?;
496 // Any run on it stopped at a cap.
497 let halted = self
498 .db
499 .prepare(
500 "SELECT halted FROM agent_runs WHERE pull_id = ? AND halted IS NOT NULL
501 ORDER BY created_at DESC LIMIT 1",
502 )
503 .bind(&[pull.id.as_str().into()])?
504 .first::<Halted>(None)
505 .await?
506 .and_then(|row| row.halted);
507 Ok::<_, worker::Error>((latest, halted))
508 };
509 let counts = async {
510 let denials = self
511 .db
512 .prepare(
513 "SELECT count(*) AS n FROM session_entries
514 WHERE pull_id = ? AND kind = 'note' AND text LIKE 'Denied:%'",
515 )
516 .bind(&[pull.id.as_str().into()])?
517 .first::<NumberRow>(None)
518 .await?
519 .map_or(0, |row| row.n);
520 let unanswered = self
521 .db
522 .prepare(
523 "SELECT count(*) AS n FROM agent_messages
524 WHERE repo_id = ? AND from_number = ? AND kind IN ('question', 'handoff')
525 AND answered_at IS NULL",
526 )
527 .bind(&[pull.repo_id.as_str().into(), pull.number.into()])?
528 .first::<NumberRow>(None)
529 .await?
530 .map_or(0, |row| row.n);
531 let planned = match pull.issue {
532 Some(number) => self
533 .db
534 .prepare(
535 "SELECT json_extract(planned.value, '$.files') AS files
536 FROM plans, json_each(plans.issues) AS planned
537 WHERE plans.repo_id = ? AND plans.status = 'applied'
538 AND json_extract(planned.value, '$.number') = ?
539 LIMIT 1",
540 )
541 .bind(&[pull.repo_id.as_str().into(), number.into()])?
542 .first::<PlannedFiles>(None)
543 .await?
544 .and_then(|row| row.files)
545 .and_then(|files| serde_json::from_str::<Vec<String>>(&files).ok())
546 .unwrap_or_default(),
547 None => Vec::new(),
548 };
549 Ok::<_, worker::Error>((denials, unanswered, planned))
550 };
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily551 try_join(checks, try_join(run, counts)).await
552 }
553
554 #[allow(clippy::too_many_arguments)]
555 fn signals_from(
556 pull: &Pull,
557 known: Signals,
558 runs: Vec<(String, String)>,
559 latest: Option<LatestRun>,
560 halted: Option<String>,
561 denials: u32,
562 unanswered: u32,
563 expected: Vec<String>,
564 ) -> (Signals, Option<String>) {
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step565 let run_id = latest.as_ref().map(|run| run.id.clone());
566 let share = |used: Option<f64>, cap: Option<f64>| match (used, cap) {
567 (Some(used), Some(cap)) if cap > 0.0 => Some(used / cap),
568 _ => None,
569 };
570 let signals = Signals {
571 flaky_checks: flaky(&runs),
572 files: pull.files.clone(),
573 expected,
574 halted,
575 budget_share: latest.as_ref().and_then(|run| share(run.cost_usd, run.budget_usd)),
576 time_share: latest
577 .as_ref()
578 .and_then(|run| share(run.minutes, run.time_cap_minutes.map(f64::from))),
579 denials,
580 unanswered,
581 self_reported: latest
582 .as_ref()
583 .and_then(|run| run.self_level.as_deref())
584 .and_then(ConfidenceLevel::parse),
585 uncertain_about: latest
586 .as_ref()
587 .and_then(|run| run.uncertain_about.as_deref())
588 .and_then(|items| serde_json::from_str::<Vec<String>>(items).ok())
589 .unwrap_or_default(),
590 ..known
591 };
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily592 (signals, run_id)
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step593 }
594
595 /// Works out how sure g1t is of a pull request a g1t agent has finished
596 /// (`known` holds what its lifecycle already read), and records it on
597 /// the pull request and on the run that left it so when it changed.
598 pub(crate) async fn assess_confidence(&self, pull: &Pull, known: Signals) -> Result<Confidence> {
599 let (signals, run_id) = self.signals(pull, known).await?;
600 let (level, reasons) = score(&signals);
601 let unchanged = pull.confidence.as_ref().filter(|was| {
602 was.level == level
603 && was.reasons == reasons
604 && was.self_reported == signals.self_reported
605 && was.uncertain_about == signals.uncertain_about
606 && was.run_id == run_id
607 });
608 if let Some(was) = unchanged {
609 return Ok(was.clone());
610 }
611 let now = rfc3339(now_ms());
612 let confidence = Confidence {
613 level,
614 reasons,
615 self_reported: signals.self_reported,
616 uncertain_about: signals.uncertain_about,
617 run_id: run_id.clone(),
618 assessed_at: now.clone(),
619 };
620 let detail = serde_json::to_string(&confidence)?;
621 self.db
622 .prepare(
623 "INSERT INTO pull_confidence (pull_id, repo_id, level, detail, updated_at) VALUES (?, ?, ?, ?, ?)
624 ON CONFLICT (pull_id) DO UPDATE SET
625 level = excluded.level, detail = excluded.detail, updated_at = excluded.updated_at",
626 )
627 .bind(&[
628 pull.id.as_str().into(),
629 pull.repo_id.as_str().into(),
630 level.as_str().into(),
631 detail.as_str().into(),
632 now.as_str().into(),
633 ])?
634 .run()
635 .await?;
636 if let Some(run_id) = &run_id {
637 self.db
638 .prepare(
639 "INSERT INTO run_confidence (run_id, pull_id, repo_id, detail, updated_at) VALUES (?, ?, ?, ?, ?)
640 ON CONFLICT (run_id) DO UPDATE SET detail = excluded.detail, updated_at = excluded.updated_at",
641 )
642 .bind(&[
643 run_id.as_str().into(),
644 pull.id.as_str().into(),
645 pull.repo_id.as_str().into(),
646 detail.as_str().into(),
647 now.as_str().into(),
648 ])?
649 .run()
650 .await?;
651 }
652 Ok(confidence)
653 }
654
655 /// What the agent of a run said of its own change, with the run's token.
656 pub(crate) async fn report_confidence(&self, a: ReportConfidenceArgs) -> Result<Outcome<bool>> {
657 let run = self
658 .db
659 .prepare("SELECT repo_id, pull_id, token_hash FROM agent_runs WHERE id = ?")
660 .bind(&[a.run_id.as_str().into()])?
661 .first::<RunTicket>(None)
662 .await?
663 .filter(|run| !a.token.is_empty() && run.token_hash == hash(&a.token));
664 let Some(run) = run else {
665 return Ok(Outcome::fail(FailureCode::NotFound, "Run not found."));
666 };
667 let Some(level) = ConfidenceLevel::parse(&a.confidence) else {
668 return Ok(Outcome::fail(FailureCode::Invalid, "confidence is high, medium or low."));
669 };
670 let uncertain = serde_json::to_string(&tidy(&a.uncertain_about))?;
671 self.db
672 .prepare(
673 "INSERT INTO run_confidence (run_id, pull_id, repo_id, self_level, uncertain_about, updated_at)
674 VALUES (?, ?, ?, ?, ?, ?)
675 ON CONFLICT (run_id) DO UPDATE SET
676 self_level = excluded.self_level, uncertain_about = excluded.uncertain_about,
677 updated_at = excluded.updated_at",
678 )
679 .bind(&[
680 a.run_id.as_str().into(),
681 run.pull_id.as_deref().map_or(JsValue::NULL, JsValue::from),
682 run.repo_id.as_str().into(),
683 level.as_str().into(),
684 uncertain.into(),
685 rfc3339(now_ms()).into(),
686 ])?
687 .run()
688 .await?;
689 Ok(Outcome::Ok(true))
690 }
691
692 /// How many comments on lines a review by g1t's agent left, which it
693 /// records at the moment it finished.
694 pub(crate) async fn review_comments(&self, pull: &Pull, finished_at: &str) -> Result<u32> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily695 // Read for this request against the latest finished review, which
696 // is the one asked about whenever it is asked.
697 if let Some(found) = self.prefetched_pull(&pull.id) {
698 return Ok(found.first::<NumberRow>(Slot::ReviewComments)?.map_or(0, |row| row.n));
699 }
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step700 Ok(self
701 .db
702 .prepare(
703 "SELECT count(*) AS n FROM comments
704 WHERE repo_id = ? AND number = ? AND author_id = ? AND created_at = ? AND path IS NOT NULL",
705 )
706 .bind(&[
707 pull.repo_id.as_str().into(),
708 pull.number.into(),
709 AGENT_ID.into(),
710 finished_at.into(),
711 ])?
712 .first::<NumberRow>(None)
713 .await?
714 .map_or(0, |row| row.n))
715 }
716
g1t is the stored author of what it opens; the person who asked is requested_by and keeps the author's rights717 /// Whether a person other than its owner (whoever asked g1t for it, or
718 /// its author) approved the change since the agent last revised it:
719 /// someone has looked, so a hold is lifted.
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step720 pub(crate) async fn person_approved(&self, pull: &Pull, revised_at: Option<&str>) -> Result<bool> {
721 #[derive(Deserialize)]
722 struct Latest {
723 verdict: String,
724 created_at: String,
725 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily726 #[derive(Deserialize)]
727 struct ByAuthor {
728 author_id: String,
729 verdict: String,
730 created_at: String,
731 }
732 let rows = match self.prefetched_pull(&pull.id) {
733 Some(found) => found
734 .rows::<ByAuthor>(Slot::Verdicts)?
735 .into_iter()
736 .rev()
737 .filter(|row| {
g1t is the stored author of what it opens; the person who asked is requested_by and keeps the author's rights738 crate::lifecycle::from_someone_else(pull, &row.author_id)
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily739 && row.author_id != crate::lifecycle::POLICY_ACTOR_ID
740 })
741 .take(20)
742 .map(|row| Latest { verdict: row.verdict, created_at: row.created_at })
743 .collect::<Vec<_>>(),
744 None => self
745 .db
746 .prepare(
747 "SELECT verdict, created_at FROM comments
748 WHERE repo_id = ? AND number = ? AND verdict IS NOT NULL
749 AND author_id != ? AND author_id != ? AND author_id != ?
750 ORDER BY id DESC LIMIT 20",
751 )
752 .bind(&[
753 pull.repo_id.as_str().into(),
754 pull.number.into(),
g1t is the stored author of what it opens; the person who asked is requested_by and keeps the author's rights755 pull.owner().id.as_str().into(),
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily756 AGENT_ID.into(),
757 crate::lifecycle::POLICY_ACTOR_ID.into(),
758 ])?
759 .all()
760 .await?
761 .results::<Latest>()?,
762 };
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step763 Ok(rows
764 .first()
765 .is_some_and(|latest| latest.verdict == "approve" && revised_at.is_none_or(|revised| latest.created_at.as_str() > revised)))
766 }
767}
768
769#[cfg(test)]
770mod tests {
771 use super::*;
772
773 fn file(path: &str, lines: u32) -> ChangedFile {
774 ChangedFile { path: path.to_owned(), additions: lines, deletions: 0 }
775 }
776
Fast pages, required checks on the branch, self-hosted runners, honest incidents777 /// A clean change: required checks pass, approved on the first
778 /// review, a test beside the code, small.
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step779 fn clean() -> Signals {
780 Signals {
Fast pages, required checks on the branch, self-hosted runners, honest incidents781 required: RequiredSignal::Passing,
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step782 agent_review: true,
783 review: Some(Verdict::Approve),
784 files: vec![file("src/retry.ts", 40), file("src/retry.test.ts", 30)],
785 ..Signals::default()
786 }
787 }
788
789 #[test]
790 fn signals_score_as_the_table_says() {
791 use ConfidenceLevel::*;
792 let cases: Vec<(&str, Signals, ConfidenceLevel, &[&str])> = vec![
Fast pages, required checks on the branch, self-hosted runners, honest incidents793 ("clean", clean(), High, &["required checks pass", "approved on first review", "tests added", "small change"]),
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step794 (
Fast pages, required checks on the branch, self-hosted runners, honest incidents795 "failing required checks",
796 Signals { required: RequiredSignal::Failing, ..clean() },
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step797 Low,
Fast pages, required checks on the branch, self-hosted runners, honest incidents798 &["required checks failing"],
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step799 ),
800 (
Fast pages, required checks on the branch, self-hosted runners, honest incidents801 "required checks not run",
802 Signals { required: RequiredSignal::NotRun, ..clean() },
803 Medium,
804 &["required checks not run"],
805 ),
806 (
807 "required checks still running",
808 Signals { required: RequiredSignal::Running, ..clean() },
809 Medium,
810 &["required checks not finished"],
811 ),
812 (
813 "failed in the merge queue",
814 Signals { queue_failed: true, ..clean() },
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step815 Low,
Fast pages, required checks on the branch, self-hosted runners, honest incidents816 &["failed in the merge queue"],
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step817 ),
818 ("one revision", Signals { revisions: 1, ..clean() }, Medium, &["1 revision"]),
819 ("three revisions", Signals { revisions: 3, ..clean() }, Low, &["3 revisions"]),
820 (
821 "no tests and three revisions",
822 Signals { revisions: 3, files: vec![file("src/retry.ts", 40)], ..clean() },
823 Low,
824 &["3 revisions", "tests not added"],
825 ),
826 (
827 "no tests",
828 Signals { files: vec![file("src/retry.ts", 40)], ..clean() },
829 Medium,
830 &["tests not added"],
831 ),
832 (
833 "docs need no tests",
834 Signals { files: vec![file("README.md", 40), file("docs/guide.md", 10)], ..clean() },
835 High,
Fast pages, required checks on the branch, self-hosted runners, honest incidents836 &["required checks pass", "approved on first review", "small change"],
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step837 ),
838 (
839 "the reviewer asks for changes",
840 Signals { review: Some(Verdict::RequestChanges), ..clean() },
841 Low,
842 &["reviewer asked for changes"],
843 ),
844 (
845 "a review with much to say",
846 Signals { review_comments: 4, ..clean() },
847 Medium,
848 &["reviewer left 4 comments"],
849 ),
850 ("no review yet", Signals { review: None, ..clean() }, Medium, &["not reviewed yet"]),
851 (
Fast pages, required checks on the branch, self-hosted runners, honest incidents852 "no reviewer agent and no required checks",
853 Signals { review: None, agent_review: false, required: RequiredSignal::NoneRequired, ..clean() },
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step854 Medium,
Fast pages, required checks on the branch, self-hosted runners, honest incidents855 &["branch has no required checks", "no review"],
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step856 ),
857 (
858 "a large change",
859 Signals { files: vec![file("src/a.ts", 900), file("src/a.test.ts", 300)], ..clean() },
860 Medium,
861 &["large change (1200 lines)"],
862 ),
863 (
864 "outside the planned area",
865 Signals {
866 expected: vec!["src/retry.ts".to_owned()],
867 files: vec![file("src/retry.ts", 10), file("lib/billing/charge.ts", 10), file("src/retry.test.ts", 5)],
868 ..clean()
869 },
870 Medium,
871 &["1 file outside the planned area"],
872 ),
873 (
874 "beside the planned files is inside",
875 Signals {
876 expected: vec!["src/webhooks/retry.ts".to_owned()],
877 files: vec![file("src/webhooks/backoff.ts", 10), file("src/webhooks/retry.test.ts", 5)],
878 ..clean()
879 },
880 High,
Fast pages, required checks on the branch, self-hosted runners, honest incidents881 &["required checks pass", "approved on first review", "tests added", "small change"],
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step882 ),
883 (
884 "a CI workflow",
885 Signals { files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
886 Medium,
887 &["touches CI workflows"],
888 ),
889 (
890 "a CI workflow and a revision",
891 Signals { revisions: 1, files: vec![file(".github/workflows/ci.yml", 5), file("src/a.test.ts", 5)], ..clean() },
892 Low,
893 &["touches CI workflows", "1 revision"],
894 ),
895 ("stopped at a cap", Signals { halted: Some("budget".to_owned()), ..clean() }, Low, &["stopped at its cost cap"]),
896 (
897 "near its cost cap",
898 Signals { budget_share: Some(0.92), ..clean() },
899 Medium,
900 &["used 92% of its cost cap"],
901 ),
902 (
903 "flaky checks",
904 Signals { flaky_checks: true, ..clean() },
905 Medium,
906 &["checks passed only on a retry"],
907 ),
908 (
909 "unanswered questions",
910 Signals { unanswered: 2, ..clean() },
911 Medium,
912 &["2 questions unanswered"],
913 ),
914 (
915 "guardrails refused a lot",
916 Signals { denials: 3, revisions: 1, ..clean() },
917 Low,
918 &["3 steps refused by guardrails", "1 revision"],
919 ),
920 (
921 "the agent says low",
922 Signals { self_reported: Some(Low), ..clean() },
923 Low,
924 &["agent says low"],
925 ),
926 (
927 "the agent cannot raise it",
928 Signals { self_reported: Some(High), revisions: 1, ..clean() },
929 Medium,
930 &["1 revision"],
931 ),
932 (
933 "the agent's doubts count",
934 Signals { self_reported: Some(High), uncertain_about: vec!["the retry limit".to_owned()], ..clean() },
935 Medium,
936 &["agent unsure about the retry limit"],
937 ),
938 ];
939 for (name, signals, level, reasons) in cases {
940 let (got, why) = score(&signals);
941 assert_eq!(got, level, "{name}: {why:?}");
942 assert_eq!(why, reasons.iter().map(|r| (*r).to_owned()).collect::<Vec<_>>(), "{name}");
943 }
944 }
945
946 #[test]
947 fn reasons_are_few_and_the_worst_come_first() {
948 let signals = Signals {
Fast pages, required checks on the branch, self-hosted runners, honest incidents949 required: RequiredSignal::Failing,
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step950 revisions: 2,
951 files: vec![file("src/a.ts", 600), file(".env.example", 1)],
952 unanswered: 1,
953 ..clean()
954 };
955 let (level, reasons) = score(&signals);
956 assert_eq!(level, ConfidenceLevel::Low);
957 assert_eq!(reasons.len(), MAX_REASONS);
Fast pages, required checks on the branch, self-hosted runners, honest incidents958 assert_eq!(reasons[0], "required checks failing");
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step959 }
960
961 #[test]
962 fn checks_that_pass_on_a_retry_are_flaky() {
963 let runs = |list: &[(&str, &str)]| list.iter().map(|(c, s)| ((*c).to_owned(), (*s).to_owned())).collect::<Vec<_>>();
964 assert!(!flaky(&runs(&[("a", "failed"), ("b", "passed")])));
965 assert!(flaky(&runs(&[("a", "failed"), ("a", "passed")])));
966 assert!(flaky(&runs(&[("a", "errored"), ("b", "passed")])));
967 assert!(!flaky(&runs(&[("a", "passed")])));
968 }
969
970 #[test]
971 fn tests_prose_and_sensitive_paths_are_told_apart() {
A file named test.js or spec.rb counts as a test: confidence no longer says tests not added when they were972 for path in ["src/__tests__/a.ts", "tests/test_api.py", "pkg/api_test.go", "src/a.spec.tsx", "crates/x/tests/it.rs", "test.js", "lib/spec.rb"] {
Thirteen MCP tools and classic token scopes; agents rate their confidence and can be put on an issue in one step973 assert!(is_test(path), "{path}");
974 }
975 for path in ["src/testing.ts", "src/contest.rs", "attest/a.rs"] {
976 assert!(!is_test(path), "{path}");
977 }
978 assert!(is_prose("docs/setup.md") && is_prose("README.md") && !is_prose("src/readme.ts"));
979 assert_eq!(sensitive(".github/workflows/ci.yml"), Some("CI workflows"));
980 assert_eq!(sensitive("apps/web/wrangler.jsonc"), Some("infrastructure"));
981 assert_eq!(sensitive("config/.env.production"), Some("secrets"));
982 assert_eq!(sensitive("src/env.ts"), None);
983 }
984
985 #[test]
986 fn doubts_are_tidied() {
987 let items = vec![" the retry limit ".to_owned(), "the retry limit".to_owned(), String::new(), "x".repeat(400)];
988 let tidied = tidy(&items);
989 assert_eq!(tidied.len(), 2);
990 assert_eq!(tidied[0], "the retry limit");
991 assert_eq!(tidied[1].chars().count(), MAX_UNCERTAIN_CHARS);
992 }
993}