Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/crates/runner/src/confidence.rs

136 lines5,767 bytesCodeBlame
1//! How sure the agent is of its own change, asked for at the end of every
2//! run that makes or revises one and reported to g1t.
3//!
4//! Like `learned`, the question rides on the prompt: the agent ends its
5//! closing summary with a `<g1t-confidence>` block of JSON, which is taken
6//! out of the summary and sent with the run's own token. g1t combines it
7//! with what it observes of the change (its checks, revisions, review,
8//! tests, size, guardrails); what it observes can only lower what the agent
9//! says, never raise it.
10
11use std::time::Duration;
12
13use serde_json::{Value, json};
14
15const OPEN: &str = "<g1t-confidence>";
16const CLOSE: &str = "</g1t-confidence>";
17const MAX_ITEMS: usize = 5;
18const MAX_ITEM_CHARS: usize = 160;
19
20/// What the agent is asked, after its task.
21pub const ASK: &str = "Also end your final message with how sure you are that your change is right and complete, as JSON inside <g1t-confidence></g1t-confidence> tags: {\"confidence\": \"high\" | \"medium\" | \"low\", \"uncertain_about\": [\"a few words for each thing you could not verify or had to guess\"]}. Say high only if you ran the checks and tests and they passed and nothing was guessed. The block is taken out of your summary.";
22
23/// The prompt with the question added.
24pub fn ask(prompt: &str) -> String {
25 format!("{prompt}\n\n{ASK}")
26}
27
28/// What the agent said: its level and what it was unsure about.
29#[derive(Debug, PartialEq)]
30pub struct SelfReport {
31 pub confidence: &'static str,
32 pub uncertain_about: Vec<String>,
33}
34
35fn clip(text: &str) -> String {
36 let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
37 text.chars().take(MAX_ITEM_CHARS).collect()
38}
39
40/// The summary without its `<g1t-confidence>` block, and what the block
41/// said. A missing or unreadable block reports nothing.
42pub fn split(summary: &str) -> (String, Option<SelfReport>) {
43 let Some(start) = summary.rfind(OPEN) else {
44 return (summary.trim().to_owned(), None);
45 };
46 let body_start = start + OPEN.len();
47 let end = summary[body_start..].find(CLOSE).map(|at| body_start + at);
48 let body = &summary[body_start..end.unwrap_or(summary.len())];
49 let rest = end.map_or("", |end| &summary[end + CLOSE.len()..]);
50 let clean = format!("{}{}", summary[..start].trim_end(), rest.trim_end()).trim().to_owned();
51 let body = body.trim().trim_start_matches("```json").trim_start_matches("```").trim_end_matches("```").trim();
52 let Ok(value) = serde_json::from_str::<Value>(body) else {
53 return (clean, None);
54 };
55 let confidence = match value["confidence"].as_str().map(|level| level.trim().to_ascii_lowercase()) {
56 Some(level) if level == "high" => "high",
57 Some(level) if level == "medium" => "medium",
58 Some(level) if level == "low" => "low",
59 _ => return (clean, None),
60 };
61 let uncertain_about = value["uncertain_about"]
62 .as_array()
63 .map(|items| {
64 items
65 .iter()
66 .filter_map(Value::as_str)
67 .map(clip)
68 .filter(|item| !item.is_empty())
69 .take(MAX_ITEMS)
70 .collect()
71 })
72 .unwrap_or_default();
73 (clean, Some(SelfReport { confidence, uncertain_about }))
74}
75
76/// Sends what the agent said with the run's token. Never fails the run.
77pub fn report(said: &SelfReport) {
78 let (Ok(run), Ok(token)) = (std::env::var("AGENT_RUN"), std::env::var("AGENT_RUN_TOKEN")) else {
79 return;
80 };
81 let api = std::env::var("G1T_API").unwrap_or_else(|_| "https://api.g1t.sh".to_owned());
82 let sent = ureq::post(&format!("{}/agent-runs/{run}/confidence", api.trim_end_matches('/')))
83 .timeout(Duration::from_secs(10))
84 .send_json(json!({
85 "token": token,
86 "confidence": said.confidence,
87 "uncertain_about": said.uncertain_about,
88 }));
89 if let Err(error) = sent {
90 eprintln!("g1t-runner: could not report how sure the agent is: {error}");
91 }
92}
93
94/// The summary with its block taken out, after reporting what it said.
95pub fn finish(summary: String) -> String {
96 let (clean, said) = split(&summary);
97 if let Some(said) = &said {
98 report(said);
99 }
100 clean
101}
102
103#[cfg(test)]
104mod tests {
105 use super::*;
106
107 #[test]
108 fn the_block_is_taken_out_and_read() {
109 let summary = "Added retries with backoff.\n\n<g1t-confidence>{\"confidence\": \"Medium\", \"uncertain_about\": [\" the retry limit \", \"\", 3]}</g1t-confidence>";
110 let (clean, said) = split(summary);
111 assert_eq!(clean, "Added retries with backoff.");
112 assert_eq!(
113 said,
114 Some(SelfReport { confidence: "medium", uncertain_about: vec!["the retry limit".to_owned()] })
115 );
116 }
117
118 #[test]
119 fn a_missing_or_unknown_level_reports_nothing() {
120 assert_eq!(split("Done."), ("Done.".to_owned(), None));
121 assert_eq!(split("Done.\n<g1t-confidence>not json</g1t-confidence>").1, None);
122 assert_eq!(split("Done.\n<g1t-confidence>{\"confidence\": \"very\"}</g1t-confidence>").1, None);
123 let (clean, said) = split("Done.\n<g1t-confidence>\n```json\n{\"confidence\": \"high\"}\n```\n</g1t-confidence>");
124 assert_eq!(clean, "Done.");
125 assert_eq!(said.map(|said| said.confidence), Some("high"));
126 }
127
128 #[test]
129 fn it_leaves_the_learned_block_alone() {
130 let summary = "Fixed it.\n<g1t-learned>[]</g1t-learned>\n<g1t-confidence>{\"confidence\": \"low\"}</g1t-confidence>";
131 let (clean, said) = split(summary);
132 assert_eq!(clean, "Fixed it.\n<g1t-learned>[]</g1t-learned>");
133 assert_eq!(said.map(|said| said.confidence), Some("low"));
134 assert!(ask("Fix it.").starts_with("Fix it.\n\n"));
135 }
136}