Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
266 linesCodeBlameRaw
1//! Auto: the model Ask answers a question with when the workspace chose
2//! none in particular. Two plain rules, no extra call to the provider:
3//!
4//! 1. [`need`]: whether the question is a lookup (who has #12, the link to
5//! a repository, how many pull requests are open) or needs more: several
6//! reads and reasoning over them, such as why or how something happened,
7//! what shipped, a summary of a day or a week, a failure to diagnose,
8//! work to plan, a long question or a long conversation.
9//! 2. [`choose`]: among the models the provider listed when it was last
10//! checked, a lookup goes to its newest fast model (Haiku, a mini or
11//! flash model) and the rest to its newest standard one (Sonnet, GPT,
12//! Gemini Pro). The largest and most expensive tiers (Opus, a `pro` of
13//! OpenAI's) are used only when the provider offers nothing else: the
14//! workspace pays for every answer on its own key.
15//!
16//! Models are told apart by the words in their names, which every provider
17//! uses the same way, so the rule needs no list of models to keep up to
18//! date. A provider that lists none (an endpoint, an Azure deployment)
19//! answers with its own model.
20
21use g1t_contracts::ai::{AskMessage, AskRole};
22
23/// What a question needs.
24#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25pub enum Need {
26 /// A lookup: one or two reads, a short answer.
27 Quick,
28 /// Several reads and reasoning over them.
29 Deep,
30}
31
32/// Words that mean a question needs more than a lookup, as the starts of
33/// words (`summar` is summary and summarize).
34const DEEP: &[&str] = &[
35 "why", "explain", "summar", "recap", "overview", "digest", "report", "ship", "deploy", "releas", "launch", "chang", "happen",
36 "compar", "review", "plan", "propos", "investigat", "debug", "diagnos", "fail", "broke", "break", "error", "regress", "incident",
37 "cause", "health", "progress", "block", "risk", "analy", "implement", "fix", "refactor", "design", "architect", "migrat",
38 "today", "yesterday", "week", "month", "since", "lately", "recent", "history", "timeline",
39];
40
41/// A question this long needs more than a lookup, in characters.
42const LONG_QUESTION: usize = 280;
43/// A conversation this long needs the stronger model to keep track, in
44/// characters.
45const LONG_CONVERSATION: usize = 12_000;
46
47/// What the conversation's last question needs.
48pub fn need(messages: &[AskMessage]) -> Need {
49 let Some(question) = messages.iter().rev().find(|message| message.role == AskRole::User) else {
50 return Need::Quick;
51 };
52 let text = question.content.to_lowercase();
53 let total: usize = messages.iter().map(|message| message.content.chars().count()).sum();
54 if text.chars().count() > LONG_QUESTION || total > LONG_CONVERSATION {
55 return Need::Deep;
56 }
57 // Several questions at once, or a list of asks.
58 if text.matches('?').count() > 1 || text.lines().filter(|line| !line.trim().is_empty()).count() > 2 {
59 return Need::Deep;
60 }
61 let words: Vec<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|word| !word.is_empty()).collect();
62 for (at, word) in words.iter().enumerate() {
63 // "How many" and "how much" count; "how" otherwise asks how
64 // something works or came to be.
65 if *word == "how" && !matches!(words.get(at + 1), Some(&("many" | "much" | "old" | "long" | "big"))) {
66 return Need::Deep;
67 }
68 if DEEP.iter().any(|stem| word.starts_with(stem)) {
69 return Need::Deep;
70 }
71 }
72 Need::Quick
73}
74
75/// How big a model is, by the words in its name.
76#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
77enum Tier {
78 /// nano, lite: the smallest.
79 Tiny,
80 /// Haiku, mini, flash, small, instant.
81 Fast,
82 /// Sonnet, GPT, Gemini Pro, and any name that says nothing.
83 Standard,
84 /// Opus, ultra, OpenAI's pro: the most expensive.
85 Largest,
86}
87
88/// The words of a model's name: `gpt-4o-mini` is gpt, 4o, mini.
89fn words(id: &str) -> Vec<String> {
90 id.to_lowercase().split(['-', '_', '.', '/', ':', ' ', '@']).filter(|word| !word.is_empty()).map(str::to_owned).collect()
91}
92
93/// A model's tier, or none for one Ask cannot use: code-only models
94/// (OpenAI's Codex answers only its Responses API), ChatGPT's own
95/// (which take no tools) and safety classifiers.
96fn tier(id: &str) -> Option<Tier> {
97 let words = words(id);
98 let has = |word: &str| words.iter().any(|w| w == word);
99 if words.iter().any(|w| w.contains("codex") || w == "chatgpt" || w.contains("guard")) {
100 return None;
101 }
102 let gemini = words.iter().any(|w| w.starts_with("gemini") || w.starts_with("gemma"));
103 Some(if has("nano") || has("lite") || has("tiny") {
104 Tier::Tiny
105 } else if has("haiku") || has("mini") || has("flash") || has("small") || has("instant") {
106 Tier::Fast
107 } else if has("opus") || has("ultra") || (has("pro") && !gemini) {
108 Tier::Largest
109 } else {
110 Tier::Standard
111 })
112}
113
114/// A model's version, from the numbers in its name: claude-sonnet-4-5 is
115/// 4.5, gpt-4.1 is 4.1, gpt-4o is 4. Dates (20250929, 2024-08-06, 2411) and
116/// sizes (70b) are not versions.
117fn version(id: &str) -> Vec<u32> {
118 let mut found = Vec::new();
119 for word in words(id) {
120 let digits: String = word.chars().take_while(char::is_ascii_digit).collect();
121 if digits.is_empty() {
122 continue;
123 }
124 let rest = &word[digits.len()..];
125 // 70b, 8x7b: a size.
126 if rest == "b" || rest.starts_with('x') || rest == "k" || rest == "m" {
127 continue;
128 }
129 match digits.parse::<u32>() {
130 Ok(number) if number < 1000 => found.push(number),
131 _ => {}
132 }
133 }
134 found
135}
136
137/// Whether a name is a dated snapshot rather than the name that follows the
138/// newest (claude-sonnet-4-5-20250929, gpt-4o-2024-08-06).
139fn dated(id: &str) -> bool {
140 words(id).iter().any(|word| word.len() >= 4 && word.chars().all(|c| c.is_ascii_digit()))
141}
142
143/// The best of some models in a tier: the newest version, an alias before a
144/// dated snapshot, then the shortest name, then by name.
145fn best<'a>(models: impl Iterator<Item = &'a String>) -> Option<&'a String> {
146 models.min_by(|a, b| {
147 version(b)
148 .cmp(&version(a))
149 .then(dated(a).cmp(&dated(b)))
150 .then(a.len().cmp(&b.len()))
151 .then(a.cmp(b))
152 })
153}
154
155/// The model Auto answers with: of `models` (what the provider listed), the
156/// best of the tier the question needs, falling back to the nearest other
157/// tier; `own` (the connection's own model) when the provider listed none
158/// Ask can use. None when there is nothing at all.
159pub fn choose(models: &[String], own: Option<&str>, need: Need) -> Option<String> {
160 let order: &[Tier] = match need {
161 Need::Quick => &[Tier::Fast, Tier::Standard, Tier::Tiny, Tier::Largest],
162 Need::Deep => &[Tier::Standard, Tier::Largest, Tier::Fast, Tier::Tiny],
163 };
164 let usable: Vec<(&String, Tier)> = models.iter().filter_map(|model| tier(model).map(|tier| (model, tier))).collect();
165 for wanted in order {
166 if let Some(model) = best(usable.iter().filter(|(_, tier)| tier == wanted).map(|(model, _)| *model)) {
167 return Some(model.clone());
168 }
169 }
170 own.map(str::trim).filter(|own| !own.is_empty()).map(str::to_owned)
171}
172
173#[cfg(test)]
174mod tests {
175 use super::*;
176
177 fn asked(question: &str) -> Vec<AskMessage> {
178 vec![AskMessage { role: AskRole::User, content: question.to_owned() }]
179 }
180
181 fn list(models: &[&str]) -> Vec<String> {
182 models.iter().map(|model| (*model).to_owned()).collect()
183 }
184
185 #[test]
186 fn lookups_are_quick_and_questions_that_need_reasoning_are_deep() {
187 for quick in [
188 "Who is assigned to #12?",
189 "Link me the docs repo",
190 "How many open pull requests are there?",
191 "List open pull requests in flagon-io/g1t",
192 "Who owns the billing service?",
193 ] {
194 assert_eq!(need(&asked(quick)), Need::Quick, "{quick}");
195 }
196 for deep in [
197 "What shipped today?",
198 "Why did Deploy #312 fail?",
199 "How does the merge queue decide what lands?",
200 "Summarize this week in flagon-io",
201 "What changed in the runner lately?",
202 "Is production healthy?",
203 "Plan the work for invoices",
204 "What's in #12? And who reviews it?",
205 ] {
206 assert_eq!(need(&asked(deep)), Need::Deep, "{deep}");
207 }
208 // A long question, and a long conversation.
209 assert_eq!(need(&asked(&"Who has it ".repeat(40))), Need::Deep);
210 let mut long = Vec::new();
211 for _ in 0..4 {
212 long.push(AskMessage { role: AskRole::User, content: "a".into() });
213 long.push(AskMessage { role: AskRole::Assistant, content: "x".repeat(4_000) });
214 }
215 long.push(AskMessage { role: AskRole::User, content: "Who has #3?".into() });
216 assert_eq!(need(&long), Need::Deep);
217 assert_eq!(need(&[]), Need::Quick);
218 }
219
220 #[test]
221 fn anthropic_lookups_go_to_haiku_and_the_rest_to_sonnet_never_opus() {
222 let models =
223 list(&["claude-3-5-haiku-20241022", "claude-haiku-5-5", "claude-opus-5-5", "claude-sonnet-4-5-20250929", "claude-sonnet-5-5"]);
224 assert_eq!(choose(&models, None, Need::Quick).as_deref(), Some("claude-haiku-5-5"));
225 assert_eq!(choose(&models, None, Need::Deep).as_deref(), Some("claude-sonnet-5-5"));
226 // Without a Sonnet, Opus; without a Haiku, Sonnet.
227 assert_eq!(choose(&list(&["claude-haiku-5-5", "claude-opus-5-5"]), None, Need::Deep).as_deref(), Some("claude-opus-5-5"));
228 assert_eq!(choose(&list(&["claude-sonnet-5-5", "claude-opus-5-5"]), None, Need::Quick).as_deref(), Some("claude-sonnet-5-5"));
229 // An alias before its dated snapshot.
230 assert_eq!(
231 choose(&list(&["claude-sonnet-4-5-20250929", "claude-sonnet-4-5"]), None, Need::Deep).as_deref(),
232 Some("claude-sonnet-4-5")
233 );
234 }
235
236 #[test]
237 fn other_providers_route_by_the_same_words() {
238 // OpenAI: mini for lookups, the standard model otherwise; never pro,
239 // Codex or ChatGPT's own.
240 let openai = list(&["chatgpt-4o-latest", "gpt-4o", "gpt-4o-mini", "gpt-5", "gpt-5-codex", "gpt-5-mini", "gpt-5-nano", "gpt-5-pro", "o3"]);
241 assert_eq!(choose(&openai, None, Need::Quick).as_deref(), Some("gpt-5-mini"));
242 assert_eq!(choose(&openai, None, Need::Deep).as_deref(), Some("gpt-5"));
243 // Gemini: flash, and pro (Gemini's standard tier).
244 let gemini = list(&["gemini-2.0-flash", "gemini-2.5-flash", "gemini-2.5-flash-lite", "gemini-2.5-pro"]);
245 assert_eq!(choose(&gemini, None, Need::Quick).as_deref(), Some("gemini-2.5-flash"));
246 assert_eq!(choose(&gemini, None, Need::Deep).as_deref(), Some("gemini-2.5-pro"));
247 // Groq: an instant model, and a versatile one; sizes are not versions.
248 let groq = list(&["llama-3.1-8b-instant", "llama-3.3-70b-versatile", "llama-guard-4-12b"]);
249 assert_eq!(choose(&groq, None, Need::Quick).as_deref(), Some("llama-3.1-8b-instant"));
250 assert_eq!(choose(&groq, None, Need::Deep).as_deref(), Some("llama-3.3-70b-versatile"));
251 // Mistral: small, and large (its standard).
252 let mistral = list(&["mistral-large-latest", "mistral-small-latest"]);
253 assert_eq!(choose(&mistral, None, Need::Quick).as_deref(), Some("mistral-small-latest"));
254 assert_eq!(choose(&mistral, None, Need::Deep).as_deref(), Some("mistral-large-latest"));
255 // One model is every answer's.
256 assert_eq!(choose(&list(&["qwen3-coder"]), None, Need::Quick).as_deref(), Some("qwen3-coder"));
257 }
258
259 #[test]
260 fn a_provider_that_lists_nothing_answers_with_its_own_model() {
261 assert_eq!(choose(&[], Some("prod-gpt"), Need::Deep).as_deref(), Some("prod-gpt"));
262 assert_eq!(choose(&list(&["gpt-5-codex"]), Some("prod-gpt"), Need::Quick).as_deref(), Some("prod-gpt"));
263 assert_eq!(choose(&[], Some(" "), Need::Quick), None);
264 assert_eq!(choose(&[], None, Need::Quick), None);
265 }
266}