Sign in

flagon-io/g1t

public

Git for AI scale: a forge for thousands of agents working on the same code at once.

gitagentscloudflarerustforgemcp

Commit

The keeper reads Cloudflare as it really answers

Probed with the keeper's token: the usage API is /billable-usage, with daily rows whose PricingQuantity still includes the free tier, so a rate is only read off days that were charged. AI Gateway ignores bracketed filters and returned every log; the filter now goes as encoded JSON. A sandbox second is now its memory and disk for the whole second plus the CPU sandboxes really use, from Containers' usage analytics (byte-seconds and CPU seconds), at billed rates or Cloudflare's published ones while the included amount covers them. The costs are first checked at once, then daily.

syntaqxcommitted Parent4187a00Browse files
2 files+159−400/2 viewed
+156−38
3939 const GIVE_UP_AFTER_MS: u64 = 3 * 60 * 60 * 1000;
4040 /// A run never finished after this died without reporting.
4141 const ABANDONED_AFTER_MS: u64 = 3 * 60 * 60 * 1000;
42−/// Too little spend to measure a cost from.
43−const MIN_MEASURED_USD: f64 = 5.0;
4442 /// Smaller moves are noise.
4543 const MIN_CHANGE: f64 = 0.02;
4644 /// A measurement outside this factor of the current cost is suspect.
9694 let mut cost = 0.0;
9795 let mut count = 0;
9896 for page in 1..=40 {
97+ // The filter goes as URL-encoded JSON; the bracket form is
98+ // ignored, and would sum every log there is. Session ids are
99+ // [a-z0-9_], which need no escaping inside it.
100+ let filter = format!(
101+ "%5B%7B%22key%22%3A%22metadata.value%22%2C%22operator%22%3A%22eq%22%2C%22value%22%3A%5B%22{session}%22%5D%7D%5D"
102+ );
99103 let url = self.api(&format!(
100− "/ai-gateway/gateways/{}/logs?per_page=50&page={page}\
101− &filters[0][key]=metadata.value&filters[0][operator]=eq&filters[0][value][0]={session}",
104+ "/ai-gateway/gateways/{}/logs?per_page=50&page={page}&filters={filter}",
102105 self.gateway
103106 ));
104107 let body = self.send(Method::Get, &url, None).await?;
114117 Ok((cost, count))
115118 }
116119
117− /// The account's billable usage this month, as Cloudflare reports it.
120+ /// The account's billable usage, one row per service per day, as
121+ /// Cloudflare reports it.
118122 async fn billable_usage(&self, from: &str, to: &str) -> Result<Vec<UsageRow>> {
119123 let body = self
120− .send(Method::Get, &self.api(&format!("/billing/usage/paygo?from={from}&to={to}")), None)
124+ .send(Method::Get, &self.api(&format!("/billable-usage?from={from}&to={to}")), None)
121125 .await?;
122126 let rows = body["result"].as_array().cloned().unwrap_or_default();
123127 Ok(rows.iter().filter_map(UsageRow::from_value).collect())
124128 }
125129
126− /// Seconds g1t's containers ran since `since` (RFC 3339), across the
127− /// account.
128− async fn container_seconds(&self, since: &str, until: &str) -> Result<f64> {
129− let query = "query ($account: String!, $since: Time!, $until: Time!) {
130+ /// What g1t's containers used from `since` to `until` (dates), as
131+ /// Cloudflare bills it: memory in byte-seconds, and CPU seconds.
132+ async fn container_usage(&self, since: &str, until: &str) -> Result<ContainerUsage> {
133+ let query = "query ($account: String!, $since: Date!, $until: Date!) {
130134 viewer { accounts(filter: { accountTag: $account }) {
131− containersMetricsAdaptiveGroups(limit: 10000, filter: { datetime_geq: $since, datetime_leq: $until }) {
132− sum { containerUptime }
135+ containersUsageAdaptiveGroups(limit: 1000, filter: { date_geq: $since, date_leq: $until }) {
136+ sum { cpuTimeSec allocatedMemory }
133137 }
134138 } }
135139 }";
140144 Some(json!({ "query": query, "variables": { "account": self.account, "since": since, "until": until } })),
141145 )
142146 .await?;
143− let groups = body["data"]["viewer"]["accounts"][0]["containersMetricsAdaptiveGroups"]
147+ let groups = body["data"]["viewer"]["accounts"][0]["containersUsageAdaptiveGroups"]
144148 .as_array()
145149 .cloned()
146150 .unwrap_or_default();
147− Ok(groups.iter().map(|g| g["sum"]["containerUptime"].as_f64().unwrap_or(0.0)).sum())
151+ Ok(groups.iter().fold(ContainerUsage::default(), |total, g| ContainerUsage {
152+ cpu_seconds: total.cpu_seconds + g["sum"]["cpuTimeSec"].as_f64().unwrap_or(0.0),
153+ memory_byte_seconds: total.memory_byte_seconds + g["sum"]["allocatedMemory"].as_f64().unwrap_or(0.0),
154+ }))
148155 }
149156 }
150157
158+#[derive(Debug, Default, Clone, Copy)]
159+pub(crate) struct ContainerUsage {
160+ cpu_seconds: f64,
161+ memory_byte_seconds: f64,
162+}
163+
164+/// g1t's sandboxes: Containers' standard-1, half a vCPU, 4 GiB, 8 GB disk.
165+const SANDBOX_GIB: f64 = 4.0;
166+const SANDBOX_DISK_GB: f64 = 8.0;
167+const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
168+
169+/// Cloudflare's published Containers rates, in dollars, used for any rate
170+/// the bill does not show yet (while usage is inside the included amount).
171+const LIST_MEMORY_GIB_SECOND: f64 = 0.000_002_5;
172+const LIST_DISK_GB_SECOND: f64 = 0.000_000_07;
173+const LIST_VCPU_SECOND: f64 = 0.000_02;
174+
175+/// What one second of a sandbox costs, in millionths of a dollar: its
176+/// memory and disk for the whole second, and the CPU sandboxes actually
177+/// use per second of running, which is billed only while busy.
178+pub(crate) fn sandbox_second_micros(usage: ContainerUsage, memory: f64, disk: f64, vcpu: f64) -> Option<f64> {
179+ let instance_seconds = usage.memory_byte_seconds / (SANDBOX_GIB * GIB);
180+ if instance_seconds < 3600.0 {
181+ return None;
182+ }
183+ let cpu_share = usage.cpu_seconds / instance_seconds;
184+ Some((SANDBOX_GIB * memory + SANDBOX_DISK_GB * disk + cpu_share * vcpu) * MICROS_PER_DOLLAR as f64)
185+}
186+
187+/// A unit's marginal rate from the bill: the median, over the days that
188+/// were charged, of cost over quantity. None while nothing was charged.
189+pub(crate) fn billed_rate(rows: &[&UsageRow]) -> Option<f64> {
190+ let mut rates: Vec<f64> = rows
191+ .iter()
192+ .filter(|r| r.cost > 0.0 && r.quantity > 0.0)
193+ .map(|r| r.cost / r.quantity)
194+ .collect();
195+ if rates.is_empty() {
196+ return None;
197+ }
198+ rates.sort_by(f64::total_cmp);
199+ Some(rates[rates.len() / 2])
200+}
201+
151202 /// One line of Cloudflare's billable usage.
152203 #[derive(Debug, Clone)]
153204 pub(crate) struct UsageRow {
177228 period_start: text(&["ChargePeriodStart", "charge_period_start"]),
178229 period_end: text(&["ChargePeriodEnd", "charge_period_end"]),
179230 service: if family.is_empty() { service } else { format!("{family} / {service}") },
180− unit: text(&["ConsumedUnit", "PricingUnit", "consumed_unit"]),
231+ unit: text(&["PricingUnit", "ConsumedUnit", "consumed_unit"]),
181232 quantity: number(&["PricingQuantity", "ConsumedQuantity", "pricing_quantity"]),
182− cost: number(&["ContractedCost", "BilledCost", "contracted_cost"]),
233+ // What g1t pays; list price if nothing was contracted.
234+ cost: Some(number(&["ContractedCost", "BilledCost", "contracted_cost"]))
235+ .filter(|cost| *cost > 0.0)
236+ .unwrap_or_else(|| number(&["ListCost", "list_cost"])),
183237 })
184238 }
185239 }
288342 })
289343 }
290344
345+ /// Whether the costs have never been checked against Cloudflare's bill.
346+ pub(crate) async fn never_checked(&self) -> Result<bool> {
347+ Ok(self
348+ .db
349+ .prepare("SELECT meter FROM prices WHERE checked_at IS NOT NULL LIMIT 1")
350+ .first::<Value>(None)
351+ .await?
352+ .is_none())
353+ }
354+
291355 /// A meter's cost and price per unit, from the book.
292356 pub(crate) async fn price(&self, meter: &str) -> Result<Option<(f64, f64)>> {
293357 #[derive(Deserialize)]
429493 return Ok(());
430494 }
431495 let now = rfc3339(now_ms());
432− let month_start = format!("{}-01", &now[..7]);
433496 let today = &now[..10];
434− let rows = keeper.billable_usage(&month_start, today).await?;
497+ let since = rfc3339(now_ms() - 30 * 24 * 60 * 60 * 1000);
498+ let rows = keeper.billable_usage(&since[..10], today).await?;
435499 for row in &rows {
436500 self.db
437501 .prepare(
453517 .await?;
454518 }
455519
456− let matching = |service: &str, unit: Option<&str>| -> (f64, f64) {
520+ let named = |words: &[&str]| -> Vec<&UsageRow> {
457521 rows.iter()
458− .filter(|r| r.service.to_lowercase().contains(service))
459− .filter(|r| unit.is_none_or(|u| r.unit.to_lowercase().contains(u)))
460− .fold((0.0, 0.0), |(q, c), r| (q + r.quantity, c + r.cost))
522+ .filter(|r| {
523+ let service = r.service.to_lowercase();
524+ words.iter().all(|word| service.contains(word))
525+ })
526+ .collect()
461527 };
462528
463− // Containers: what they cost, over the seconds they ran.
464− let (_, container_cost) = matching("container", None);
465− if container_cost >= MIN_MEASURED_USD {
466− let seconds = keeper.container_seconds(&format!("{month_start}T00:00:00Z"), &now).await?;
467− if seconds > 0.0 {
468− let per_second = container_cost * MICROS_PER_DOLLAR as f64 / seconds;
469− for meter in ["sandbox_second", "build_second"] {
470− self.measure(meter, per_second, &format!("Cloudflare billed ${container_cost:.2} for {seconds:.0} container-seconds this month")).await?;
471− }
529+ // Containers: each resource at what the bill shows it costs, or
530+ // the published rate while the included amount still covers it,
531+ // over how much CPU g1t's sandboxes really use per second.
532+ let memory = billed_rate(&named(&["container memory"]));
533+ let disk = billed_rate(&named(&["container disk"]));
534+ let vcpu = billed_rate(&named(&["container vcpu"]));
535+ let usage = keeper.container_usage(&since[..10], today).await?;
536+ if let Some(per_second) = sandbox_second_micros(
537+ usage,
538+ memory.unwrap_or(LIST_MEMORY_GIB_SECOND),
539+ disk.unwrap_or(LIST_DISK_GB_SECOND),
540+ vcpu.unwrap_or(LIST_VCPU_SECOND),
541+ ) {
542+ let instance_seconds = usage.memory_byte_seconds / (SANDBOX_GIB * GIB);
543+ let billed = [("memory", memory), ("disk", disk), ("vCPU", vcpu)]
544+ .iter()
545+ .filter(|(_, rate)| rate.is_some())
546+ .map(|(name, _)| *name)
547+ .collect::<Vec<_>>();
548+ let reason = format!(
549+ "Sandboxes used {:.2} vCPU per second over {:.0} hours of Cloudflare Containers in the last 30 days; {}",
550+ usage.cpu_seconds / instance_seconds,
551+ instance_seconds / 3600.0,
552+ if billed.is_empty() {
553+ "rates are Cloudflare's published ones".to_owned()
554+ } else {
555+ format!("{} at what Cloudflare billed", billed.join(", "))
556+ },
557+ );
558+ for meter in ["sandbox_second", "build_second"] {
559+ self.measure(meter, per_second, &reason).await?;
472560 }
473561 }
474− // Workers for Platforms: per million requests and CPU milliseconds.
475− for (meter, unit, scale) in [("app_requests", "request", 1e6), ("app_cpu", "ms", 1e6)] {
476− let (quantity, cost) = matching("workers for platforms", Some(unit));
477− if cost >= MIN_MEASURED_USD && quantity > 0.0 {
478− let per = cost * MICROS_PER_DOLLAR as f64 / quantity * scale;
479− self.measure(meter, per, &format!("Cloudflare billed ${cost:.2} for {quantity:.0} {unit}s this month")).await?;
562+ // Apps run as Workers: per million requests and CPU milliseconds,
563+ // once the bill shows them charged.
564+ let app_meters: [(&str, &[&str], &str); 2] = [
565+ ("app_requests", &["workers", "requests"], "requests"),
566+ ("app_cpu", &["workers cpu"], "CPU ms"),
567+ ];
568+ for (meter, words, unit) in app_meters {
569+ if let Some(rate) = billed_rate(&named(words)) {
570+ let reason = format!("Cloudflare billed Workers {unit} at ${:.2} per million", rate * 1e6);
571+ self.measure(meter, rate * 1e6 * MICROS_PER_DOLLAR as f64, &reason).await?;
480572 }
481573 }
482574 self.db
537629 }
538630
539631 #[test]
632+ fn a_sandbox_second_is_its_memory_and_disk_and_the_cpu_it_uses() {
633+ // An hour of sandboxes that kept a fifth of a vCPU busy.
634+ let usage = ContainerUsage { cpu_seconds: 720.0, memory_byte_seconds: 3600.0 * 4.0 * GIB };
635+ let micros = sandbox_second_micros(usage, LIST_MEMORY_GIB_SECOND, LIST_DISK_GB_SECOND, LIST_VCPU_SECOND).unwrap();
636+ // 4 x 2.5 + 8 x 0.07 + 0.2 x 20 = 14.56
637+ assert!((micros - 14.56).abs() < 1e-9, "{micros}");
638+ // Too little use to say anything.
639+ assert!(sandbox_second_micros(ContainerUsage { cpu_seconds: 1.0, memory_byte_seconds: GIB }, 1.0, 1.0, 1.0).is_none());
640+ }
641+
642+ #[test]
643+ fn a_billed_rate_is_the_median_of_the_charged_days() {
644+ let row = |quantity: f64, cost: f64| UsageRow {
645+ period_start: String::new(),
646+ period_end: String::new(),
647+ service: "Containers / Container Memory".into(),
648+ unit: "Count".into(),
649+ quantity,
650+ cost,
651+ };
652+ let rows = [row(100.0, 0.0), row(100.0, 0.0002), row(100.0, 0.00025), row(100.0, 0.00025)];
653+ assert_eq!(billed_rate(&rows.iter().collect::<Vec<_>>()), Some(0.000_002_5));
654+ assert_eq!(billed_rate(&[&row(5.0, 0.0)]), None);
655+ }
656+
657+ #[test]
540658 fn usage_rows_are_read_by_their_focus_names() {
541659 let row = UsageRow::from_value(&json!({
542660 "ServiceFamilyName": "Containers",
543661 "ServiceName": "Memory",
544− "ConsumedUnit": "GiB-seconds",
662+ "PricingUnit": "GiB-seconds",
545663 "PricingQuantity": "1200.5",
546664 "ContractedCost": 0.003,
547665 "ChargePeriodStart": "2026-10-01",
+3−2
859859 if let Err(error) = billing.settle_runs(&keeper).await {
860860 worker::console_error!("settling runs failed: {error}");
861861 }
862− // Once a day: check every cost against what Cloudflare billed.
863− if event.cron() == keeper::DAILY {
862+ // Once a day, and at once if the costs were never checked: check every
863+ // cost against what Cloudflare billed.
864+ if event.cron() == keeper::DAILY || billing.never_checked().await.unwrap_or(false) {
864865 if let Err(error) = billing.reconcile(&keeper).await {
865866 worker::console_error!("checking costs against Cloudflare failed: {error}");
866867 }