Costs: read AI Gateway's analytics, planning runs as model cost, comped meters never money in
AI Gateway's analytics came back empty because the query grouped by the wholesale dimension: Cloudflare then answers no rows and no error. The same query without it returns 562 requests costing $11.11 for 2026-10-02 to 10-08 with the usage token, which has the permissions it needs. wholesale is now a filter on two aliased groups (own keys, and requests Cloudflare billed itself). The gateway is read over its own window, 31 days until it has answered, so the drift's 7 days are all there. When the gateway's side is still empty, the models drift says why as far as the run can tell: requests logged with no price, a token Cloudflare refuses for the gateway (REST 403/404), a gateway nothing went through, or a read that failed. An agent's planning run is ledger task "plan", which is also the plan's payments' revenue_map key, so its model cost went to running g1t (where Cloudflare's bill is the cost) and was dropped: $0.0748 of flagon-io's on 2026-10-07, shown as "Running g1t" charged $0.0897. It is reconciled as "planning" (models) now, and credits follow it. A 100%-discount workspace's month-end meters (actions cache, embeddings, scans) counted their pre-discount charge as cash: flagon-io "paid" $0.0023 and g1t's own spend showed it as money in. They are given (comped) with no cash, like its ledger usage.
| 254 | 254 | ||
| 255 | 255 | /// What AI Gateway priced each day's requests at, per provider and model, | |
| 256 | 256 | /// for g1t's gateway only: GraphQL `aiGatewayRequestsAdaptiveGroups`, with | |
| 257 | − | /// `sum.cost` (dollars), the tokens it priced and whether Cloudflare billed | |
| 258 | − | /// the request itself (`wholesale`). Field names checked against the | |
| 259 | − | /// schema (`AccountAiGatewayRequestsAdaptiveGroupsSum` and `…Dimensions`). | |
| 257 | + | /// `sum.cost` (dollars) and the tokens it priced; once for the requests | |
| 258 | + | /// g1t's own provider keys paid (`wholesale: 0`) and once for those | |
| 259 | + | /// Cloudflare billed itself (`wholesale: 1`). Field names checked against | |
| 260 | + | /// the schema (`AccountAiGatewayRequestsAdaptiveGroupsSum`, `…Dimensions` | |
| 261 | + | /// and `…Filter_InputObject`). | |
| 262 | + | /// | |
| 263 | + | /// `wholesale` is a filter here, never a dimension: grouped by it, | |
| 264 | + | /// Cloudflare answers no rows at all and no error. Seen 2026-10-08: with | |
| 265 | + | /// `wholesale` in `dimensions` the query returned nothing for 562 requests | |
| 266 | + | /// that cost $11.11, which it returns without it. | |
| 260 | 267 | pub(crate) const GATEWAY_QUERY: &str = "query ($account: String!, $gateway: String!, $since: Date!, $until: Date!) { | |
| 261 | 268 | viewer { accounts(filter: { accountTag: $account }) { | |
| 262 | − | aiGatewayRequestsAdaptiveGroups(limit: 10000, filter: { date_geq: $since, date_leq: $until, gateway: $gateway }) { | |
| 269 | + | own: aiGatewayRequestsAdaptiveGroups(limit: 10000, filter: { date_geq: $since, date_leq: $until, gateway: $gateway, wholesale: 0 }) { | |
| 263 | 270 | count | |
| 264 | 271 | sum { cost tokensIn tokensOut cacheReadTokens cacheWriteTokens } | |
| 265 | − | dimensions { date provider model wholesale } | |
| 272 | + | dimensions { date provider model } | |
| 266 | 273 | } | |
| 274 | + | wholesale: aiGatewayRequestsAdaptiveGroups(limit: 10000, filter: { date_geq: $since, date_leq: $until, gateway: $gateway, wholesale: 1 }) { | |
| 275 | + | count | |
| 276 | + | sum { cost tokensIn tokensOut cacheReadTokens cacheWriteTokens } | |
| 277 | + | dimensions { date provider model } | |
| 278 | + | } | |
| 267 | 279 | } } | |
| 268 | 280 | }"; | |
| 269 | 281 | ||
| ⋯ | |||
| 279 | 291 | if let Some(errors) = body["errors"].as_array().filter(|e| !e.is_empty()) { | |
| 280 | 292 | return Err(format!("AI Gateway analytics failed: {}", Value::Array(errors.clone()))); | |
| 281 | 293 | } | |
| 282 | − | let groups = body["data"]["viewer"]["accounts"][0]["aiGatewayRequestsAdaptiveGroups"].as_array().cloned().unwrap_or_default(); | |
| 294 | + | let account = &body["data"]["viewer"]["accounts"][0]; | |
| 295 | + | let groups = |alias: &str| account[alias].as_array().cloned().unwrap_or_default(); | |
| 283 | 296 | let mut lines = Vec::new(); | |
| 284 | − | for g in &groups { | |
| 297 | + | for (wholesale, g) in groups("own").into_iter().map(|g| (false, g)).chain(groups("wholesale").into_iter().map(|g| (true, g))) { | |
| 285 | 298 | let d = &g["dimensions"]; | |
| 286 | 299 | let Some(day) = d["date"].as_str().filter(|day| day.len() >= 10) else { continue }; | |
| 287 | 300 | let provider = d["provider"].as_str().unwrap_or("unknown"); | |
| 288 | 301 | let model = d["model"].as_str().unwrap_or("unknown"); | |
| 289 | − | let wholesale = d["wholesale"].as_u64().unwrap_or(0) == 1; | |
| 290 | 302 | let name = format!("{}{}", if wholesale { GATEWAY_WHOLESALE } else { "" }, slug(&format!("{provider} {model}"))); | |
| 291 | 303 | let sum = |key: &str| g["sum"][key].as_f64().unwrap_or(0.0); | |
| 292 | 304 | let line = |meter: String, unit: &str, quantity: f64, cost_usd: f64| CostLine { | |
| ⋯ | |||
| 323 | 335 | pub wholesale_usd: f64, | |
| 324 | 336 | /// Runs settled with the gateway's figure short (see `keeper::settled_cost`). | |
| 325 | 337 | pub short_runs: u32, | |
| 338 | + | /// Requests the gateway logged, priced or not. | |
| 339 | + | pub requests: f64, | |
| 326 | 340 | } | |
| 327 | 341 | ||
| 342 | + | /// What the last read of AI Gateway's analytics found, so the models drift | |
| 343 | + | /// can say why the gateway's side is empty rather than guess. | |
| 344 | + | #[derive(Clone, Debug, Default, PartialEq)] | |
| 345 | + | pub(crate) enum GatewayRead { | |
| 346 | + | /// Not read: no `AI_GATEWAY_ID`, or no token to read it with. | |
| 347 | + | #[default] | |
| 348 | + | NotRead, | |
| 349 | + | /// GraphQL refused it, or did not answer. | |
| 350 | + | Failed(String), | |
| 351 | + | /// It answered with requests. | |
| 352 | + | Rows, | |
| 353 | + | /// It answered with no rows. `visible`: whether the token it was read | |
| 354 | + | /// with can see the gateway (`GET …/ai-gateway/gateways/{id}`: 403 | |
| 355 | + | /// without AI Gateway Read, 404 for an id that is not there), to tell | |
| 356 | + | /// that from a gateway nothing went through; None when that could not | |
| 357 | + | /// be told. | |
| 358 | + | Empty { visible: Option<bool> }, | |
| 359 | + | } | |
| 360 | + | ||
| 328 | 361 | /// The caveats in AI Gateway's lines (any days, any order). | |
| 329 | 362 | pub(crate) fn gateway_caveats(lines: &[(String, f64, f64)]) -> GatewayCaveats { | |
| 330 | 363 | let mut out = GatewayCaveats::default(); | |
| ⋯ | |||
| 338 | 371 | } else if meter.ends_with(GATEWAY_CACHE_WRITE) { | |
| 339 | 372 | out.cache_write_tokens += quantity; | |
| 340 | 373 | } else { | |
| 374 | + | out.requests += quantity; | |
| 341 | 375 | *cost.entry(meter.as_str()).or_default() += cost_usd; | |
| 342 | 376 | if meter.starts_with(GATEWAY_WHOLESALE) { | |
| 343 | 377 | out.wholesale_usd += cost_usd; | |
| ⋯ | |||
| 543 | 577 | impl Billing { | |
| 544 | 578 | /// Reads Cloudflare's bill for the days due (see `window`) into | |
| 545 | 579 | /// `cost_lines`: the days read and how many lines. None without a | |
| 546 | − | /// token. What could not be read is added to `problems`. | |
| 547 | − | pub(crate) async fn read_cloudflare(&self, keeper: &Keeper, problems: &mut Vec<String>) -> Result<Option<(String, String, u32)>> { | |
| 580 | + | /// token. What could not be read is added to `problems`, and what AI | |
| 581 | + | /// Gateway's analytics answered to `gateway`. | |
| 582 | + | pub(crate) async fn read_cloudflare( | |
| 583 | + | &self, | |
| 584 | + | keeper: &Keeper, | |
| 585 | + | problems: &mut Vec<String>, | |
| 586 | + | gateway: &mut GatewayRead, | |
| 587 | + | ) -> Result<Option<(String, String, u32)>> { | |
| 548 | 588 | if !keeper.can_read_bill() { | |
| 549 | 589 | return Ok(None); | |
| 550 | 590 | } | |
| ⋯ | |||
| 579 | 619 | } | |
| 580 | 620 | // What AI Gateway priced g1t's own provider traffic at, each day: | |
| 581 | 621 | // the total the ledger's model cost is checked against (`margin`). | |
| 622 | + | // Over its own window: until it has answered with a line, the 31 | |
| 623 | + | // days GraphQL keeps, whatever the bill's window is. | |
| 582 | 624 | if !keeper.gateway().is_empty() { | |
| 625 | + | let last = self | |
| 626 | + | .db | |
| 627 | + | .prepare("SELECT MAX(day) AS day FROM cost_lines WHERE source = ?") | |
| 628 | + | .bind(&[SOURCE_GATEWAY.into()])? | |
| 629 | + | .first::<Last>(None) | |
| 630 | + | .await? | |
| 631 | + | .and_then(|l| l.day); | |
| 632 | + | let (from, to) = window(last.as_deref(), now_ms()); | |
| 583 | 633 | match keeper | |
| 584 | − | .gateway_graphql(gateway_variables(keeper.account(), keeper.gateway(), &since, &until)) | |
| 634 | + | .gateway_graphql(gateway_variables(keeper.account(), keeper.gateway(), &from, &to)) | |
| 585 | 635 | .await | |
| 586 | 636 | .map_err(|e| e.to_string()) | |
| 587 | 637 | .and_then(|body| lines_from_gateway(&body)) | |
| ⋯ | |||
| 591 | 641 | // re-read day must not keep its old line. | |
| 592 | 642 | self.db | |
| 593 | 643 | .prepare("DELETE FROM cost_lines WHERE source = ?1 AND day >= ?2 AND day <= ?3") | |
| 594 | − | .bind(&[SOURCE_GATEWAY.into(), since.as_str().into(), until.as_str().into()])? | |
| 644 | + | .bind(&[SOURCE_GATEWAY.into(), from.as_str().into(), to.as_str().into()])? | |
| 595 | 645 | .run() | |
| 596 | 646 | .await?; | |
| 647 | + | *gateway = if lines.is_empty() { GatewayRead::Empty { visible: keeper.gateway_visible().await } } else { GatewayRead::Rows }; | |
| 597 | 648 | written += self.upsert_lines(&lines, &fetched_at).await?; | |
| 598 | 649 | } | |
| 599 | − | Err(error) => problems.push(format!("AI Gateway's analytics could not be read: {error}")), | |
| 650 | + | Err(error) => { | |
| 651 | + | *gateway = GatewayRead::Failed(error.clone()); | |
| 652 | + | problems.push(format!("AI Gateway's analytics could not be read: {error}")); | |
| 653 | + | } | |
| 600 | 654 | } | |
| 601 | 655 | } | |
| 602 | 656 | Ok(Some((since, until, written))) | |
| ⋯ | |||
| 784 | 838 | /// AI Gateway's analytics in the shape of the schema | |
| 785 | 839 | /// (`aiGatewayRequestsAdaptiveGroups`: `count`, `sum`, `dimensions`). | |
| 786 | 840 | fn gateway_fixture() -> Value { | |
| 787 | − | let group = |day: &str, provider: &str, model: &str, wholesale: u8, count: u64, cost: f64, tokens: (f64, f64, f64, f64)| { | |
| 841 | + | let group = |day: &str, provider: &str, model: &str, count: u64, cost: f64, tokens: (f64, f64, f64, f64)| { | |
| 788 | 842 | json!({ | |
| 789 | 843 | "count": count, | |
| 790 | 844 | "sum": { "cost": cost, "tokensIn": tokens.0, "tokensOut": tokens.1, "cacheReadTokens": tokens.2, "cacheWriteTokens": tokens.3 }, | |
| 791 | − | "dimensions": { "date": day, "provider": provider, "model": model, "wholesale": wholesale } | |
| 845 | + | "dimensions": { "date": day, "provider": provider, "model": model } | |
| 792 | 846 | }) | |
| 793 | 847 | }; | |
| 794 | − | json!({ "data": { "viewer": { "accounts": [{ "aiGatewayRequestsAdaptiveGroups": [ | |
| 795 | − | group("2026-10-05", "anthropic", "claude-sonnet-5-5", 0, 120, 4.25, (900_000.0, 40_000.0, 3_000_000.0, 200_000.0)), | |
| 796 | − | group("2026-10-05", "anthropic", "claude-haiku-4-5-20251001", 0, 300, 0.40, (400_000.0, 20_000.0, 0.0, 0.0)), | |
| 797 | − | group("2026-10-06", "anthropic", "claude-new-1", 0, 12, 0.0, (80_000.0, 4_000.0, 0.0, 0.0)), | |
| 798 | − | group("2026-10-06", "openai", "gpt-x", 1, 5, 0.10, (1_000.0, 100.0, 0.0, 0.0)), | |
| 799 | − | ] }] } }, "errors": null }) | |
| 848 | + | json!({ "data": { "viewer": { "accounts": [{ | |
| 849 | + | "own": [ | |
| 850 | + | group("2026-10-05", "anthropic", "claude-sonnet-5-5", 120, 4.25, (900_000.0, 40_000.0, 3_000_000.0, 200_000.0)), | |
| 851 | + | group("2026-10-05", "anthropic", "claude-haiku-4-5-20251001", 300, 0.40, (400_000.0, 20_000.0, 0.0, 0.0)), | |
| 852 | + | group("2026-10-06", "anthropic", "claude-new-1", 12, 0.0, (80_000.0, 4_000.0, 0.0, 0.0)), | |
| 853 | + | ], | |
| 854 | + | "wholesale": [group("2026-10-06", "openai", "gpt-x", 5, 0.10, (1_000.0, 100.0, 0.0, 0.0))], | |
| 855 | + | }] } }, "errors": null }) | |
| 800 | 856 | } | |
| 801 | 857 | ||
| 802 | 858 | #[test] | |
| 803 | 859 | fn the_gateway_query_names_the_fields_its_schema_has() { | |
| 804 | 860 | // As checked against Cloudflare's GraphQL schema (introspection of | |
| 805 | 861 | // AccountAiGatewayRequestsAdaptiveGroups{,Sum,Dimensions,Filter}). | |
| 806 | − | for field in ["aiGatewayRequestsAdaptiveGroups", "date_geq", "date_leq", "gateway: $gateway", "count", "cost", "tokensIn", "tokensOut", "cacheReadTokens", "cacheWriteTokens", "date", "provider", "model", "wholesale"] { | |
| 862 | + | for field in ["aiGatewayRequestsAdaptiveGroups", "date_geq", "date_leq", "gateway: $gateway", "wholesale: 0", "wholesale: 1", "count", "cost", "tokensIn", "tokensOut", "cacheReadTokens", "cacheWriteTokens", "date", "provider", "model"] { | |
| 807 | 863 | assert!(GATEWAY_QUERY.contains(field), "{field}"); | |
| 808 | 864 | } | |
| 865 | + | // Grouped by `wholesale`, Cloudflare answers no rows and no error: | |
| 866 | + | // it is only ever a filter. | |
| 867 | + | for dimensions in GATEWAY_QUERY.split("dimensions {").skip(1) { | |
| 868 | + | let dimensions = &dimensions[..dimensions.find('}').unwrap()]; | |
| 869 | + | assert!(!dimensions.contains("wholesale"), "{dimensions}"); | |
| 870 | + | } | |
| 809 | 871 | let body = gateway_variables("acct", "g1t", "2026-10-01", "2026-10-07"); | |
| 810 | 872 | assert_eq!(body["variables"]["gateway"], "g1t"); | |
| 811 | 873 | } | |
| ⋯ | |||
| 835 | 897 | assert_eq!(caveats.unpriced, vec!["anthropic_claude_new_1".to_owned()]); | |
| 836 | 898 | assert_eq!((caveats.cache_read_tokens, caveats.cache_write_tokens), (3_000_000.0, 200_000.0)); | |
| 837 | 899 | assert!((caveats.wholesale_usd - 0.10).abs() < 1e-9); | |
| 900 | + | // Every request it logged, priced or not, from both answers. | |
| 901 | + | assert_eq!(caveats.requests, 437.0); | |
| 838 | 902 | // A model priced on one day and not another is priced. | |
| 839 | 903 | let mixed = vec![ | |
| 840 | 904 | ("m".to_owned(), 3.0, 0.5), | |
| 283 | 283 | } | |
| 284 | 284 | ||
| 285 | 285 | /// The key a usage line is reconciled under, as `margin::usage_rows` reads | |
| 286 | − | /// it: builds apart from a deployment's requests. | |
| 286 | + | /// it: builds apart from a deployment's requests, and an agent's planning | |
| 287 | + | /// run apart from the plan's payments (`margin::PLANNING_KEY`). | |
| 287 | 288 | pub(crate) fn usage_key(task: Option<&str>, reference: &str) -> String { | |
| 288 | 289 | match task { | |
| 289 | 290 | Some("deployments") if reference.starts_with("deploy/") => "builds".to_owned(), | |
| 291 | + | Some("plan") => crate::margin::PLANNING_KEY.to_owned(), | |
| 290 | 292 | Some(task) => task.to_owned(), | |
| 291 | 293 | None => "other".to_owned(), | |
| 292 | 294 | } | |
| ⋯ | |||
| 1140 | 1142 | assert_eq!(usage_key(Some("deployments"), "deploy/abc"), "builds"); | |
| 1141 | 1143 | assert_eq!(usage_key(Some("deployments"), "requests/2026-10"), "deployments"); | |
| 1142 | 1144 | assert_eq!(usage_key(Some("implement"), "run_1"), "implement"); | |
| 1145 | + | // An agent's planning run, never the plan's payments. | |
| 1146 | + | assert_eq!(usage_key(Some("plan"), "run_2"), "planning"); | |
| 1143 | 1147 | assert_eq!(usage_key(None, "crd_a"), "other"); | |
| 1144 | 1148 | } | |
| 1145 | 1149 | ||
| 116 | 116 | } | |
| 117 | 117 | ||
| 118 | 118 | /// A GraphQL query over AI Gateway's analytics: with the keeper's token | |
| 119 | − | /// (AI Gateway Read) first, and on failure with the bill's. Not the | |
| 120 | − | /// other way round: Cloudflare answers a token that cannot see AI | |
| 121 | − | /// Gateway with no rows, not an error, so the bill's token would read | |
| 122 | − | /// as a gateway that priced nothing. | |
| 119 | + | /// (AI Gateway Read) first, and on failure with the bill's. A token | |
| 120 | + | /// without Account Analytics Read gets an error ("caller does not hold | |
| 121 | + | /// any of the required permissions for this dataset"); when the answer | |
| 122 | + | /// has no rows, `gateway_visible` tells a token that cannot see the | |
| 123 | + | /// gateway from a gateway nothing went through. | |
| 123 | 124 | pub(crate) async fn gateway_graphql(&self, body: Value) -> Result<Value> { | |
| 124 | 125 | let Some(token) = &self.token else { | |
| 125 | 126 | return self.graphql(body).await; | |
| ⋯ | |||
| 133 | 134 | } | |
| 134 | 135 | } | |
| 135 | 136 | ||
| 137 | + | /// Whether the token AI Gateway's analytics are read with can see the | |
| 138 | + | /// gateway: the REST API answers 403 for a token without AI Gateway | |
| 139 | + | /// Read and 404 for a gateway id that is not there. None when the | |
| 140 | + | /// answer says neither (Cloudflare down, no token). | |
| 141 | + | pub(crate) async fn gateway_visible(&self) -> Option<bool> { | |
| 142 | + | let token = self.token.as_deref().or(self.billing_token.as_deref())?; | |
| 143 | + | match send_with(token, Method::Get, &self.api(&format!("/ai-gateway/gateways/{}", self.gateway)), None).await { | |
| 144 | + | Ok(_) => Some(true), | |
| 145 | + | Err(error) => refused(&error.to_string()).then_some(false), | |
| 146 | + | } | |
| 147 | + | } | |
| 148 | + | ||
| 136 | 149 | /// What AI Gateway priced a session's requests at, and how many there | |
| 137 | 150 | /// were, with the requests it had no price for. | |
| 138 | 151 | async fn session_cost(&self, session: &str) -> Result<SessionCost> { | |
| ⋯ | |||
| 200 | 213 | } | |
| 201 | 214 | } | |
| 202 | 215 | ||
| 216 | + | /// Whether an error from `send_with` is Cloudflare saying no to the token | |
| 217 | + | /// (401, 403) or that there is no such thing for it (404), rather than | |
| 218 | + | /// failing. | |
| 219 | + | pub(crate) fn refused(error: &str) -> bool { | |
| 220 | + | ["Cloudflare answered 401", "Cloudflare answered 403", "Cloudflare answered 404"].iter().any(|s| error.contains(s)) | |
| 221 | + | } | |
| 222 | + | ||
| 203 | 223 | /// A request to Cloudflare's API with a bearer token; anything but 200 is | |
| 204 | 224 | /// an error with what Cloudflare said. | |
| 205 | 225 | async fn send_with(token: &str, method: Method, url: &str, body: Option<Value>) -> Result<Value> { | |
| 44 | 44 | pub(crate) const OVERHEAD: [&str; 1] = ["platform"]; | |
| 45 | 45 | /// Buckets Cloudflare does not bill: their cost is g1t's own figure. | |
| 46 | 46 | pub(crate) const NOT_CLOUDFLARE: [&str; 1] = ["models"]; | |
| 47 | + | /// The key an agent's planning run is reconciled under. Its ledger task | |
| 48 | + | /// is `plan`, which is also the plan's payments' key (`revenue_map`: the | |
| 49 | + | /// platform bucket), so read as `plan` its model cost went to running g1t, | |
| 50 | + | /// where Cloudflare's bill is the cost, and was lost. No `revenue_map` | |
| 51 | + | /// row: models, like every agent run. | |
| 52 | + | pub(crate) const PLANNING_KEY: &str = "planning"; | |
| 47 | 53 | /// The days drift is judged over. | |
| 48 | 54 | const DRIFT_DAYS: u64 = 7; | |
| 49 | 55 | /// The days a workspace's cost is set against its revenue. | |
| ⋯ | |||
| 432 | 438 | out | |
| 433 | 439 | } | |
| 434 | 440 | ||
| 441 | + | /// A month-end meter's day on a 100%-discount workspace: all of it given | |
| 442 | + | /// (comped) and none of it money in. The snapshot holds what the month | |
| 443 | + | /// would charge before the discount, which the month's close takes off in | |
| 444 | + | /// full; counted as paid, flagon-io's cache, embeddings and scans read as | |
| 445 | + | /// money in ($0.0023 on 2026-10-08). | |
| 446 | + | pub(crate) fn comped_meter(mut u: UsageRow) -> UsageRow { | |
| 447 | + | u.given = Given { comped: u.value, ..Given::default() }; | |
| 448 | + | u.cash = 0; | |
| 449 | + | u | |
| 450 | + | } | |
| 451 | + | ||
| 435 | 452 | /// Margin as a share of what was charged, in percent; None when nothing was. | |
| 436 | 453 | pub(crate) fn margin_percent(revenue_micros: i64, cost_micros: i64) -> Option<f64> { | |
| 437 | 454 | (revenue_micros > 0).then(|| (revenue_micros - cost_micros) as f64 * 100.0 / revenue_micros as f64) | |
| ⋯ | |||
| 676 | 693 | ||
| 677 | 694 | /// The models drift's detail: the gateway's total against the ledger's, | |
| 678 | 695 | /// and any testing reset in the window. | |
| 679 | − | pub(crate) fn models_detail(drift: &Drift, caveats: &costs::GatewayCaveats, resets: &[ResetNote]) -> String { | |
| 696 | + | pub(crate) fn models_detail(drift: &Drift, caveats: &costs::GatewayCaveats, resets: &[ResetNote], read: &costs::GatewayRead) -> String { | |
| 680 | 697 | if drift.cloudflare <= 0.0 { | |
| 681 | − | return format!( | |
| 682 | − | "Models: the ledger's model cost is {} over the last {DRIFT_DAYS} days and AI Gateway priced nothing, so the two were not compared. Either the gateway's analytics cannot be seen (Cloudflare answers a token without AI Gateway: Read with no rows, not an error; billing reads them with CLOUDFLARE_USAGE_TOKEN, then CLOUDFLARE_BILLING_TOKEN), or model calls went around the gateway.", | |
| 698 | + | let head = format!( | |
| 699 | + | "Models: the ledger's model cost is {} over the last {DRIFT_DAYS} days and AI Gateway priced nothing, so the two were not compared.", | |
| 683 | 700 | dollars(drift.ours as i64) | |
| 684 | 701 | ); | |
| 702 | + | let why = if caveats.requests > 0.0 { | |
| 703 | + | format!( | |
| 704 | + | "The gateway logged {} requests in those days but put no price on them: it has no price for the models used{}. Add their prices to the gateway, or route those models where they are priced.", | |
| 705 | + | crate::features::thousands(caveats.requests.round() as u64), | |
| 706 | + | if caveats.unpriced.is_empty() { String::new() } else { format!(" ({})", caveats.unpriced.join(", ")) } | |
| 707 | + | ) | |
| 708 | + | } else { | |
| 709 | + | match read { | |
| 710 | + | costs::GatewayRead::Empty { visible: Some(false) } => "The token billing reads AI Gateway with (CLOUDFLARE_USAGE_TOKEN, else CLOUDFLARE_BILLING_TOKEN) cannot see the gateway named in AI_GATEWAY_ID: Cloudflare refused it (no Account, AI Gateway, Read on the token, or no gateway by that id). Give the token AI Gateway Read, or fix AI_GATEWAY_ID.".to_owned(), | |
| 711 | + | costs::GatewayRead::Empty { visible: Some(true) } => "The token can see the gateway and it logged no requests in those days: model calls went around it. Check that every caller of a hosted model uses the gateway's URL (services/models, the runner's ANTHROPIC_BASE_URL).".to_owned(), | |
| 712 | + | costs::GatewayRead::Failed(error) => format!("AI Gateway's analytics could not be read: {error}"), | |
| 713 | + | costs::GatewayRead::NotRead => "AI Gateway was not read on this run: no AI_GATEWAY_ID, or no CLOUDFLARE_USAGE_TOKEN or CLOUDFLARE_BILLING_TOKEN.".to_owned(), | |
| 714 | + | costs::GatewayRead::Rows | costs::GatewayRead::Empty { visible: None } => "The gateway answered without requests for these days, and whether its token can see the gateway could not be told: either the token lacks AI Gateway Read, or model calls went around the gateway.".to_owned(), | |
| 715 | + | } | |
| 716 | + | }; | |
| 717 | + | return format!("{head} {why}"); | |
| 685 | 718 | } | |
| 686 | 719 | let lower = drift.ours < drift.cloudflare; | |
| 687 | 720 | let wiped = resets.iter().any(|r| !r.recorded); | |
| ⋯ | |||
| 1016 | 1049 | /// day has come, and raise or clear alerts. | |
| 1017 | 1050 | pub(crate) async fn costs_daily(&self, env: &Env, keeper: &crate::keeper::Keeper) -> Result<CostsRun> { | |
| 1018 | 1051 | let mut run = CostsRun::default(); | |
| 1019 | − | let (since, until) = match self.read_cloudflare(keeper, &mut run.problems).await? { | |
| 1052 | + | let mut gateway = costs::GatewayRead::default(); | |
| 1053 | + | let (since, until) = match self.read_cloudflare(keeper, &mut run.problems, &mut gateway).await? { | |
| 1020 | 1054 | Some((since, until, lines)) => { | |
| 1021 | 1055 | run.lines = lines; | |
| 1022 | 1056 | (since, until) | |
| ⋯ | |||
| 1048 | 1082 | let window = day_before(&until, costs::BACKFILL_DAYS - 1); | |
| 1049 | 1083 | let reconcile_from = if window < since { window } else { since.clone() }; | |
| 1050 | 1084 | run.days = self.reconcile_range(&reconcile_from, &until).await?; | |
| 1051 | − | let drift = self.find_drift(&until).await?; | |
| 1085 | + | let drift = self.find_drift(&until, &gateway).await?; | |
| 1052 | 1086 | run.proposals = self.measure_units(&until).await?; | |
| 1053 | 1087 | self.apply_due_versions().await?; | |
| 1054 | 1088 | run.alerts = self.raise_alerts(env, &until, &drift).await?; | |
| ⋯ | |||
| 1102 | 1136 | .db | |
| 1103 | 1137 | .prepare(format!( | |
| 1104 | 1138 | "SELECT substr(created_at, 1, 10) AS day, workspace, | |
| 1105 | − | CASE WHEN task = 'deployments' AND reference LIKE 'deploy/%' THEN 'builds' ELSE COALESCE(task, 'other') END AS key, | |
| 1139 | + | CASE WHEN task = 'deployments' AND reference LIKE 'deploy/%' THEN 'builds' | |
| 1140 | + | WHEN task = 'plan' THEN '{planning}' ELSE COALESCE(task, 'other') END AS key, | |
| 1106 | 1141 | CASE WHEN workspace IN ({internal}) THEN 1 ELSE 0 END AS internal, | |
| 1107 | 1142 | CASE WHEN billed_to = 'workspace' THEN 1 ELSE 0 END AS own_provider, | |
| 1108 | 1143 | -SUM(amount_micros) AS cash, | |
| ⋯ | |||
| 1116 | 1151 | WHERE kind = 'usage' AND created_at >= ?1 AND created_at <= ?2 AND COALESCE(task, '') NOT IN ({charged_here}) | |
| 1117 | 1152 | AND (?3 = '' OR workspace = ?3) | |
| 1118 | 1153 | GROUP BY 1, 2, 3, 4, 5", | |
| 1119 | − | internal = crate::sales::INTERNAL_SQL | |
| 1154 | + | internal = crate::sales::INTERNAL_SQL, | |
| 1155 | + | planning = PLANNING_KEY | |
| 1120 | 1156 | )) | |
| 1121 | 1157 | .bind(&[since.into(), end.as_str().into(), only_sql.into()])? | |
| 1122 | 1158 | .all() | |
| ⋯ | |||
| 1177 | 1213 | .filter(|s| crate::storage::CHARGED_HERE.contains(&s.source.as_str()) || s.source == "domains") | |
| 1178 | 1214 | .map(|s| (s.day, s.workspace, s.source, s.cost_micros, s.charge_micros)) | |
| 1179 | 1215 | .collect::<Vec<_>>(); | |
| 1180 | − | out.extend(pending_deltas(&snaps).into_iter().filter(|u| u.day.as_str() >= since).map(|mut u| { | |
| 1181 | − | if internal.contains(&u.workspace) { | |
| 1182 | − | u.given = Given { comped: u.value, ..Given::default() }; | |
| 1183 | − | } | |
| 1184 | − | u | |
| 1185 | − | })); | |
| 1216 | + | out.extend( | |
| 1217 | + | pending_deltas(&snaps) | |
| 1218 | + | .into_iter() | |
| 1219 | + | .filter(|u| u.day.as_str() >= since) | |
| 1220 | + | .map(|u| if internal.contains(&u.workspace) { comped_meter(u) } else { u }), | |
| 1221 | + | ); | |
| 1186 | 1222 | // The plan's price, spread over the 30 days it pays for, so a month's | |
| 1187 | 1223 | // payment does not read as one very good day and 29 bad ones. | |
| 1188 | 1224 | #[derive(Deserialize)] | |
| ⋯ | |||
| 1450 | 1486 | ||
| 1451 | 1487 | /// Drift over the last week, written to `cost_drift` (replacing the | |
| 1452 | 1488 | /// last run's), with unmapped Cloudflare meters as leaks. | |
| 1453 | − | async fn find_drift(&self, until: &str) -> Result<Vec<(Drift, String)>> { | |
| 1489 | + | async fn find_drift(&self, until: &str, read: &costs::GatewayRead) -> Result<Vec<(Drift, String)>> { | |
| 1454 | 1490 | let since = day_before(until, DRIFT_DAYS - 1); | |
| 1455 | 1491 | let settings = self.cost_settings().await?; | |
| 1456 | 1492 | let rules = self.rules().await?; | |
| ⋯ | |||
| 1476 | 1512 | } | |
| 1477 | 1513 | let title = costs::bucket_title(bucket); | |
| 1478 | 1514 | let detail = match drift.kind { | |
| 1479 | − | DriftKind::Cost if NOT_CLOUDFLARE.contains(&bucket.as_str()) => models_detail(&drift, &caveats, &resets), | |
| 1515 | + | DriftKind::Cost if NOT_CLOUDFLARE.contains(&bucket.as_str()) => models_detail(&drift, &caveats, &resets, read), | |
| 1480 | 1516 | DriftKind::Count => format!( | |
| 1481 | 1517 | "{title}: g1t counted {}, Cloudflare {} over the last {DRIFT_DAYS} days ({:+.1}%). Customers are charged for what g1t counts; check what Cloudflare counts as a unit and change the repos service's operation_mapping (set_operation_mapping).", | |
| 1482 | 1518 | crate::features::thousands(drift.ours.max(0.0).round() as u64), | |
| ⋯ | |||
| 2246 | 2282 | } | |
| 2247 | 2283 | ||
| 2248 | 2284 | #[test] | |
| 2285 | + | fn a_planning_run_is_model_cost_not_running_g1t() { | |
| 2286 | + | // flagon-io's planning run on 2026-10-07 cost $0.0748 of model | |
| 2287 | + | // calls; read under the ledger's task, `plan`, it went to running | |
| 2288 | + | // g1t, where Cloudflare's bill is the cost, and the model cost was | |
| 2289 | + | // lost from the statement and the drift. | |
| 2290 | + | let usage = vec![ | |
| 2291 | + | usage("2026-10-07", "flagon-io", PLANNING_KEY, 89_741, 0, 74_784), | |
| 2292 | + | usage("2026-10-07", "acme", "plan", 666_666, 666_666, 0), | |
| 2293 | + | ]; | |
| 2294 | + | let (days, _) = fold(&rules(), &revenue_map(), &[], &[], &usage, &BTreeSet::new()); | |
| 2295 | + | let get = |bucket: &str| days.iter().find(|d| d.bucket == bucket).unwrap(); | |
| 2296 | + | assert_eq!((get("models").cost(), get("models").value_micros), (74_784, 89_741)); | |
| 2297 | + | assert_eq!((get("platform").own_cost_micros, get("platform").cash_micros), (0, 666_666)); | |
| 2298 | + | assert!(!revenue_map().contains_key(PLANNING_KEY)); | |
| 2299 | + | } | |
| 2300 | + | ||
| 2301 | + | #[test] | |
| 2302 | + | fn a_comped_workspaces_month_end_meters_are_given_never_money_in() { | |
| 2303 | + | let snap = |day: &str, cost: i64, charge: i64| (day.to_string(), "flagon-io".to_string(), "cache".to_string(), cost, charge); | |
| 2304 | + | let rows: Vec<UsageRow> = pending_deltas(&[snap("2026-10-07", 1, 2), snap("2026-10-08", 473, 568)]).into_iter().map(comped_meter).collect(); | |
| 2305 | + | assert_eq!(rows.iter().map(|r| (r.cash, r.value, r.given.comped)).collect::<Vec<_>>(), vec![(0, 2, 2), (0, 566, 566)]); | |
| 2306 | + | let internal: BTreeSet<String> = ["flagon-io".to_string()].into(); | |
| 2307 | + | let (days, workspaces) = fold(&rules(), &revenue_map(), &[], &[], &rows, &internal); | |
| 2308 | + | assert!(days.iter().all(|d| d.cash_micros == 0)); | |
| 2309 | + | assert!(workspaces.iter().all(|w| w.revenue == 0)); | |
| 2310 | + | } | |
| 2311 | + | ||
| 2312 | + | #[test] | |
| 2249 | 2313 | fn artifacts_events_count_when_the_bill_does_not() { | |
| 2250 | 2314 | let lines = vec![ | |
| 2251 | 2315 | line("2026-10-05", SOURCE_ARTIFACTS, "artifacts", "events_pull", 120.0, 0.0), | |
| ⋯ | |||
| 2477 | 2541 | // rows), so it is said. Under the minimum, or no model cost: nothing. | |
| 2478 | 2542 | let silent = drifts("models", &[day("models", 0, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000); | |
| 2479 | 2543 | assert_eq!(silent, vec![Drift { bucket: "models".into(), kind: DriftKind::Cost, ours: 1_000_000.0, cloudflare: 0.0, delta_percent: None }]); | |
| 2480 | − | let said = models_detail(&silent[0], &costs::GatewayCaveats::default(), &[]); | |
| 2481 | − | assert!(said.contains("$1.00") && said.contains("priced nothing") && said.contains("AI Gateway: Read"), "{said}"); | |
| 2544 | + | // Why it is empty, as far as the run could tell. | |
| 2545 | + | let why = |caveats: &costs::GatewayCaveats, read: costs::GatewayRead| models_detail(&silent[0], caveats, &[], &read); | |
| 2546 | + | let none = costs::GatewayCaveats::default(); | |
| 2547 | + | let said = why(&none, costs::GatewayRead::Empty { visible: Some(false) }); | |
| 2548 | + | assert!(said.contains("$1.00") && said.contains("priced nothing") && said.contains("cannot see the gateway") && said.contains("AI Gateway Read"), "{said}"); | |
| 2549 | + | let said = why(&none, costs::GatewayRead::Empty { visible: Some(true) }); | |
| 2550 | + | assert!(said.contains("logged no requests") && said.contains("went around it"), "{said}"); | |
| 2551 | + | let said = why(&none, costs::GatewayRead::Failed("Cloudflare answered 500".into())); | |
| 2552 | + | assert!(said.contains("could not be read: Cloudflare answered 500"), "{said}"); | |
| 2553 | + | assert!(why(&none, costs::GatewayRead::NotRead).contains("not read on this run")); | |
| 2554 | + | assert!(why(&none, costs::GatewayRead::Empty { visible: None }).contains("could not be told")); | |
| 2555 | + | // Requests with no price: neither the token nor a bypass. | |
| 2556 | + | let unpriced = costs::GatewayCaveats { requests: 42.0, unpriced: vec!["anthropic_claude_new_1".into()], ..Default::default() }; | |
| 2557 | + | let said = why(&unpriced, costs::GatewayRead::Rows); | |
| 2558 | + | assert!(said.contains("logged 42 requests") && said.contains("no price for the models used (anthropic_claude_new_1)"), "{said}"); | |
| 2482 | 2559 | assert!(drifts("models", &[day("models", 0, 50_000, 60_000, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); | |
| 2483 | 2560 | assert!(drifts("models", &[day("models", 0, 0, 0, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); | |
| 2484 | 2561 | // The detail says which way and why it may be off. | |
| 2485 | 2562 | let caveats = costs::GatewayCaveats { cache_read_tokens: 3_000_000.0, unpriced: vec!["anthropic_claude_new_1".into()], ..Default::default() }; | |
| 2486 | − | let detail = models_detail(&short[0], &caveats, &[]); | |
| 2563 | + | let detail = models_detail(&short[0], &caveats, &[], &costs::GatewayRead::default()); | |
| 2487 | 2564 | assert!(detail.contains("$5.00") && detail.contains("$3.00") && detail.contains("were not charged"), "{detail}"); | |
| 2488 | 2565 | assert!(detail.contains("3,000,000 prompt-cache read") && detail.contains("no price for anthropic_claude_new_1"), "{detail}"); | |
| 2489 | 2566 | } | |
| ⋯ | |||
| 2623 | 2700 | ] | |
| 2624 | 2701 | ); | |
| 2625 | 2702 | let drift = Drift { bucket: "models".into(), kind: DriftKind::Cost, ours: 2_490_000.0, cloudflare: 11_110_000.0, delta_percent: Some(-77.6) }; | |
| 2626 | − | let detail = models_detail(&drift, &costs::GatewayCaveats::default(), ¬es); | |
| 2703 | + | let detail = models_detail(&drift, &costs::GatewayCaveats::default(), ¬es, &costs::GatewayRead::default()); | |
| 2627 | 2704 | assert!(detail.contains("includes model usage wiped by a testing reset of syntaqx on 2026-10-07"), "{detail}"); | |
| 2628 | 2705 | assert!(detail.contains("not a leak") && detail.contains("leaves the 7 days on 2026-10-14"), "{detail}"); | |
| 2629 | 2706 | assert!(!detail.contains("a gap that stays is a leak"), "{detail}"); | |
| ⋯ | |||
| 2634 | 2711 | assert!(!wiped_not_leaked(&leak, ¬es[1..])); | |
| 2635 | 2712 | assert!(!wiped_not_leaked(&Drift { bucket: "actions_cache".into(), ..leak }, ¬es)); | |
| 2636 | 2713 | // No reset: the detail is as before. | |
| 2637 | − | assert!(models_detail(&drift, &costs::GatewayCaveats::default(), &[]).contains("a gap that stays is a leak")); | |
| 2714 | + | assert!(models_detail(&drift, &costs::GatewayCaveats::default(), &[], &costs::GatewayRead::default()).contains("a gap that stays is a leak")); | |
| 2638 | 2715 | } | |
| 2639 | 2716 | } | |