| 495 | 495 | | pub(crate) enum DriftKind { |
| 496 | 496 | | /// g1t counted a different number of units than Cloudflare did. |
| 497 | 497 | | Count, |
| 498 | | − | /// What Cloudflare charged differs from what the price book says the |
| 499 | | − | /// same usage cost. |
| 498 | + | /// The price book's cost of a bucket's usage differs from what the |
| 499 | + | /// same usage comes to at Cloudflare's list prices, before the included |
| 500 | + | /// amounts (a price may be stale); on `models`, the ledger's model cost |
| 501 | + | /// differs from what AI Gateway priced the same traffic at. |
| 500 | 502 | | Cost, |
| 501 | 503 | | /// Cloudflare charged for something nothing charges customers for. |
| 502 | 504 | | Leak, |
| ⋯ |
| 526 | 528 | | } |
| 527 | 529 | | |
| 528 | 530 | | /// Drift over a window for one bucket: counts more than `threshold` |
| 529 | | − | /// percent apart, a bill that far from the price book's cost of the same |
| 530 | | − | /// usage, and cost with nothing charged for it. Under `min_cost_micros` |
| 531 | | − | /// in all, cost says nothing. |
| 532 | | − | pub(crate) fn drifts(bucket: &str, days: &[ProductDay], threshold: f64, counted: bool, min_cost_micros: i64) -> Vec<Drift> { |
| 531 | + | /// percent apart, the price book's cost of the usage that far from what the |
| 532 | + | /// same usage comes to at Cloudflare's list prices (`list_micros`, from |
| 533 | + | /// `list_costs`), and cost with nothing charged for it. Under |
| 534 | + | /// `min_cost_micros` in all, cost says nothing. |
| 535 | + | /// |
| 536 | + | /// The price book is set against the list cost, never against what |
| 537 | + | /// Cloudflare billed: the bill is net of the included amounts and the price |
| 538 | + | /// book's cost is of every unit, so a bucket whose usage mostly fits in |
| 539 | + | /// them would read as a stale price when nothing changed. Without a list |
| 540 | + | /// cost (a meter in the bucket with no list price) the price book is not |
| 541 | + | /// checked. A leak is still what Cloudflare billed: money spent. |
| 542 | + | pub(crate) fn drifts(bucket: &str, days: &[ProductDay], threshold: f64, counted: bool, min_cost_micros: i64, list_micros: Option<f64>) -> Vec<Drift> { |
| 533 | 543 | | let overhead = OVERHEAD.contains(&bucket); |
| 534 | 544 | | let sum = |f: &dyn Fn(&ProductDay) -> f64| days.iter().map(f).sum::<f64>(); |
| 535 | 545 | | let cf_cost = sum(&|d| d.cf_cost_micros as f64); |
| ⋯ |
| 548 | 558 | | out.push(Drift { bucket: bucket.into(), kind: DriftKind::Count, ours: own_quantity, cloudflare: cf_quantity, delta_percent: delta }); |
| 549 | 559 | | } |
| 550 | 560 | | } |
| 551 | | − | let enough = cf_cost.max(own_cost) >= min_cost_micros as f64; |
| 552 | 561 | | // Models: what AI Gateway priced g1t's own provider traffic at (its |
| 553 | 562 | | // lines, as "Cloudflare's" side) against the ledger's model cost. Only |
| 554 | 563 | | // once the gateway has been read; then the ledger having none of it is |
| 555 | | − | // drift too (traffic no run was charged for). |
| 556 | | − | let models = NOT_CLOUDFLARE.contains(&bucket) && cf_cost > 0.0; |
| 557 | | − | if enough && !overhead && cf_cost > 0.0 && (own_cost > 0.0 || models) { |
| 558 | | − | let delta = delta_percent(own_cost, cf_cost); |
| 564 | + | // drift too (traffic no run was charged for). Every other bucket: its |
| 565 | + | // usage at Cloudflare's list prices, before the included amounts. |
| 566 | + | let not_cloudflare = NOT_CLOUDFLARE.contains(&bucket); |
| 567 | + | let theirs = if not_cloudflare { cf_cost } else { list_micros.unwrap_or(0.0) }; |
| 568 | + | let enough = theirs.max(own_cost) >= min_cost_micros as f64; |
| 569 | + | let models = not_cloudflare && cf_cost > 0.0; |
| 570 | + | if enough && !overhead && theirs > 0.0 && (own_cost > 0.0 || models) { |
| 571 | + | let delta = delta_percent(own_cost, theirs); |
| 559 | 572 | | if delta.is_some_and(|d| d.abs() > threshold) { |
| 560 | | − | out.push(Drift { bucket: bucket.into(), kind: DriftKind::Cost, ours: own_cost, cloudflare: cf_cost, delta_percent: delta }); |
| 573 | + | out.push(Drift { bucket: bucket.into(), kind: DriftKind::Cost, ours: own_cost, cloudflare: theirs, delta_percent: delta }); |
| 561 | 574 | | } |
| 562 | 575 | | } |
| 563 | 576 | | // The ledger has model cost and the gateway priced none of it: a token |
| ⋯ |
| 572 | 585 | | out |
| 573 | 586 | | } |
| 574 | 587 | | |
| 588 | + | /// What each bucket's billable usage over a window comes to at |
| 589 | + | /// Cloudflare's list prices, before the included amounts (`cycle::list_cost` |
| 590 | + | /// line by line, in micros): what the price book's cost of the same usage |
| 591 | + | /// is checked against. None for a bucket with usage on a meter that has no |
| 592 | + | /// list price: its usage cannot be priced like for like, so it is not |
| 593 | + | /// checked. Other sources (Artifacts events, AI Gateway) are left out. |
| 594 | + | pub(crate) fn list_costs(rules: &[Rule], lines: &[LineRow]) -> BTreeMap<String, Option<f64>> { |
| 595 | + | let mut out: BTreeMap<String, Option<f64>> = BTreeMap::new(); |
| 596 | + | for line in lines.iter().filter(|l| l.source == SOURCE_BILLABLE) { |
| 597 | + | let bucket = costs::classify(rules, &line.product, &line.meter).map_or(UNMAPPED, |r| r.bucket.as_str()); |
| 598 | + | let entry = out.entry(bucket.to_owned()).or_insert(Some(0.0)); |
| 599 | + | if line.quantity <= 0.0 { |
| 600 | + | continue; |
| 601 | + | } |
| 602 | + | *entry = match (*entry, crate::cycle::list_cost(&line.product, &line.meter, line.quantity)) { |
| 603 | + | (Some(sum), Some(cost)) => Some(sum + cost * 1_000_000.0), |
| 604 | + | _ => None, |
| 605 | + | }; |
| 606 | + | } |
| 607 | + | out |
| 608 | + | } |
| 609 | + | |
| 575 | 610 | | /// What can make AI Gateway's cost differ from what the providers bill, |
| 576 | 611 | | /// said for staff: cache tokens (priced by the gateway at its own rates for |
| 577 | 612 | | /// them, which may lag the provider's), requests Cloudflare billed itself, |
| ⋯ |
| 893 | 928 | | out |
| 894 | 929 | | } |
| 895 | 930 | | |
| 896 | | − | /// Cloudflare's marginal rate for one of its units: the median over the |
| 897 | | − | /// charged days of cost over quantity, in dollars. None while the included |
| 898 | | − | /// amounts still cover it. Each item is a day's (quantity, cost). |
| 931 | + | /// Cloudflare's rate for one of its units: the median over the costed |
| 932 | + | /// days of cost over quantity, in dollars. None while nothing is costed. |
| 933 | + | /// Each item is a day's (quantity, cost), from `rate_line`. |
| 899 | 934 | | pub(crate) fn billed_rate(days: &[(f64, f64)]) -> Option<f64> { |
| 900 | 935 | | let mut rates: Vec<f64> = days.iter().filter(|(q, c)| *q > 0.0 && *c > 0.0).map(|(q, c)| c / q).collect(); |
| 901 | 936 | | if rates.is_empty() { |
| ⋯ |
| 905 | 940 | | Some(rates[rates.len() / 2]) |
| 906 | 941 | | } |
| 907 | 942 | | |
| 943 | + | /// A line's (quantity, cost) for `billed_rate`: at its list price for all |
| 944 | + | /// of the quantity where the meter has one, never what the cycle billed, |
| 945 | + | /// which is net of the included amounts (nothing until the cycle passes |
| 946 | + | /// them, part of a day's usage on the day it does, then whole millions) and |
| 947 | + | /// so says nothing about the price of each unit. A meter with no list |
| 948 | + | /// price has only what Cloudflare billed; the median over the charged days |
| 949 | + | /// leaves out the day its included amount ran out. |
| 950 | + | pub(crate) fn rate_line(product: &str, meter: &str, quantity: f64, cost_usd: f64) -> (f64, f64) { |
| 951 | + | (quantity, crate::cycle::list_cost(product, meter, quantity).unwrap_or(cost_usd)) |
| 952 | + | } |
| 953 | + | |
| 908 | 954 | | /// What one of g1t's units costs, from Cloudflare's rate per its own unit |
| 909 | 955 | | /// and how many of Cloudflare's units each of g1t's took: if Cloudflare |
| 910 | 956 | | /// counts three operations for every git operation g1t counts, a git |
| ⋯ |
| 1568 | 1614 | | } |
| 1569 | 1615 | | let caveats = self.gateway_caveats(&since, until).await?; |
| 1570 | 1616 | | let resets = self.resets_since(&since, until).await?; |
| 1617 | + | // The same days' usage at list prices, before the included amounts: |
| 1618 | + | // what the price book is checked against (never the bill, which is |
| 1619 | + | // net of them). |
| 1620 | + | let lines = self |
| 1621 | + | .db |
| 1622 | + | .prepare("SELECT day, source, product, meter, quantity, cost_usd FROM cost_lines WHERE source = ?1 AND day >= ?2 AND day <= ?3") |
| 1623 | + | .bind(&[SOURCE_BILLABLE.into(), since.as_str().into(), until.into()])? |
| 1624 | + | .all() |
| 1625 | + | .await? |
| 1626 | + | .results::<LineRow>()?; |
| 1627 | + | let list = list_costs(&rules, &lines); |
| 1571 | 1628 | | let mut found = Vec::new(); |
| 1572 | 1629 | | if let Some(drift) = unpriced_drift(&caveats) { |
| 1573 | 1630 | | found.push(drift); |
| ⋯ |
| 1577 | 1634 | | let threshold = bucket_rules.iter().map(|r| r.drift_percent).fold(f64::INFINITY, f64::min); |
| 1578 | 1635 | | let threshold = if threshold.is_finite() { threshold } else { 10.0 }; |
| 1579 | 1636 | | let counted = bucket_rules.iter().any(|r| r.own_meter.is_some()); |
| 1580 | | − | for drift in drifts(bucket, days, threshold, counted, settings.min_daily_cost_micros) { |
| 1637 | + | let list_micros = list.get(bucket).copied().flatten(); |
| 1638 | + | for drift in drifts(bucket, days, threshold, counted, settings.min_daily_cost_micros, list_micros) { |
| 1581 | 1639 | | if wiped_not_leaked(&drift, &resets) { |
| 1582 | 1640 | | continue; |
| 1583 | 1641 | | } |
| ⋯ |
| 1591 | 1649 | | drift.delta_percent.unwrap_or(0.0) |
| 1592 | 1650 | | ), |
| 1593 | 1651 | | DriftKind::Cost => format!( |
| 1594 | | − | "{title}: Cloudflare charged {} over the last {DRIFT_DAYS} days; the price book's cost of the same usage is {} ({:+.1}%). A price may be stale: see the proposals.", |
| 1652 | + | "{title}: the last {DRIFT_DAYS} days' usage comes to {} at Cloudflare's list prices, before the included amounts; the price book's cost of the same usage is {} ({:+.1}%). A price may be stale: see the proposals.", |
| 1595 | 1653 | | dollars(drift.cloudflare as i64), |
| 1596 | 1654 | | dollars(drift.ours as i64), |
| 1597 | 1655 | | drift.delta_percent.unwrap_or(0.0) |
| ⋯ |
| 1676 | 1734 | | let mine: Vec<(f64, f64)> = lines |
| 1677 | 1735 | | .iter() |
| 1678 | 1736 | | .filter(|l| costs::classify(&rules, &l.product, &l.meter).is_some_and(|r| r.product == s.product && r.meter == s.meter)) |
| 1679 | | − | .map(|l| (l.quantity, l.cost_usd)) |
| 1737 | + | .map(|l| rate_line(&l.product, &l.meter, l.quantity, l.cost_usd)) |
| 1680 | 1738 | | .collect(); |
| 1681 | 1739 | | let Some(rate) = billed_rate(&mine) else { continue }; |
| 1682 | 1740 | | let cf_units: f64 = mine.iter().map(|(q, _)| q).sum(); |
| ⋯ |
| 2584 | 2642 | | fn counts_more_than_the_threshold_apart_are_drift() { |
| 2585 | 2643 | | // Cloudflare counted 30,000 operations where g1t counted 10,000: |
| 2586 | 2644 | | // binding reads, perhaps. -66.7%. |
| 2587 | | − | let drift = drifts("git", &[day("git", 3_000_000, 1_500_000, 1_800_000, 30_000.0, 10_000.0)], 10.0, true, 100_000); |
| 2645 | + | let drift = drifts("git", &[day("git", 3_000_000, 1_500_000, 1_800_000, 30_000.0, 10_000.0)], 10.0, true, 100_000, Some(3_000_000.0)); |
| 2588 | 2646 | | assert_eq!(drift.iter().map(|d| d.kind).collect::<Vec<_>>(), vec![DriftKind::Count, DriftKind::Cost]); |
| 2589 | 2647 | | assert!((drift[0].delta_percent.unwrap() + 66.666).abs() < 0.01); |
| 2590 | 2648 | | // 9% apart: within 10%. |
| 2591 | | − | assert!(drifts("git", &[day("git", 1_000_000, 1_000_000, 1_200_000, 10_000.0, 10_900.0)], 10.0, true, 100_000).is_empty()); |
| 2649 | + | assert!(drifts("git", &[day("git", 1_000_000, 1_000_000, 1_200_000, 10_000.0, 10_900.0)], 10.0, true, 100_000, Some(1_000_000.0)).is_empty()); |
| 2592 | 2650 | | // Uncounted products have no count drift. |
| 2593 | | − | assert!(drifts("sandboxes", &[day("sandboxes", 1_000_000, 1_050_000, 1_200_000, 5.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2651 | + | assert!(drifts("sandboxes", &[day("sandboxes", 1_000_000, 1_050_000, 1_200_000, 5.0, 0.0)], 10.0, false, 100_000, Some(1_000_000.0)).is_empty()); |
| 2594 | 2652 | | } |
| 2595 | 2653 | | |
| 2596 | 2654 | | #[test] |
| 2597 | 2655 | | fn cost_with_no_revenue_is_a_leak_but_not_for_running_g1t() { |
| 2598 | | − | let leak = drifts("actions_cache", &[day("actions_cache", 400_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000); |
| 2656 | + | let leak = drifts("actions_cache", &[day("actions_cache", 400_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None); |
| 2599 | 2657 | | assert_eq!(leak.len(), 1); |
| 2600 | 2658 | | assert_eq!(leak[0].kind, DriftKind::Leak); |
| 2601 | | − | assert!(drifts("platform", &[day("platform", 5_000_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2659 | + | assert!(drifts("platform", &[day("platform", 5_000_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2602 | 2660 | | // Pennies say nothing. |
| 2603 | | − | assert!(drifts("actions_cache", &[day("actions_cache", 50_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2604 | | − | assert!(drifts(UNMAPPED, &[day(UNMAPPED, 250_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000)[0].kind == DriftKind::Leak); |
| 2661 | + | assert!(drifts("actions_cache", &[day("actions_cache", 50_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2662 | + | assert!(drifts(UNMAPPED, &[day(UNMAPPED, 250_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None)[0].kind == DriftKind::Leak); |
| 2663 | + | } |
| 2664 | + | |
| 2665 | + | /// A week of sandboxes mostly inside the cycle's included amounts: |
| 2666 | + | /// Cloudflare billed $0.0922 (the memory past 25 GiB-hours), and the |
| 2667 | + | /// same usage is $1.70 at list prices. |
| 2668 | + | fn sandbox_week() -> (Vec<Rule>, Vec<LineRow>) { |
| 2669 | + | let mut rules = rules(); |
| 2670 | + | rules.push(rule("durable_objects", "durable_objects_compute_duration", "sandboxes", None)); |
| 2671 | + | let lines = vec![ |
| 2672 | + | line("2026-10-07", SOURCE_BILLABLE, "containers", "container_memory_per_gib_second", 126_870.0, 0.092_175), |
| 2673 | + | line("2026-10-07", SOURCE_BILLABLE, "containers", "container_vcpu", 12_000.0, 0.0), |
| 2674 | + | line("2026-10-07", SOURCE_BILLABLE, "containers", "container_disk_per_gb_second", 253_740.0, 0.0), |
| 2675 | + | line("2026-10-07", SOURCE_BILLABLE, "durable_objects", "durable_objects_compute_duration", 90_000.0, 0.0), |
| 2676 | + | // Not billable usage: no part of the list cost. |
| 2677 | + | line("2026-10-07", SOURCE_ARTIFACTS, "artifacts", "events_push", 40.0, 0.0), |
| 2678 | + | ]; |
| 2679 | + | (rules, lines) |
| 2680 | + | } |
| 2681 | + | |
| 2682 | + | #[test] |
| 2683 | + | fn usage_inside_the_included_amounts_is_not_a_stale_price() { |
| 2684 | + | let (rules, lines) = sandbox_week(); |
| 2685 | + | let list = list_costs(&rules, &lines); |
| 2686 | + | let sandboxes = list["sandboxes"].unwrap(); |
| 2687 | + | assert!((sandboxes - 1_699_936.8).abs() < 1.0, "{sandboxes}"); |
| 2688 | + | // The price book's cost of the same usage is $1.69: within 1% of |
| 2689 | + | // the list, though Cloudflare billed $0.0922 after the included |
| 2690 | + | // amounts. Before, that read as +1733.6%. |
| 2691 | + | let week = day("sandboxes", 92_175, 1_690_000, 2_028_000, 0.0, 0.0); |
| 2692 | + | assert!(drifts("sandboxes", std::slice::from_ref(&week), 10.0, false, 100_000, Some(sandboxes)).is_empty()); |
| 2693 | + | let net = drifts("sandboxes", &[week], 10.0, false, 100_000, Some(92_175.0)); |
| 2694 | + | assert_eq!(net[0].kind, DriftKind::Cost, "billed against the price book was the false alarm"); |
| 2695 | + | } |
| 2696 | + | |
| 2697 | + | #[test] |
| 2698 | + | fn a_stale_price_is_still_drift() { |
| 2699 | + | let (rules, lines) = sandbox_week(); |
| 2700 | + | let sandboxes = list_costs(&rules, &lines)["sandboxes"].unwrap(); |
| 2701 | + | // The price book still costs the same usage at $1.20: a list price |
| 2702 | + | // rose and the book did not follow. |
| 2703 | + | let found = drifts("sandboxes", &[day("sandboxes", 92_175, 1_200_000, 1_440_000, 0.0, 0.0)], 10.0, false, 100_000, Some(sandboxes)); |
| 2704 | + | assert_eq!(found.len(), 1); |
| 2705 | + | assert_eq!((found[0].kind, found[0].ours, found[0].cloudflare.round()), (DriftKind::Cost, 1_200_000.0, 1_699_937.0)); |
| 2706 | + | assert!((found[0].delta_percent.unwrap() + 29.4).abs() < 0.1); |
| 2707 | + | // Nothing billed at all (the whole week inside the included |
| 2708 | + | // amounts) is checked all the same. |
| 2709 | + | assert_eq!(drifts("sandboxes", &[day("sandboxes", 0, 1_200_000, 1_440_000, 0.0, 0.0)], 10.0, false, 100_000, Some(sandboxes)).len(), 1); |
| 2710 | + | } |
| 2711 | + | |
| 2712 | + | #[test] |
| 2713 | + | fn a_bucket_with_a_meter_with_no_list_price_is_not_checked() { |
| 2714 | + | let (rules, mut lines) = sandbox_week(); |
| 2715 | + | lines.push(line("2026-10-07", SOURCE_BILLABLE, "artifacts", "storage", 3.0, 0.4)); |
| 2716 | + | lines.push(line("2026-10-07", SOURCE_BILLABLE, "artifacts", "operations", 0.0, 0.0)); |
| 2717 | + | let list = list_costs(&rules, &lines); |
| 2718 | + | assert_eq!(list["git"], None); |
| 2719 | + | assert!(list["sandboxes"].is_some()); |
| 2720 | + | // No list cost: no cost drift, but a leak is still what was billed. |
| 2721 | + | assert!(drifts("git", &[day("git", 3_000_000, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2722 | + | assert_eq!(drifts("git", &[day("git", 3_000_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None)[0].kind, DriftKind::Leak); |
| 2723 | + | } |
| 2724 | + | |
| 2725 | + | #[test] |
| 2726 | + | fn a_unit_rate_is_the_list_price_where_there_is_one() { |
| 2727 | + | // Billed $0.20 for 11.16M CPU ms (9.16M past the included 30M, in |
| 2728 | + | // whole millions): $0.02 a million at list, whatever was billed. |
| 2729 | + | let (q, c) = rate_line("workers", "workers_cpu_ms", 11_160_000.0, 0.20); |
| 2730 | + | assert!((billed_rate(&[(q, c)]).unwrap() * 1e6 - 0.02).abs() < 1e-12); |
| 2731 | + | // No list price: what Cloudflare billed. |
| 2732 | + | assert_eq!(rate_line("artifacts", "operations", 1_000.0, 0.5), (1_000.0, 0.5)); |
| 2605 | 2733 | | } |
| 2606 | 2734 | | |
| 2607 | 2735 | | #[test] |
| ⋯ |
| 2636 | 2764 | | let on = |day: &str, cf: f64, own: f64| ProductDay { day: day.into(), bucket: "git".into(), cf_quantity: cf, own_quantity: own, ..ProductDay::default() }; |
| 2637 | 2765 | | // Five days of Cloudflare's count before g1t's meter, then two that match. |
| 2638 | 2766 | | let days = vec![on("2026-10-01", 500.0, 0.0), on("2026-10-05", 300.0, 0.0), on("2026-10-06", 210.0, 231.0), on("2026-10-07", 450.0, 458.0)]; |
| 2639 | | − | assert!(drifts("git", &days, 10.0, true, 0).iter().all(|d| d.kind != DriftKind::Count)); |
| 2767 | + | assert!(drifts("git", &days, 10.0, true, 0, None).iter().all(|d| d.kind != DriftKind::Count)); |
| 2640 | 2768 | | // A real gap on the days both counted still shows. |
| 2641 | 2769 | | let days = vec![on("2026-10-01", 500.0, 0.0), on("2026-10-06", 400.0, 231.0), on("2026-10-07", 600.0, 300.0)]; |
| 2642 | | − | let found = drifts("git", &days, 10.0, true, 0); |
| 2770 | + | let found = drifts("git", &days, 10.0, true, 0, None); |
| 2643 | 2771 | | let count = found.iter().find(|d| d.kind == DriftKind::Count).unwrap(); |
| 2644 | 2772 | | assert_eq!((count.ours, count.cloudflare), (531.0, 1000.0)); |
| 2645 | 2773 | | // A meter that never counted is compared over every day. |
| 2646 | 2774 | | let days = vec![on("2026-10-06", 400.0, 0.0)]; |
| 2647 | | − | assert!(drifts("git", &days, 10.0, true, 0).iter().any(|d| d.kind == DriftKind::Count)); |
| 2775 | + | assert!(drifts("git", &days, 10.0, true, 0, None).iter().any(|d| d.kind == DriftKind::Count)); |
| 2648 | 2776 | | } |
| 2649 | 2777 | | |
| 2650 | 2778 | | #[test] |
| ⋯ |
| 2752 | 2880 | | #[test] |
| 2753 | 2881 | | fn the_gateways_total_against_the_ledgers_model_cost_is_drift() { |
| 2754 | 2882 | | // The gateway priced $5 of g1t's own traffic; the ledger has $3. |
| 2755 | | − | let short = drifts("models", &[day("models", 5_000_000, 3_000_000, 3_600_000, 0.0, 0.0)], 10.0, false, 100_000); |
| 2883 | + | let short = drifts("models", &[day("models", 5_000_000, 3_000_000, 3_600_000, 0.0, 0.0)], 10.0, false, 100_000, None); |
| 2756 | 2884 | | assert_eq!(short.iter().map(|d| d.kind).collect::<Vec<_>>(), vec![DriftKind::Cost]); |
| 2757 | 2885 | | assert!((short[0].delta_percent.unwrap() + 40.0).abs() < 1e-9); |
| 2758 | 2886 | | // Gateway traffic with nothing on the ledger at all: cost drift and a leak. |
| 2759 | | − | let none = drifts("models", &[day("models", 2_000_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000); |
| 2887 | + | let none = drifts("models", &[day("models", 2_000_000, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None); |
| 2760 | 2888 | | assert_eq!(none.iter().map(|d| d.kind).collect::<Vec<_>>(), vec![DriftKind::Cost, DriftKind::Leak]); |
| 2761 | 2889 | | // Within the threshold: nothing. |
| 2762 | | − | assert!(drifts("models", &[day("models", 1_050_000, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2890 | + | assert!(drifts("models", &[day("models", 1_050_000, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2763 | 2891 | | // The gateway priced nothing against a ledger that has model cost: |
| 2764 | 2892 | | // not agreement (a token that cannot see AI Gateway reads as no |
| 2765 | 2893 | | // rows), so it is said. Under the minimum, or no model cost: nothing. |
| 2766 | | − | let silent = drifts("models", &[day("models", 0, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000); |
| 2894 | + | let silent = drifts("models", &[day("models", 0, 1_000_000, 1_200_000, 0.0, 0.0)], 10.0, false, 100_000, None); |
| 2767 | 2895 | | assert_eq!(silent, vec![Drift { bucket: "models".into(), kind: DriftKind::Cost, ours: 1_000_000.0, cloudflare: 0.0, delta_percent: None }]); |
| 2768 | 2896 | | // Why it is empty, as far as the run could tell. |
| 2769 | 2897 | | let why = |caveats: &costs::GatewayCaveats, read: costs::GatewayRead| models_detail(&silent[0], caveats, &[], &read); |
| ⋯ |
| 2780 | 2908 | | let unpriced = costs::GatewayCaveats { requests: 42.0, unpriced: vec!["anthropic_claude_new_1".into()], ..Default::default() }; |
| 2781 | 2909 | | let said = why(&unpriced, costs::GatewayRead::Rows); |
| 2782 | 2910 | | assert!(said.contains("logged 42 requests") && said.contains("no price for the models used (anthropic_claude_new_1)"), "{said}"); |
| 2783 | | − | assert!(drifts("models", &[day("models", 0, 50_000, 60_000, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2784 | | − | assert!(drifts("models", &[day("models", 0, 0, 0, 0.0, 0.0)], 10.0, false, 100_000).is_empty()); |
| 2911 | + | assert!(drifts("models", &[day("models", 0, 50_000, 60_000, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2912 | + | assert!(drifts("models", &[day("models", 0, 0, 0, 0.0, 0.0)], 10.0, false, 100_000, None).is_empty()); |
| 2785 | 2913 | | // The detail says which way and why it may be off. |
| 2786 | 2914 | | let caveats = costs::GatewayCaveats { cache_read_tokens: 3_000_000.0, unpriced: vec!["anthropic_claude_new_1".into()], ..Default::default() }; |
| 2787 | 2915 | | let detail = models_detail(&short[0], &caveats, &[], &costs::GatewayRead::default()); |
| ⋯ |
| 2851 | 2979 | | let models = |days: &[ProductDay]| days.iter().find(|d| d.bucket == "models").cloned().unwrap(); |
| 2852 | 2980 | | // Without it: AI Gateway's $11.11 against the ledger's $2.49. |
| 2853 | 2981 | | let (days, _) = gateway_and_ledger(false); |
| 2854 | | − | let drift = drifts("models", &[models(&days)], 10.0, false, 100_000); |
| 2982 | + | let drift = drifts("models", &[models(&days)], 10.0, false, 100_000, None); |
| 2855 | 2983 | | assert_eq!(drift.iter().map(|d| d.kind).collect::<Vec<_>>(), vec![DriftKind::Cost]); |
| 2856 | 2984 | | assert_eq!((drift[0].ours, drift[0].cloudflare), (2_490_000.0, 11_110_000.0)); |
| 2857 | 2985 | | // With it: the ledger's model cost and the reset's add up to the gateway's. |
| 2858 | 2986 | | let (days, _) = gateway_and_ledger(true); |
| 2859 | 2987 | | let m = models(&days); |
| 2860 | 2988 | | assert_eq!(m.own_cost_micros, 11_110_000); |
| 2861 | | − | assert!(drifts("models", &[m], 10.0, false, 100_000).is_empty()); |
| 2989 | + | assert!(drifts("models", &[m], 10.0, false, 100_000, None).is_empty()); |
| 2862 | 2990 | | // The reset's own row makes no bucket of its own. |
| 2863 | 2991 | | assert!(!days.iter().any(|d| d.bucket.is_empty())); |
| 2864 | 2992 | | } |
