| 1 | //! What Cloudflare charges g1t, day by day and product by product. |
| 2 | //! |
| 3 | //! Prices are what g1t pays plus 20%, so g1t has to know what it pays, as |
| 4 | //! Cloudflare counts it, not as g1t assumes it. Some of what g1t runs on |
| 5 | //! is in beta (Artifacts bills "operations" from 2026-10-14 without saying |
| 6 | //! exactly which calls are one), so nothing here assumes a product list: |
| 7 | //! every line Cloudflare bills is kept, whatever it is, and how a line |
| 8 | //! maps to what g1t sells is data (`cost_map`), changed without a deploy. |
| 9 | //! |
| 10 | //! Once a day (`keeper::DAILY`) billing reads: |
| 11 | //! |
| 12 | //! - **Billable usage** (`GET /accounts/{id}/billable-usage`, FOCUS |
| 13 | //! columns): one row per service per day, with its quantity and what it |
| 14 | //! cost g1t. Every Cloudflare product g1t uses appears here once it is |
| 15 | //! used: Workers, Workers for Platforms, D1, KV, R2, Queues, Containers, |
| 16 | //! Durable Objects, Artifacts, Browser Rendering, Workers AI, Vectorize, |
| 17 | //! Cloudflare for SaaS and Email. |
| 18 | //! - **Artifacts events** (GraphQL `artifactsEventsAdaptiveGroups`), by |
| 19 | //! event type and day: what Artifacts itself counted, to compare with |
| 20 | //! what g1t counted (`margin`). |
| 21 | //! |
| 22 | //! Each becomes cost lines in `cost_lines`, one per (day, source, |
| 23 | //! product, meter), upserted, so reading a day again replaces it. The bill |
| 24 | //! is read over whole billing cycles, every page of it, and priced over |
| 25 | //! each cycle (`cycle`): its included amounts are the account's, once a |
| 26 | //! cycle. The Artifacts events: the first run reads the last 31 days; after |
| 27 | //! that the last few, since Cloudflare restates recent days as usage |
| 28 | //! settles. |
| 29 | //! |
| 30 | //! The token is `CLOUDFLARE_BILLING_TOKEN` (Account: Billing Read and |
| 31 | //! Account Analytics Read), or the keeper's `CLOUDFLARE_USAGE_TOKEN`, |
| 32 | //! which has both. Without either, nothing is read and nothing fails. |
| 33 | |
| 34 | use std::collections::{BTreeMap, BTreeSet}; |
| 35 | |
| 36 | use g1t_contracts::repos::{GitOperationsArgs, WorkspaceGitOperations}; |
| 37 | use g1t_contracts::time::rfc3339; |
| 38 | use g1t_kit::now_ms; |
| 39 | use serde::Deserialize; |
| 40 | use serde_json::{Value, json}; |
| 41 | use worker::Result; |
| 42 | |
| 43 | use crate::Billing; |
| 44 | use crate::keeper::Keeper; |
| 45 | |
| 46 | /// Where a cost line came from. |
| 47 | pub(crate) const SOURCE_BILLABLE: &str = "billable_usage"; |
| 48 | pub(crate) const SOURCE_ARTIFACTS: &str = "artifacts_events"; |
| 49 | |
| 50 | /// How far back the first run reads: the 31 days GraphQL keeps. |
| 51 | pub(crate) const BACKFILL_DAYS: u64 = 31; |
| 52 | /// How many recent days every later run reads again. |
| 53 | pub(crate) const RESTATE_DAYS: u64 = 4; |
| 54 | |
| 55 | pub(crate) const DAY_MS: u64 = 24 * 60 * 60 * 1000; |
| 56 | |
| 57 | /// One day of one meter of one Cloudflare product. |
| 58 | #[derive(Clone, Debug, Default, PartialEq)] |
| 59 | pub(crate) struct CostLine { |
| 60 | /// YYYY-MM-DD, UTC. |
| 61 | pub day: String, |
| 62 | pub source: &'static str, |
| 63 | /// `containers`, `workers`, `workers_kv`, `artifacts`, … from |
| 64 | /// Cloudflare's own family name. |
| 65 | pub product: String, |
| 66 | /// The service within it, such as `container_memory_per_gib_second` |
| 67 | /// or `events_push`. |
| 68 | pub meter: String, |
| 69 | pub unit: String, |
| 70 | pub quantity: f64, |
| 71 | /// What g1t pays, in dollars: for billable usage, set over the billing |
| 72 | /// cycle by `cycle::price_lines` (Cloudflare's own cost where its lines |
| 73 | /// carry one, else the list price past the included amount). |
| 74 | pub cost_usd: f64, |
| 75 | /// The name as Cloudflare gave it, for people. |
| 76 | pub raw_name: String, |
| 77 | /// What Cloudflare's line itself said it cost (contracted, billed or |
| 78 | /// effective; never list, which is before the included amounts). |
| 79 | pub billed_usd: f64, |
| 80 | /// Of `quantity`, what is past the cycle's included amount. |
| 81 | pub billable_quantity: f64, |
| 82 | /// Where `cost_usd` came from: `cycle::BASIS_*`, or empty for lines |
| 83 | /// that are not billable usage. |
| 84 | pub basis: &'static str, |
| 85 | } |
| 86 | |
| 87 | /// `Workers for Platforms CPU ms (First 60M ms are included)` → |
| 88 | /// `workers_for_platforms_cpu_ms`: lower case, words joined by `_`, and |
| 89 | /// what is in parentheses (the included amount, which changes) left out. |
| 90 | pub(crate) fn slug(text: &str) -> String { |
| 91 | let mut out = String::new(); |
| 92 | let mut depth = 0u32; |
| 93 | let mut gap = false; |
| 94 | for c in text.chars() { |
| 95 | match c { |
| 96 | '(' => depth += 1, |
| 97 | ')' => depth = depth.saturating_sub(1), |
| 98 | _ if depth > 0 => {} |
| 99 | c if c.is_ascii_alphanumeric() => { |
| 100 | if gap && !out.is_empty() { |
| 101 | out.push('_'); |
| 102 | } |
| 103 | gap = false; |
| 104 | out.push(c.to_ascii_lowercase()); |
| 105 | } |
| 106 | _ => gap = true, |
| 107 | } |
| 108 | } |
| 109 | out |
| 110 | } |
| 111 | |
| 112 | /// The product and meter for a family and service name, such as |
| 113 | /// (`D1`, `D1 - Rows Read (first 25 billion included)`) → (`d1`, |
| 114 | /// `d1_rows_read`). Without a family, the service's first word. |
| 115 | pub(crate) fn product_and_meter(family: &str, service: &str) -> (String, String) { |
| 116 | let (family, service) = if family.trim().is_empty() { |
| 117 | match service.split_once(" / ") { |
| 118 | Some((family, service)) => (family, service), |
| 119 | None => (service.split_whitespace().next().unwrap_or("other"), service), |
| 120 | } |
| 121 | } else { |
| 122 | (family, service) |
| 123 | }; |
| 124 | let product = slug(family); |
| 125 | let product = if product.is_empty() { "other".to_owned() } else { product }; |
| 126 | // Kept whole: `Workers for Platforms Requests` under `Workers` must stay |
| 127 | // `workers_for_platforms_requests`. |
| 128 | let meter = slug(service); |
| 129 | let meter = if meter.is_empty() { "usage".to_owned() } else { meter }; |
| 130 | (product, meter) |
| 131 | } |
| 132 | |
| 133 | fn text<'a>(row: &'a Value, keys: &[&str]) -> &'a str { |
| 134 | keys.iter().find_map(|k| row[*k].as_str()).unwrap_or_default() |
| 135 | } |
| 136 | |
| 137 | fn number(row: &Value, keys: &[&str]) -> Option<f64> { |
| 138 | keys.iter() |
| 139 | .find_map(|k| row[*k].as_f64().or_else(|| row[*k].as_str().and_then(|s| s.trim().parse().ok()))) |
| 140 | } |
| 141 | |
| 142 | /// FOCUS's consumed quantity: what was used, in the meter's own units. |
| 143 | const CONSUMED: [&str; 2] = ["ConsumedQuantity", "consumed_quantity"]; |
| 144 | /// FOCUS's pricing quantity: in pricing units, which can be blocks. |
| 145 | const PRICING: [&str; 2] = ["PricingQuantity", "pricing_quantity"]; |
| 146 | /// What a line cost g1t, as Cloudflare says: contracted, billed, effective. |
| 147 | /// Never `ListCost`: that is before the included amounts. |
| 148 | const COSTS: [&[&str]; 3] = [&["ContractedCost", "contracted_cost"], &["BilledCost", "billed_cost"], &["EffectiveCost", "effective_cost"]]; |
| 149 | |
| 150 | fn billed(row: &Value) -> f64 { |
| 151 | COSTS.iter().find_map(|keys| number(row, keys).filter(|c| *c > 0.0)).unwrap_or(0.0) |
| 152 | } |
| 153 | |
| 154 | /// One row of billable usage, read leniently: the API is new, and its |
| 155 | /// field names are FOCUS's (in either case). None without a service or a |
| 156 | /// day. The quantity is what was consumed (the included amounts are applied |
| 157 | /// to it over the cycle, `cycle`), else the pricing quantity. |
| 158 | pub(crate) fn line_from_focus(row: &Value) -> Option<CostLine> { |
| 159 | let service = text(row, &["ServiceName", "service_name", "service"]); |
| 160 | let day = text(row, &["ChargePeriodStart", "charge_period_start", "UsageDate", "date"]); |
| 161 | if service.is_empty() || day.len() < 10 { |
| 162 | return None; |
| 163 | } |
| 164 | let family = text(row, &["ServiceFamilyName", "service_family_name", "ServiceCategory"]); |
| 165 | let (product, meter) = product_and_meter(family, service); |
| 166 | let consumed = number(row, &CONSUMED); |
| 167 | let billed = billed(row); |
| 168 | Some(CostLine { |
| 169 | day: day[..10].to_owned(), |
| 170 | source: SOURCE_BILLABLE, |
| 171 | product, |
| 172 | meter, |
| 173 | unit: if consumed.is_some() { |
| 174 | text(row, &["ConsumedUnit", "consumed_unit", "PricingUnit", "pricing_unit"]) |
| 175 | } else { |
| 176 | text(row, &["PricingUnit", "pricing_unit", "ConsumedUnit", "consumed_unit"]) |
| 177 | } |
| 178 | .to_owned(), |
| 179 | quantity: consumed.or_else(|| number(row, &PRICING)).unwrap_or(0.0), |
| 180 | cost_usd: billed, |
| 181 | raw_name: if family.is_empty() { service.to_owned() } else { format!("{family} / {service}") }, |
| 182 | billed_usd: billed, |
| 183 | ..CostLine::default() |
| 184 | }) |
| 185 | } |
| 186 | |
| 187 | /// What a read of billable usage got back, to tell from sudo whether it |
| 188 | /// was all of it and in which units. |
| 189 | #[derive(Clone, Debug, Default, PartialEq)] |
| 190 | pub(crate) struct ReadStats { |
| 191 | pub rows: u32, |
| 192 | pub pages: u32, |
| 193 | pub consumed_rows: u32, |
| 194 | pub pricing_only_rows: u32, |
| 195 | pub costed_rows: u32, |
| 196 | } |
| 197 | |
| 198 | pub(crate) fn read_stats(body: &Value, pages: u32) -> ReadStats { |
| 199 | let rows = body["result"].as_array().cloned().unwrap_or_default(); |
| 200 | let mut stats = ReadStats { rows: rows.len() as u32, pages, ..ReadStats::default() }; |
| 201 | for row in &rows { |
| 202 | if number(row, &CONSUMED).is_some() { |
| 203 | stats.consumed_rows += 1; |
| 204 | } else if number(row, &PRICING).is_some() { |
| 205 | stats.pricing_only_rows += 1; |
| 206 | } |
| 207 | if billed(row) > 0.0 { |
| 208 | stats.costed_rows += 1; |
| 209 | } |
| 210 | } |
| 211 | stats |
| 212 | } |
| 213 | |
| 214 | /// The query for the page after `page` of a v4 answer: by cursor where it |
| 215 | /// gives one, else by page number while `total_pages` says there are more. |
| 216 | /// None when that was the last. |
| 217 | pub(crate) fn next_page(body: &Value, page: u32) -> Option<String> { |
| 218 | let info = &body["result_info"]; |
| 219 | if body["result"].as_array().is_some_and(Vec::is_empty) { |
| 220 | return None; |
| 221 | } |
| 222 | if let Some(cursor) = info["cursor"].as_str().filter(|c| !c.is_empty()) { |
| 223 | let encoded: String = cursor |
| 224 | .bytes() |
| 225 | .map(|b| if b.is_ascii_alphanumeric() || b"-_.~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") }) |
| 226 | .collect(); |
| 227 | return Some(format!("cursor={encoded}")); |
| 228 | } |
| 229 | let total = info["total_pages"].as_u64().or_else(|| { |
| 230 | let (count, per) = (info["total_count"].as_u64()?, info["per_page"].as_u64().filter(|p| *p > 0)?); |
| 231 | Some(count.div_ceil(per)) |
| 232 | })?; |
| 233 | (u64::from(page) < total).then(|| match info["per_page"].as_u64() { |
| 234 | Some(per) => format!("page={}&per_page={per}", page + 1), |
| 235 | None => format!("page={}", page + 1), |
| 236 | }) |
| 237 | } |
| 238 | |
| 239 | /// Lines with the same key added together, in key order. Cloudflare can |
| 240 | /// give one service several rows a day (regions, tiers); an upsert of |
| 241 | /// each would keep only the last. |
| 242 | pub(crate) fn aggregate(lines: Vec<CostLine>) -> Vec<CostLine> { |
| 243 | let mut out: Vec<CostLine> = Vec::new(); |
| 244 | for line in lines { |
| 245 | match out |
| 246 | .iter_mut() |
| 247 | .find(|l| l.day == line.day && l.source == line.source && l.product == line.product && l.meter == line.meter) |
| 248 | { |
| 249 | Some(existing) => { |
| 250 | existing.quantity += line.quantity; |
| 251 | existing.cost_usd += line.cost_usd; |
| 252 | existing.billed_usd += line.billed_usd; |
| 253 | existing.billable_quantity += line.billable_quantity; |
| 254 | } |
| 255 | None => out.push(line), |
| 256 | } |
| 257 | } |
| 258 | out.sort_by(|a, b| (&a.day, a.source, &a.product, &a.meter).cmp(&(&b.day, b.source, &b.product, &b.meter))); |
| 259 | out |
| 260 | } |
| 261 | |
| 262 | /// Every line of a billable-usage answer, aggregated. Errors when |
| 263 | /// Cloudflare says it failed. |
| 264 | pub(crate) fn lines_from_billable(body: &Value) -> std::result::Result<Vec<CostLine>, String> { |
| 265 | if body["success"] == Value::Bool(false) { |
| 266 | return Err(format!("billable usage failed: {}", body["errors"])); |
| 267 | } |
| 268 | let rows = body["result"].as_array().cloned().unwrap_or_default(); |
| 269 | Ok(aggregate(rows.iter().filter_map(line_from_focus).collect())) |
| 270 | } |
| 271 | |
| 272 | /// The GraphQL query for Artifacts' events by type and day. |
| 273 | pub(crate) const ARTIFACTS_QUERY: &str = "query ($account: String!, $since: Date!, $until: Date!) { |
| 274 | viewer { accounts(filter: { accountTag: $account }) { |
| 275 | artifactsEventsAdaptiveGroups(limit: 10000, filter: { date_geq: $since, date_leq: $until }) { |
| 276 | count |
| 277 | dimensions { date eventType repositoryName } |
| 278 | } |
| 279 | } } |
| 280 | }"; |
| 281 | |
| 282 | pub(crate) fn artifacts_variables(account: &str, since: &str, until: &str) -> Value { |
| 283 | json!({ "query": ARTIFACTS_QUERY, "variables": { "account": account, "since": since, "until": until } }) |
| 284 | } |
| 285 | |
| 286 | /// Artifacts' events as lines (`events_push`, `events_pull`, …), with no |
| 287 | /// cost: billable usage carries the cost. Errors when GraphQL does. |
| 288 | pub(crate) fn lines_from_artifacts(body: &Value) -> std::result::Result<Vec<CostLine>, String> { |
| 289 | if let Some(errors) = body["errors"].as_array().filter(|e| !e.is_empty()) { |
| 290 | return Err(format!("Artifacts events failed: {}", Value::Array(errors.clone()))); |
| 291 | } |
| 292 | let groups = body["data"]["viewer"]["accounts"][0]["artifactsEventsAdaptiveGroups"] |
| 293 | .as_array() |
| 294 | .cloned() |
| 295 | .unwrap_or_default(); |
| 296 | Ok(aggregate( |
| 297 | groups |
| 298 | .iter() |
| 299 | .filter_map(|g| { |
| 300 | let day = g["dimensions"]["date"].as_str()?; |
| 301 | let kind = g["dimensions"]["eventType"].as_str()?; |
| 302 | if day.len() < 10 { |
| 303 | return None; |
| 304 | } |
| 305 | Some(CostLine { |
| 306 | day: day[..10].to_owned(), |
| 307 | source: SOURCE_ARTIFACTS, |
| 308 | product: "artifacts".to_owned(), |
| 309 | meter: format!("events_{}", slug(kind)), |
| 310 | unit: "events".to_owned(), |
| 311 | quantity: g["count"].as_f64().unwrap_or(0.0), |
| 312 | cost_usd: 0.0, |
| 313 | raw_name: format!("Artifacts / {kind}"), |
| 314 | ..CostLine::default() |
| 315 | }) |
| 316 | }) |
| 317 | .collect(), |
| 318 | )) |
| 319 | } |
| 320 | |
| 321 | /// Where a cost line came from: AI Gateway's analytics, what it priced |
| 322 | /// g1t's own provider traffic at. No longer read since g1t stopped running |
| 323 | /// agents; the lines already kept stay in the history. |
| 324 | pub(crate) const SOURCE_GATEWAY: &str = "ai_gateway"; |
| 325 | /// The meter prefix of requests Cloudflare billed itself (unified billing), |
| 326 | /// which are on Cloudflare's bill as well as here. |
| 327 | pub(crate) const GATEWAY_WHOLESALE: &str = "wholesale__"; |
| 328 | |
| 329 | /// The workspace a repository in the store belongs to: keys are |
| 330 | /// `<workspace>--<repo>`; a pull request's working copy (`pulls--<id>`) is |
| 331 | /// its repository's workspace's, from `owners` (repos' `pull_owners`), else |
| 332 | /// no one's. |
| 333 | pub(crate) fn workspace_of_store_key(key: &str, owners: &BTreeMap<String, String>) -> Option<String> { |
| 334 | let (workspace, rest) = key.split_once("--")?; |
| 335 | if workspace.is_empty() || rest.is_empty() { |
| 336 | return None; |
| 337 | } |
| 338 | if workspace == "pulls" { |
| 339 | return owners.get(rest).map(|w| w.to_lowercase()); |
| 340 | } |
| 341 | Some(workspace.to_lowercase()) |
| 342 | } |
| 343 | |
| 344 | /// The pull request ids whose working copies Artifacts counted events for. |
| 345 | pub(crate) fn pull_ids(body: &Value) -> Vec<String> { |
| 346 | let groups = body["data"]["viewer"]["accounts"][0]["artifactsEventsAdaptiveGroups"].as_array().cloned().unwrap_or_default(); |
| 347 | let ids: BTreeSet<String> = groups |
| 348 | .iter() |
| 349 | .filter_map(|g| g["dimensions"]["repositoryName"].as_str()?.strip_prefix("pulls--").map(str::to_owned)) |
| 350 | .filter(|id| !id.is_empty()) |
| 351 | .collect(); |
| 352 | ids.into_iter().collect() |
| 353 | } |
| 354 | |
| 355 | /// Artifacts' billable operations per workspace and day, by repository |
| 356 | /// name: how Cloudflare's own count shares out. The meter is |
| 357 | /// `cloudflare_git`, which shares out the git bucket's cost (`margin`). |
| 358 | pub(crate) fn artifacts_by_workspace(body: &Value, owners: &BTreeMap<String, String>) -> Vec<(String, String, f64)> { |
| 359 | let groups = body["data"]["viewer"]["accounts"][0]["artifactsEventsAdaptiveGroups"].as_array().cloned().unwrap_or_default(); |
| 360 | let mut out: BTreeMap<(String, String), f64> = BTreeMap::new(); |
| 361 | for g in &groups { |
| 362 | let (Some(day), Some(kind)) = (g["dimensions"]["date"].as_str(), g["dimensions"]["eventType"].as_str()) else { continue }; |
| 363 | if day.len() < 10 || !ARTIFACTS_OPERATIONS.contains(&format!("events_{}", slug(kind)).as_str()) { |
| 364 | continue; |
| 365 | } |
| 366 | let Some(workspace) = g["dimensions"]["repositoryName"].as_str().and_then(|key| workspace_of_store_key(key, owners)) else { continue }; |
| 367 | *out.entry((day[..10].to_owned(), workspace)).or_default() += g["count"].as_f64().unwrap_or(0.0); |
| 368 | } |
| 369 | out.into_iter().map(|((day, workspace), count)| (day, workspace, count)).collect() |
| 370 | } |
| 371 | |
| 372 | /// Artifacts' event types that are operations it bills: what its |
| 373 | /// pricing names (create, push, pull, clone) and the metrics list. |
| 374 | /// Errors such as `rateLimited` are not. |
| 375 | pub(crate) const ARTIFACTS_OPERATIONS: [&str; 5] = ["events_create", "events_fork", "events_push", "events_pull", "events_delete"]; |
| 376 | |
| 377 | /// The days to read: the 31 before today on the first run (`last` is |
| 378 | /// None), else the last few, never further back than the backfill. |
| 379 | pub(crate) fn window(last_fetched_day: Option<&str>, now_ms: u64) -> (String, String) { |
| 380 | let day = |ms: u64| g1t_contracts::time::rfc3339(ms)[..10].to_owned(); |
| 381 | let today = day(now_ms); |
| 382 | let since = match last_fetched_day { |
| 383 | None => day(now_ms.saturating_sub(BACKFILL_DAYS * DAY_MS)), |
| 384 | Some(last) => { |
| 385 | let restate = day(now_ms.saturating_sub(RESTATE_DAYS * DAY_MS)); |
| 386 | let floor = day(now_ms.saturating_sub(BACKFILL_DAYS * DAY_MS)); |
| 387 | // A gap since the last run is read too, up to the backfill. |
| 388 | let from = if last < restate.as_str() { last.to_owned() } else { restate }; |
| 389 | if from < floor { floor } else { from } |
| 390 | } |
| 391 | }; |
| 392 | (since, today) |
| 393 | } |
| 394 | |
| 395 | /// Every day from `since` to `until`, inclusive. |
| 396 | pub(crate) fn days_between(since: &str, until: &str) -> Vec<String> { |
| 397 | let start = g1t_contracts::time::parse_rfc3339(&format!("{since}T00:00:00Z")); |
| 398 | let end = g1t_contracts::time::parse_rfc3339(&format!("{until}T00:00:00Z")); |
| 399 | match (start, end) { |
| 400 | (Some(start), Some(end)) if start <= end => (0..=((end - start) / DAY_MS)) |
| 401 | .map(|n| g1t_contracts::time::rfc3339(start + n * DAY_MS)[..10].to_owned()) |
| 402 | .collect(), |
| 403 | _ => Vec::new(), |
| 404 | } |
| 405 | } |
| 406 | |
| 407 | /// A rule from `cost_map`: which Cloudflare lines feed which of g1t's |
| 408 | /// products. |
| 409 | #[derive(Clone, Debug, PartialEq, serde::Deserialize)] |
| 410 | pub(crate) struct Rule { |
| 411 | /// Cloudflare's product, as slugged here. |
| 412 | pub product: String, |
| 413 | /// A meter prefix, or `*` for any meter of the product. |
| 414 | pub meter: String, |
| 415 | /// g1t's product it is a cost of: `sandboxes`, `git`, `platform`, … |
| 416 | pub bucket: String, |
| 417 | /// The price book meter whose cost it measures, if any. |
| 418 | pub price_meter: Option<String>, |
| 419 | /// g1t's own count of the same units, to compare quantities. |
| 420 | pub own_meter: Option<String>, |
| 421 | /// How far g1t's count may be from Cloudflare's before it is drift. |
| 422 | pub drift_percent: f64, |
| 423 | } |
| 424 | |
| 425 | /// The rule for a line: its product's rule with the longest matching |
| 426 | /// meter prefix, `*` last. None means no one decided what pays for it: |
| 427 | /// a leak until someone does. |
| 428 | pub(crate) fn classify<'a>(rules: &'a [Rule], product: &str, meter: &str) -> Option<&'a Rule> { |
| 429 | rules |
| 430 | .iter() |
| 431 | .filter(|r| r.product == product && (r.meter == "*" || meter.starts_with(r.meter.as_str()))) |
| 432 | .max_by_key(|r| if r.meter == "*" { 0 } else { r.meter.len() + 1 }) |
| 433 | } |
| 434 | |
| 435 | /// What a bucket is called in sudo. |
| 436 | pub(crate) fn bucket_title(bucket: &str) -> String { |
| 437 | match bucket { |
| 438 | "sandboxes" => "Sandboxes and builds".into(), |
| 439 | "deployments" => "Deployments".into(), |
| 440 | "git" => "Git operations".into(), |
| 441 | "repo_storage" => "Repository storage".into(), |
| 442 | "actions_cache" => "Actions cache".into(), |
| 443 | "embeddings" => "Search embeddings".into(), |
| 444 | "observability" => "Observability".into(), |
| 445 | "packages" => "Package storage".into(), |
| 446 | "security" => "Security scans".into(), |
| 447 | "domains" => "Custom domains".into(), |
| 448 | "models" => "Models".into(), |
| 449 | "platform" => "Running g1t (paid by the plan)".into(), |
| 450 | UNMAPPED => "Not mapped".into(), |
| 451 | other => other.replace('_', " "), |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | /// The bucket of a line no rule claims. |
| 456 | pub(crate) const UNMAPPED: &str = "unmapped"; |
| 457 | |
| 458 | /// A day's count per workspace from counts "from this day to the end of |
| 459 | /// its month" (what the repos service's `git_operations` answers with a |
| 460 | /// `since`): each day's is its own less the next day's, within a month. |
| 461 | /// The last day (today, so far) is its own. |
| 462 | pub(crate) fn daily_from_cumulative(days: &[String], cumulative: &[BTreeMap<String, u64>]) -> Vec<(String, String, u64)> { |
| 463 | let mut out = Vec::new(); |
| 464 | for (index, day) in days.iter().enumerate() { |
| 465 | let Some(today) = cumulative.get(index) else { break }; |
| 466 | let next = days |
| 467 | .get(index + 1) |
| 468 | .filter(|next| next[..7] == day[..7]) |
| 469 | .and_then(|_| cumulative.get(index + 1)); |
| 470 | for (workspace, count) in today { |
| 471 | let later = next.and_then(|n| n.get(workspace)).copied().unwrap_or(0); |
| 472 | let own = count.saturating_sub(later); |
| 473 | if own > 0 { |
| 474 | out.push((day.clone(), workspace.clone(), own)); |
| 475 | } |
| 476 | } |
| 477 | } |
| 478 | out |
| 479 | } |
| 480 | |
| 481 | /// One row of the repos service's `artifacts_usage`, read leniently while |
| 482 | /// its shape settles: a day, a workspace, a raw meter and a count. |
| 483 | pub(crate) fn raw_usage_row(row: &Value) -> Option<(String, String, String, f64)> { |
| 484 | let day = text(row, &["day", "date"]); |
| 485 | let workspace = text(row, &["namespace", "workspace"]); |
| 486 | let meter = text(row, &["meter", "kind", "event", "operation"]); |
| 487 | let count = number(row, &["count", "quantity", "operations", "value"])?; |
| 488 | (day.len() >= 10 && !workspace.is_empty() && !meter.is_empty()) |
| 489 | .then(|| (day[..10].to_owned(), workspace.to_lowercase(), slug(meter), count)) |
| 490 | } |
| 491 | |
| 492 | /// The repos service's operation mapping, as `artifacts_usage` returns it |
| 493 | /// beside the rows: for each raw meter (slugged), how many operations it |
| 494 | /// is to Cloudflare (`cost_operations`) and to the customer |
| 495 | /// (`billable_operations`). Repos owns this mapping |
| 496 | /// (`set_operation_mapping`); billing only reads it. |
| 497 | pub(crate) fn operation_mapping(body: &Value) -> BTreeMap<String, (f64, f64)> { |
| 498 | body["mapping"] |
| 499 | .as_array() |
| 500 | .map(|rows| { |
| 501 | rows.iter() |
| 502 | .filter_map(|r| { |
| 503 | let meter = r["meter"].as_str()?; |
| 504 | Some((slug(meter), (r["cost_operations"].as_f64().unwrap_or(0.0), r["billable_operations"].as_f64().unwrap_or(0.0)))) |
| 505 | }) |
| 506 | .collect() |
| 507 | }) |
| 508 | .unwrap_or_default() |
| 509 | } |
| 510 | |
| 511 | /// Raw counts weighted by one column of the mapping: what Cloudflare |
| 512 | /// should count (`cost`), or what customers are charged for. |
| 513 | pub(crate) fn weighted(raw: &[(String, f64)], mapping: &BTreeMap<String, (f64, f64)>, cost: bool) -> f64 { |
| 514 | raw.iter() |
| 515 | .map(|(meter, count)| count * mapping.get(meter).map_or(0.0, |(c, b)| if cost { *c } else { *b })) |
| 516 | .sum() |
| 517 | } |
| 518 | |
| 519 | impl Billing { |
| 520 | /// Reads Cloudflare's bill into `cost_lines`: the billable usage over |
| 521 | /// whole billing cycles (`cycle::bill_since`), every page of it, priced |
| 522 | /// over each cycle (`cycle::price_lines`) and replacing what was kept |
| 523 | /// for those days; Artifacts' events for the days due (`window`). |
| 524 | /// Returns those days, how many lines, and the first day of the bill's |
| 525 | /// read. None without a token. What could not be read is added to |
| 526 | /// `problems`. |
| 527 | pub(crate) async fn read_cloudflare( |
| 528 | &self, |
| 529 | keeper: &Keeper, |
| 530 | problems: &mut Vec<String>, |
| 531 | ) -> Result<Option<(String, String, u32, String)>> { |
| 532 | if !keeper.can_read_bill() { |
| 533 | return Ok(None); |
| 534 | } |
| 535 | #[derive(Deserialize)] |
| 536 | struct Last { |
| 537 | day: Option<String>, |
| 538 | } |
| 539 | let last = self |
| 540 | .db |
| 541 | .prepare("SELECT MAX(day) AS day FROM cost_lines WHERE source = ?") |
| 542 | .bind(&[SOURCE_BILLABLE.into()])? |
| 543 | .first::<Last>(None) |
| 544 | .await? |
| 545 | .and_then(|l| l.day); |
| 546 | let (since, until) = window(last.as_deref(), now_ms()); |
| 547 | let fetched_at = rfc3339(now_ms()); |
| 548 | let mut written = 0; |
| 549 | // The bill over whole cycles, every time: Cloudflare posts a day a |
| 550 | // day or two late and restates recent ones, and a cycle's included |
| 551 | // amounts can only be applied to all of its days. |
| 552 | let anchor = self.cycle_anchor().await?; |
| 553 | let bill_since = crate::cycle::bill_since(&until, anchor, RESTATE_DAYS as u32, last.is_none()); |
| 554 | let read = keeper |
| 555 | .billable_usage_pages(&bill_since, &until) |
| 556 | .await |
| 557 | .map_err(|e| e.to_string()) |
| 558 | .and_then(|(body, pages)| Ok((lines_from_billable(&body)?, read_stats(&body, pages)))); |
| 559 | match read { |
| 560 | Ok((_, stats)) if stats.rows == 0 && last.is_some() => { |
| 561 | problems.push(format!("Cloudflare's billable usage answered no rows for {bill_since} to {until}; what was read before is kept.")); |
| 562 | self.keep_read(&bill_since, &until, &stats, &fetched_at).await?; |
| 563 | } |
| 564 | Ok((mut lines, stats)) => { |
| 565 | crate::cycle::price_lines(&mut lines, anchor); |
| 566 | // The days read are replaced whole: a meter gone from a day |
| 567 | // must not keep its old line. |
| 568 | self.db |
| 569 | .prepare("DELETE FROM cost_lines WHERE source = ?1 AND day >= ?2 AND day <= ?3") |
| 570 | .bind(&[SOURCE_BILLABLE.into(), bill_since.as_str().into(), until.as_str().into()])? |
| 571 | .run() |
| 572 | .await?; |
| 573 | written += self.upsert_lines(&lines, &fetched_at).await?; |
| 574 | self.keep_read(&bill_since, &until, &stats, &fetched_at).await?; |
| 575 | } |
| 576 | Err(error) => problems.push(format!("Cloudflare's billable usage could not be read: {error}")), |
| 577 | } |
| 578 | match keeper.graphql(artifacts_variables(keeper.account(), &since, &until)).await.map_err(|e| e.to_string()) { |
| 579 | Ok(body) => match lines_from_artifacts(&body) { |
| 580 | Ok(lines) => { |
| 581 | written += self.upsert_lines(&lines, &fetched_at).await?; |
| 582 | let owners = self.pull_owners(&pull_ids(&body)).await; |
| 583 | self.keep_cloudflare_counts(&since, &until, &artifacts_by_workspace(&body, &owners), &fetched_at).await?; |
| 584 | } |
| 585 | Err(error) => problems.push(format!("Artifacts events could not be read: {error}")), |
| 586 | }, |
| 587 | Err(error) => problems.push(format!("Artifacts events could not be read: {error}")), |
| 588 | } |
| 589 | Ok(Some((since, until, written, bill_since))) |
| 590 | } |
| 591 | |
| 592 | /// What the last read of billable usage got back (`cost_reads`). |
| 593 | async fn keep_read(&self, since: &str, until: &str, stats: &ReadStats, read_at: &str) -> Result<()> { |
| 594 | self.db |
| 595 | .prepare( |
| 596 | "INSERT INTO cost_reads (source, read_at, since, until, rows, pages, consumed_rows, pricing_only_rows, costed_rows) |
| 597 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) |
| 598 | ON CONFLICT (source) DO UPDATE SET read_at = ?2, since = ?3, until = ?4, rows = ?5, pages = ?6, |
| 599 | consumed_rows = ?7, pricing_only_rows = ?8, costed_rows = ?9", |
| 600 | ) |
| 601 | .bind(&[ |
| 602 | SOURCE_BILLABLE.into(), |
| 603 | read_at.into(), |
| 604 | since.into(), |
| 605 | until.into(), |
| 606 | stats.rows.into(), |
| 607 | stats.pages.into(), |
| 608 | stats.consumed_rows.into(), |
| 609 | stats.pricing_only_rows.into(), |
| 610 | stats.costed_rows.into(), |
| 611 | ])? |
| 612 | .run() |
| 613 | .await?; |
| 614 | Ok(()) |
| 615 | } |
| 616 | |
| 617 | /// Cloudflare's own per-workspace counts for the days, replacing what |
| 618 | /// was kept for them. |
| 619 | /// The workspace of each pull request's working copy, from repos; |
| 620 | /// nothing while repos does not answer (those events stay no one's). |
| 621 | async fn pull_owners(&self, pulls: &[String]) -> BTreeMap<String, String> { |
| 622 | let Some(repos) = &self.repos else { return BTreeMap::new() }; |
| 623 | if pulls.is_empty() { |
| 624 | return BTreeMap::new(); |
| 625 | } |
| 626 | match g1t_kit::call::<_, Value>(repos, "pull_owners", &json!({ "pulls": pulls })).await { |
| 627 | Ok(body) => body["owners"] |
| 628 | .as_object() |
| 629 | .map(|o| o.iter().filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_owned()))).collect()) |
| 630 | .unwrap_or_default(), |
| 631 | Err(error) => { |
| 632 | worker::console_error!("pull_owners: {error}"); |
| 633 | BTreeMap::new() |
| 634 | } |
| 635 | } |
| 636 | } |
| 637 | |
| 638 | async fn keep_cloudflare_counts(&self, since: &str, until: &str, counts: &[(String, String, f64)], fetched_at: &str) -> Result<()> { |
| 639 | self.db |
| 640 | .prepare("DELETE FROM own_counts WHERE meter = 'cloudflare_git' AND day >= ?1 AND day <= ?2") |
| 641 | .bind(&[since.into(), until.into()])? |
| 642 | .run() |
| 643 | .await?; |
| 644 | for chunk in counts.chunks(50) { |
| 645 | let mut statements = Vec::with_capacity(chunk.len()); |
| 646 | for (day, workspace, count) in chunk { |
| 647 | statements.push( |
| 648 | self.db |
| 649 | .prepare("INSERT OR REPLACE INTO own_counts (day, meter, workspace, quantity, fetched_at) VALUES (?, 'cloudflare_git', ?, ?, ?)") |
| 650 | .bind(&[day.as_str().into(), workspace.as_str().into(), (*count).into(), fetched_at.into()])?, |
| 651 | ); |
| 652 | } |
| 653 | self.db.batch(statements).await?; |
| 654 | } |
| 655 | Ok(()) |
| 656 | } |
| 657 | |
| 658 | /// Upserts lines, a day's line replacing what was read for it before. |
| 659 | async fn upsert_lines(&self, lines: &[CostLine], fetched_at: &str) -> Result<u32> { |
| 660 | for chunk in lines.chunks(50) { |
| 661 | let mut statements = Vec::with_capacity(chunk.len()); |
| 662 | for line in chunk { |
| 663 | statements.push( |
| 664 | self.db |
| 665 | .prepare( |
| 666 | "INSERT INTO cost_lines (day, source, product, meter, unit, quantity, cost_usd, raw_name, fetched_at, billed_usd, billable_quantity, basis) |
| 667 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) |
| 668 | ON CONFLICT (day, source, product, meter) DO UPDATE SET |
| 669 | unit = ?5, quantity = ?6, cost_usd = ?7, raw_name = ?8, fetched_at = ?9, |
| 670 | billed_usd = ?10, billable_quantity = ?11, basis = ?12", |
| 671 | ) |
| 672 | .bind(&[ |
| 673 | line.day.as_str().into(), |
| 674 | line.source.into(), |
| 675 | line.product.as_str().into(), |
| 676 | line.meter.as_str().into(), |
| 677 | line.unit.as_str().into(), |
| 678 | line.quantity.into(), |
| 679 | line.cost_usd.into(), |
| 680 | line.raw_name.as_str().into(), |
| 681 | fetched_at.into(), |
| 682 | line.billed_usd.into(), |
| 683 | line.billable_quantity.into(), |
| 684 | line.basis.into(), |
| 685 | ])?, |
| 686 | ); |
| 687 | } |
| 688 | self.db.batch(statements).await?; |
| 689 | } |
| 690 | Ok(lines.len() as u32) |
| 691 | } |
| 692 | |
| 693 | /// The mapping from Cloudflare's meters to g1t's products. |
| 694 | pub(crate) async fn rules(&self) -> Result<Vec<Rule>> { |
| 695 | #[derive(Deserialize)] |
| 696 | struct Row { |
| 697 | product: String, |
| 698 | meter: String, |
| 699 | bucket: String, |
| 700 | price_meter: Option<String>, |
| 701 | own_meter: Option<String>, |
| 702 | drift_percent: f64, |
| 703 | } |
| 704 | Ok(self |
| 705 | .db |
| 706 | .prepare("SELECT product, meter, bucket, price_meter, own_meter, drift_percent FROM cost_map") |
| 707 | .all() |
| 708 | .await? |
| 709 | .results::<Row>()? |
| 710 | .into_iter() |
| 711 | .map(|r| Rule { |
| 712 | product: r.product, |
| 713 | meter: r.meter, |
| 714 | bucket: r.bucket, |
| 715 | price_meter: r.price_meter, |
| 716 | own_meter: r.own_meter, |
| 717 | drift_percent: r.drift_percent, |
| 718 | }) |
| 719 | .collect()) |
| 720 | } |
| 721 | |
| 722 | /// g1t's own counts for the days, all from the repos service, which |
| 723 | /// owns the mapping from raw meters to operations (`operation_mapping`): |
| 724 | /// |
| 725 | /// - `git_operations`: what customers are charged for, as repos counts |
| 726 | /// it (`git_operations`, already through its mapping). |
| 727 | /// - `cost_operations`: what g1t expects Cloudflare to bill, from the |
| 728 | /// raw meters (`artifacts_usage`) and the mapping's cost column. |
| 729 | /// - `artifacts_<meter>`: each raw meter. |
| 730 | pub(crate) async fn count_own(&self, since: &str, until: &str) -> Result<()> { |
| 731 | let Some(repos) = &self.repos else { return Ok(()) }; |
| 732 | let days = days_between(since, until); |
| 733 | let fetched_at = rfc3339(now_ms()); |
| 734 | let mut rows: Vec<(String, String, String, f64)> = Vec::new(); |
| 735 | |
| 736 | // Raw meters and repos' mapping; skipped while repos does not answer. |
| 737 | if let Ok(body) = g1t_kit::call::<_, Value>(repos, "artifacts_usage", &json!({ "from": since, "to": until })).await { |
| 738 | let mapping = operation_mapping(&body); |
| 739 | let mut by: BTreeMap<(String, String), Vec<(String, f64)>> = BTreeMap::new(); |
| 740 | for (day, workspace, meter, count) in body["rows"].as_array().map(|l| l.iter().filter_map(raw_usage_row).collect::<Vec<_>>()).unwrap_or_default() { |
| 741 | by.entry((day, workspace)).or_default().push((meter, count)); |
| 742 | } |
| 743 | for ((day, workspace), counts) in by { |
| 744 | // Several stores (namespaces) can give the same meter. |
| 745 | let mut merged: BTreeMap<String, f64> = BTreeMap::new(); |
| 746 | for (meter, count) in &counts { |
| 747 | *merged.entry(meter.clone()).or_default() += count; |
| 748 | } |
| 749 | for (meter, count) in &merged { |
| 750 | rows.push((day.clone(), format!("artifacts_{meter}"), workspace.clone(), *count)); |
| 751 | } |
| 752 | if !mapping.is_empty() { |
| 753 | rows.push((day, "cost_operations".to_owned(), workspace, weighted(&counts, &mapping, true))); |
| 754 | } |
| 755 | } |
| 756 | } |
| 757 | // What customers are charged for, as repos counts it through its mapping. |
| 758 | let mut cumulative = Vec::with_capacity(days.len()); |
| 759 | for day in &days { |
| 760 | let list: Vec<WorkspaceGitOperations> = g1t_kit::call( |
| 761 | repos, |
| 762 | "git_operations", |
| 763 | &GitOperationsArgs { month: day[..7].to_owned(), since: Some(format!("{day}T00")), namespace: None }, |
| 764 | ) |
| 765 | .await?; |
| 766 | cumulative.push(list.into_iter().map(|w| (w.namespace.to_lowercase(), w.operations)).collect::<BTreeMap<_, _>>()); |
| 767 | } |
| 768 | for (day, workspace, count) in daily_from_cumulative(&days, &cumulative) { |
| 769 | rows.push((day, "git_operations".to_owned(), workspace, count as f64)); |
| 770 | } |
| 771 | |
| 772 | // Each day's counts replace what was there. |
| 773 | self.db |
| 774 | .prepare("DELETE FROM own_counts WHERE day >= ?1 AND day <= ?2 AND meter NOT LIKE 'cloudflare_%'") |
| 775 | .bind(&[since.into(), until.into()])? |
| 776 | .run() |
| 777 | .await?; |
| 778 | for chunk in rows.chunks(50) { |
| 779 | let mut statements = Vec::with_capacity(chunk.len()); |
| 780 | for (day, meter, workspace, quantity) in chunk { |
| 781 | statements.push( |
| 782 | self.db |
| 783 | .prepare( |
| 784 | "INSERT INTO own_counts (day, meter, workspace, quantity, fetched_at) VALUES (?1, ?2, ?3, ?4, ?5) |
| 785 | ON CONFLICT (day, meter, workspace) DO UPDATE SET quantity = ?4, fetched_at = ?5", |
| 786 | ) |
| 787 | .bind(&[day.as_str().into(), meter.as_str().into(), workspace.as_str().into(), (*quantity).into(), fetched_at.as_str().into()])?, |
| 788 | ); |
| 789 | } |
| 790 | self.db.batch(statements).await?; |
| 791 | } |
| 792 | Ok(()) |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | #[cfg(test)] |
| 797 | mod tests { |
| 798 | use super::*; |
| 799 | |
| 800 | /// Billable usage as Cloudflare answered g1t on 2026-10-06 (two days, |
| 801 | /// trimmed), plus rows past the included amounts, which cost money. |
| 802 | fn billable_fixture() -> Value { |
| 803 | json!({ |
| 804 | "success": true, |
| 805 | "errors": [], |
| 806 | "result": [ |
| 807 | { |
| 808 | "ChargePeriodStart": "2026-10-03T00:00:00Z", |
| 809 | "ChargePeriodEnd": "2026-10-04T00:00:00Z", |
| 810 | "ServiceFamilyName": "Containers", |
| 811 | "ServiceName": "Container Memory, per GiB-Second (First 25 GiB-hours included)", |
| 812 | "PricingUnit": "Count", |
| 813 | "PricingQuantity": "14000", |
| 814 | "ContractedCost": 0, "BilledCost": 0, "ListCost": 0 |
| 815 | }, |
| 816 | { |
| 817 | "ChargePeriodStart": "2026-10-03T00:00:00Z", |
| 818 | "ServiceFamilyName": "D1", |
| 819 | "ServiceName": "D1 - Rows Read (first 25 billion included)", |
| 820 | "PricingUnit": "Count", |
| 821 | "PricingQuantity": 1200000, |
| 822 | "BilledCost": 0 |
| 823 | }, |
| 824 | { |
| 825 | "ChargePeriodStart": "2026-10-15T00:00:00Z", |
| 826 | "ServiceFamilyName": "Artifacts", |
| 827 | "ServiceName": "Artifacts Operations (First 10,000 included)", |
| 828 | "PricingUnit": "Count", |
| 829 | "PricingQuantity": "30000", |
| 830 | "BilledCost": "3.00", |
| 831 | "ListCost": 4.5 |
| 832 | }, |
| 833 | { |
| 834 | "ChargePeriodStart": "2026-10-15T00:00:00Z", |
| 835 | "ServiceFamilyName": "Artifacts", |
| 836 | "ServiceName": "Artifacts Operations (First 10,000 included)", |
| 837 | "PricingUnit": "Count", |
| 838 | "PricingQuantity": "10000", |
| 839 | "BilledCost": "1.50" |
| 840 | }, |
| 841 | { |
| 842 | "ChargePeriodStart": "2026-10-15T00:00:00Z", |
| 843 | "ServiceFamilyName": "Workers", |
| 844 | "ServiceName": "Workers for Platforms Requests (First 20M are included)", |
| 845 | "PricingUnit": "Count", |
| 846 | "PricingQuantity": 25000000, |
| 847 | "ListCost": 1.5 |
| 848 | }, |
| 849 | { "ServiceName": "No day, no line" } |
| 850 | ] |
| 851 | }) |
| 852 | } |
| 853 | |
| 854 | #[test] |
| 855 | fn names_become_products_and_meters() { |
| 856 | assert_eq!(slug("Workers for Platforms CPU ms (First 60M ms are included)"), "workers_for_platforms_cpu_ms"); |
| 857 | assert_eq!(product_and_meter("D1", "D1 - Rows Read (first 25 billion included)"), ("d1".into(), "d1_rows_read".into())); |
| 858 | assert_eq!(product_and_meter("Workers KV", "KV Read Operations (First 10M is included)"), ("workers_kv".into(), "kv_read_operations".into())); |
| 859 | assert_eq!(product_and_meter("", "Containers / Container vCPU"), ("containers".into(), "container_vcpu".into())); |
| 860 | assert_eq!(product_and_meter("Email", ""), ("email".into(), "usage".into())); |
| 861 | } |
| 862 | |
| 863 | #[test] |
| 864 | fn billable_usage_is_parsed_and_rows_of_one_meter_are_added() { |
| 865 | let lines = lines_from_billable(&billable_fixture()).unwrap(); |
| 866 | assert_eq!(lines.len(), 4, "{lines:#?}"); |
| 867 | let artifacts = lines.iter().find(|l| l.product == "artifacts").unwrap(); |
| 868 | // Two rows of the same day and meter: one line, added up. |
| 869 | assert_eq!(artifacts.meter, "artifacts_operations"); |
| 870 | assert_eq!(artifacts.quantity, 40_000.0); |
| 871 | assert!((artifacts.cost_usd - 4.5).abs() < 1e-9, "billed, not list: {}", artifacts.cost_usd); |
| 872 | let wfp = lines.iter().find(|l| l.meter.starts_with("workers_for_platforms")).unwrap(); |
| 873 | assert_eq!(wfp.product, "workers"); |
| 874 | // A list cost is before the included amounts: never what g1t pays. |
| 875 | // The cycle prices it (`cycle::price_lines`). |
| 876 | assert_eq!((wfp.cost_usd, wfp.billed_usd), (0.0, 0.0)); |
| 877 | let memory = lines.iter().find(|l| l.product == "containers").unwrap(); |
| 878 | assert_eq!((memory.day.as_str(), memory.quantity, memory.cost_usd), ("2026-10-03", 14_000.0, 0.0)); |
| 879 | assert!(lines_from_billable(&json!({ "success": false, "errors": [{ "code": 10000 }] })).is_err()); |
| 880 | } |
| 881 | |
| 882 | #[test] |
| 883 | fn the_quantity_is_what_was_consumed_not_the_pricing_blocks() { |
| 884 | // FOCUS: ConsumedQuantity in the meter's units; PricingQuantity in |
| 885 | // pricing units, which can be blocks of a million. |
| 886 | let row = json!({ |
| 887 | "ChargePeriodStart": "2026-10-07T00:00:00Z", |
| 888 | "ServiceFamilyName": "Workers", |
| 889 | "ServiceName": "Workers CPU ms (first 30M are included)", |
| 890 | "ConsumedQuantity": 11160000, "ConsumedUnit": "ms", |
| 891 | "PricingQuantity": 11.16, "PricingUnit": "1M ms", |
| 892 | "BilledCost": 0, "ListCost": 0.2232 |
| 893 | }); |
| 894 | let line = line_from_focus(&row).unwrap(); |
| 895 | assert_eq!((line.quantity, line.unit.as_str(), line.cost_usd), (11_160_000.0, "ms", 0.0)); |
| 896 | let body = json!({ "result": [row, { "ServiceName": "x", "ChargePeriodStart": "2026-10-07", "PricingQuantity": 3, "BilledCost": 0.5 }] }); |
| 897 | assert_eq!( |
| 898 | read_stats(&body, 2), |
| 899 | ReadStats { rows: 2, pages: 2, consumed_rows: 1, pricing_only_rows: 1, costed_rows: 1 } |
| 900 | ); |
| 901 | } |
| 902 | |
| 903 | #[test] |
| 904 | fn every_page_of_an_answer_is_read() { |
| 905 | let page = |info: Value| json!({ "success": true, "result": [{ "ServiceName": "x" }], "result_info": info }); |
| 906 | assert_eq!(next_page(&page(json!({ "page": 1, "per_page": 50, "total_pages": 3 })), 1).as_deref(), Some("page=2&per_page=50")); |
| 907 | assert_eq!(next_page(&page(json!({ "page": 3, "per_page": 50, "total_pages": 3 })), 3), None); |
| 908 | // From the count when the pages are not given. |
| 909 | assert_eq!(next_page(&page(json!({ "per_page": 100, "total_count": 230 })), 2).as_deref(), Some("page=3&per_page=100")); |
| 910 | assert_eq!(next_page(&page(json!({ "cursor": "a+b/c=" })), 1).as_deref(), Some("cursor=a%2Bb%2Fc%3D")); |
| 911 | assert_eq!(next_page(&page(json!({ "cursor": "" })), 1), None); |
| 912 | // No paging said, or nothing on the page: that was all. |
| 913 | assert_eq!(next_page(&json!({ "result": [{}] }), 1), None); |
| 914 | assert_eq!(next_page(&json!({ "result": [], "result_info": { "total_pages": 9 } }), 1), None); |
| 915 | } |
| 916 | |
| 917 | #[test] |
| 918 | fn reading_the_same_days_twice_gives_the_same_lines() { |
| 919 | // Idempotent: the same answer aggregates to the same keys and |
| 920 | // amounts, so the upsert replaces rather than adds. |
| 921 | let once = lines_from_billable(&billable_fixture()).unwrap(); |
| 922 | let twice = lines_from_billable(&billable_fixture()).unwrap(); |
| 923 | assert_eq!(once, twice); |
| 924 | let again = aggregate(once.clone()); |
| 925 | assert_eq!(again, once); |
| 926 | } |
| 927 | |
| 928 | #[test] |
| 929 | fn artifacts_events_are_counted_by_type_and_day() { |
| 930 | let body = json!({ |
| 931 | "data": { "viewer": { "accounts": [{ "artifactsEventsAdaptiveGroups": [ |
| 932 | { "count": 120, "dimensions": { "date": "2026-10-05", "eventType": "pull" } }, |
| 933 | { "count": 30, "dimensions": { "date": "2026-10-05", "eventType": "push" } }, |
| 934 | { "count": 2, "dimensions": { "date": "2026-10-05", "eventType": "rateLimited" } }, |
| 935 | { "count": 5, "dimensions": { "date": "2026-10-06", "eventType": "pull" } } |
| 936 | ] }] } }, |
| 937 | "errors": null |
| 938 | }); |
| 939 | let lines = lines_from_artifacts(&body).unwrap(); |
| 940 | assert_eq!(lines.len(), 4); |
| 941 | assert_eq!(lines[0].meter, "events_pull"); |
| 942 | assert_eq!(lines[0].quantity, 120.0); |
| 943 | assert!(lines.iter().any(|l| l.meter == "events_ratelimited")); |
| 944 | // By workspace, from the repository's store key; forks say none. |
| 945 | let by_repo = json!({ "data": { "viewer": { "accounts": [{ "artifactsEventsAdaptiveGroups": [ |
| 946 | { "count": 100, "dimensions": { "date": "2026-10-05", "eventType": "pull", "repositoryName": "acme--api" } }, |
| 947 | { "count": 20, "dimensions": { "date": "2026-10-05", "eventType": "push", "repositoryName": "acme--web" } }, |
| 948 | { "count": 7, "dimensions": { "date": "2026-10-05", "eventType": "fork", "repositoryName": "pulls--123" } }, |
| 949 | { "count": 3, "dimensions": { "date": "2026-10-05", "eventType": "serverError", "repositoryName": "beta--x" } } |
| 950 | ] }] } } }); |
| 951 | assert_eq!(artifacts_by_workspace(&by_repo, &BTreeMap::new()), vec![("2026-10-05".to_string(), "acme".to_string(), 120.0)]); |
| 952 | let none = BTreeMap::new(); |
| 953 | assert_eq!(workspace_of_store_key("Acme--api", &none), Some("acme".into())); |
| 954 | assert_eq!(workspace_of_store_key("pulls--9", &none), None); |
| 955 | assert_eq!(workspace_of_store_key("plain", &none), None); |
| 956 | let owners = BTreeMap::from([("9".to_string(), "Acme".to_string())]); |
| 957 | assert_eq!(workspace_of_store_key("pulls--9", &owners), Some("acme".into())); |
| 958 | assert_eq!(workspace_of_store_key("pulls--10", &owners), None); |
| 959 | let operations: f64 = lines |
| 960 | .iter() |
| 961 | .filter(|l| l.day == "2026-10-05" && ARTIFACTS_OPERATIONS.contains(&l.meter.as_str())) |
| 962 | .map(|l| l.quantity) |
| 963 | .sum(); |
| 964 | assert_eq!(operations, 150.0); |
| 965 | let failed = json!({ "data": null, "errors": [{ "message": "unknown field" }] }); |
| 966 | assert!(lines_from_artifacts(&failed).is_err()); |
| 967 | } |
| 968 | |
| 969 | #[test] |
| 970 | fn the_first_run_backfills_31_days_and_later_runs_restate_a_few() { |
| 971 | let now = g1t_contracts::time::parse_rfc3339("2026-10-06T04:17:00Z").unwrap(); |
| 972 | assert_eq!(window(None, now), ("2026-09-05".into(), "2026-10-06".into())); |
| 973 | assert_eq!(window(Some("2026-10-06"), now), ("2026-10-02".into(), "2026-10-06".into())); |
| 974 | // A week without a run: from the last day read. |
| 975 | assert_eq!(window(Some("2026-09-28"), now), ("2026-09-28".into(), "2026-10-06".into())); |
| 976 | // Never past the backfill. |
| 977 | assert_eq!(window(Some("2026-01-01"), now), ("2026-09-05".into(), "2026-10-06".into())); |
| 978 | assert_eq!(days_between("2026-09-29", "2026-10-02"), vec!["2026-09-29", "2026-09-30", "2026-10-01", "2026-10-02"]); |
| 979 | assert!(days_between("2026-10-02", "2026-10-01").is_empty()); |
| 980 | } |
| 981 | |
| 982 | #[test] |
| 983 | fn a_days_git_operations_are_its_count_less_the_next_days() { |
| 984 | let days: Vec<String> = ["2026-09-29", "2026-09-30", "2026-10-01", "2026-10-02"].iter().map(|d| d.to_string()).collect(); |
| 985 | let map = |pairs: &[(&str, u64)]| pairs.iter().map(|(w, n)| (w.to_string(), *n)).collect::<BTreeMap<_, _>>(); |
| 986 | // From each day to the end of its month. |
| 987 | let cumulative = vec![map(&[("acme", 30), ("beta", 4)]), map(&[("acme", 10)]), map(&[("acme", 7)]), map(&[("acme", 2)])]; |
| 988 | let daily = daily_from_cumulative(&days, &cumulative); |
| 989 | assert_eq!( |
| 990 | daily, |
| 991 | vec![ |
| 992 | ("2026-09-29".into(), "acme".into(), 20), |
| 993 | ("2026-09-29".into(), "beta".into(), 4), |
| 994 | // The month's last day is its own. |
| 995 | ("2026-09-30".into(), "acme".into(), 10), |
| 996 | ("2026-10-01".into(), "acme".into(), 5), |
| 997 | // Today so far. |
| 998 | ("2026-10-02".into(), "acme".into(), 2), |
| 999 | ] |
| 1000 | ); |
| 1001 | } |
| 1002 | |
| 1003 | #[test] |
| 1004 | fn raw_meters_are_weighted_by_the_repos_mapping() { |
| 1005 | // As repos' artifacts_usage answers: rows, and its operation_mapping. |
| 1006 | let body = json!({ |
| 1007 | "rows": [ |
| 1008 | { "day": "2026-10-15", "store": "g1t", "workspace": "Acme", "meter": "git.fetch", "count": 12, "bytes_in": 0, "bytes_out": 0 }, |
| 1009 | { "day": "2026-10-15", "store": "g1t", "workspace": "acme", "meter": "git.receive_pack", "count": 3, "bytes_in": 0, "bytes_out": 0 }, |
| 1010 | { "day": "2026-10-15", "store": "g1t", "workspace": "acme", "meter": "binding.read_blob", "count": 400, "bytes_in": 0, "bytes_out": 0 } |
| 1011 | ], |
| 1012 | "mapping": [ |
| 1013 | { "meter": "git.fetch", "cost_operations": 1, "billable_operations": 1 }, |
| 1014 | { "meter": "git.receive_pack", "cost_operations": 1, "billable_operations": 1 }, |
| 1015 | { "meter": "binding.read_blob", "cost_operations": 0, "billable_operations": 0 } |
| 1016 | ], |
| 1017 | "truncated": false |
| 1018 | }); |
| 1019 | let row = raw_usage_row(&body["rows"][0]).unwrap(); |
| 1020 | assert_eq!(row, ("2026-10-15".into(), "acme".into(), "git_fetch".into(), 12.0)); |
| 1021 | assert!(raw_usage_row(&json!({ "day": "2026-10-15", "count": 1 })).is_none()); |
| 1022 | let mut mapping = operation_mapping(&body); |
| 1023 | let raw: Vec<(String, f64)> = body["rows"].as_array().unwrap().iter().filter_map(raw_usage_row).map(|r| (r.2, r.3)).collect(); |
| 1024 | assert_eq!(weighted(&raw, &mapping, true), 15.0); |
| 1025 | assert_eq!(weighted(&raw, &mapping, false), 15.0); |
| 1026 | // Cloudflare turns out to bill binding reads: repos changes one row |
| 1027 | // (set_operation_mapping), and the bill g1t expects follows. |
| 1028 | mapping.insert("binding_read_blob".into(), (1.0, 0.0)); |
| 1029 | assert_eq!(weighted(&raw, &mapping, true), 415.0); |
| 1030 | assert_eq!(weighted(&raw, &mapping, false), 15.0); |
| 1031 | assert!(operation_mapping(&json!({})).is_empty()); |
| 1032 | } |
| 1033 | |
| 1034 | #[test] |
| 1035 | fn the_most_specific_rule_claims_a_line() { |
| 1036 | let rule = |product: &str, meter: &str, bucket: &str| Rule { |
| 1037 | product: product.into(), |
| 1038 | meter: meter.into(), |
| 1039 | bucket: bucket.into(), |
| 1040 | price_meter: None, |
| 1041 | own_meter: None, |
| 1042 | drift_percent: 10.0, |
| 1043 | }; |
| 1044 | let rules = vec![rule("workers", "*", "platform"), rule("workers", "workers_for_platforms", "deployments"), rule("artifacts", "*", "git")]; |
| 1045 | assert_eq!(classify(&rules, "workers", "workers_for_platforms_cpu_ms").unwrap().bucket, "deployments"); |
| 1046 | assert_eq!(classify(&rules, "workers", "workers_cpu_ms").unwrap().bucket, "platform"); |
| 1047 | assert_eq!(classify(&rules, "artifacts", "artifacts_operations").unwrap().bucket, "git"); |
| 1048 | assert!(classify(&rules, "browser_rendering", "browser_hours").is_none()); |
| 1049 | } |
| 1050 | } |