Sign in

flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp

g1t/services/billing/src/overages.rs

404 lines18,254 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1//! Accidental overages: protecting g1t without punishing customers.
2//!
3//! Spikes pause new compute before a runaway gets far (see `compute`).
4//! When a month still goes well past a workspace's usual, or hits a spike,
5//! it shows in sudo's Overages queue with its typical month, this month,
6//! what it cost g1t, the margin, and the runs that caused it.
7//!
8//! **Goodwill credit.** One click, once per workspace in 12 months: a
9//! credit for the overage above its typical month. It always includes g1t's
10//! margin on that overage; of what the overage cost g1t, it covers at most
11//! `OVERAGE_FORGIVE_COST_MICROS` ($50), so one forgiveness never costs g1t
12//! more than that. More than that, or a second within 12 months, needs a
13//! typed reason, and sudo shows what g1t absorbs in real cost. Every credit
14//! is in `admin_actions` with who and why, and on the customer's statement
15//! as "Credit from g1t: accidental usage on <date>".
16
17use g1t_contracts::billing::{
18 AdminGoodwillArgs, AdminOveragesArgs, AdminVelocityArgs, EntryKind, Goodwill, LedgerEntry, Overage, PlanKind, Velocity,
19};
20use g1t_contracts::time::rfc3339;
21use g1t_contracts::{FailureCode, Outcome, new_id};
22use g1t_kit::now_ms;
23use serde::Deserialize;
24use worker::Result;
25
26use crate::features::dollars;
27use crate::limits::previous_month;
28use crate::{Billing, LedgerRow};
29
30/// A month counts as an overage at twice the typical month, and at least
31/// this much above it.
32const OVERAGE_MIN_MICROS: i64 = 10_000_000;
33/// How long one goodwill credit lasts before another is one click again.
34const GOODWILL_DAYS: u64 = 365;
35/// Reference prefix of goodwill credits. Starts `crd`, so like every
36/// credit from g1t it never counts as a payment.
37pub(crate) const GOODWILL_PREFIX: &str = "crd_goodwill_";
38
39/// The median of the months given; 0 with none.
40pub(crate) fn typical(months: &[i64]) -> i64 {
41 let mut sorted: Vec<i64> = months.iter().map(|m| (*m).max(0)).collect();
42 if sorted.is_empty() {
43 return 0;
44 }
45 sorted.sort_unstable();
46 let middle = sorted.len() / 2;
47 if sorted.len() % 2 == 0 { (sorted[middle - 1] + sorted[middle]) / 2 } else { sorted[middle] }
48}
49
50/// Whether this month is well past the typical one.
51pub(crate) fn is_overage(this_month: i64, typical: i64) -> bool {
52 this_month - typical >= OVERAGE_MIN_MICROS && this_month >= typical * 2
53}
54
55/// The one-click goodwill credit for a month: the overage above the
56/// typical month, split into g1t's margin and what g1t paid for it. The
57/// credit is the margin, always, plus the cost up to `cap`.
58pub(crate) fn goodwill(this_month: i64, typical: i64, margin_percent: u32, cap: i64) -> Goodwill {
59 let overage = (this_month - typical).max(0);
60 // What g1t paid of it: the charge less the margin, rounded so the
61 // margin is never overstated.
62 let cost = (overage * 100 + i64::from(100 + margin_percent) - 1) / i64::from(100 + margin_percent);
63 let cost = cost.min(overage);
64 let margin = overage - cost;
65 let absorbed = cost.min(cap.max(0));
66 Goodwill { overage_micros: overage, margin_micros: margin, cost_micros: cost, credit_micros: margin + absorbed, absorbed_micros: absorbed }
67}
68
69/// What a credit of `amount` costs g1t in real money, given the overage's
70/// margin: whatever of it is not margin.
71pub(crate) fn absorbed_by(amount: i64, quote: &Goodwill) -> i64 {
72 (amount - quote.margin_micros).max(0)
73}
74
75/// Whether a goodwill credit of `amount` needs a typed reason: past the
76/// one-click credit, or a second within 12 months.
77pub(crate) fn needs_reason(amount: i64, quote: &Goodwill, given_within_year: bool) -> bool {
78 amount > quote.credit_micros || given_within_year
79}
80
81impl Billing {
82 /// A workspace's charges this month, and its last three months'.
83 async fn months_charged(&self, workspace: &str) -> Result<(i64, Vec<i64>)> {
84 let now = rfc3339(now_ms());
85 let this = now[..7].to_owned();
86 let mut earlier = vec![];
87 let mut month = previous_month(&this);
88 for _ in 0..3 {
89 earlier.push(month.clone());
90 month = previous_month(&month);
91 }
92 #[derive(Deserialize)]
93 struct Row {
94 month: String,
95 charged: Option<i64>,
96 }
97 let rows = self
98 .db
99 .prepare(
100 "SELECT substr(created_at, 1, 7) AS month, -SUM(amount_micros) AS charged FROM ledger
101 WHERE workspace = ? AND kind = 'usage' AND created_at >= ? GROUP BY 1",
102 )
103 .bind(&[workspace.into(), format!("{}-01", earlier[2]).into()])?
104 .all()
105 .await?
106 .results::<Row>()?;
107 let get = |m: &str| rows.iter().find(|r| r.month == m).and_then(|r| r.charged).unwrap_or(0).max(0);
108 // Months before the workspace's first are not "typical" zeros.
109 let first = rows.iter().map(|r| r.month.clone()).min();
110 let history: Vec<i64> = earlier
111 .iter()
112 .filter(|m| first.as_deref().is_some_and(|first| m.as_str() >= first))
113 .map(|m| get(m))
114 .collect();
115 Ok((get(&this), history))
116 }
117
118 /// When the workspace last had a goodwill credit, if in the last year.
119 async fn last_goodwill(&self, workspace: &str) -> Result<Option<String>> {
120 #[derive(Deserialize)]
121 struct Row {
122 at: Option<String>,
123 }
124 let since = rfc3339(now_ms() - GOODWILL_DAYS * 24 * 60 * 60 * 1000);
125 Ok(self
126 .db
127 .prepare("SELECT MAX(created_at) AS at FROM ledger WHERE workspace = ? AND reference LIKE ? AND created_at >= ?")
128 .bind(&[workspace.into(), format!("{GOODWILL_PREFIX}%").into(), since.into()])?
129 .first::<Row>(None)
130 .await?
131 .and_then(|r| r.at))
132 }
133
134 async fn overage_of(&self, workspace: &str, force: bool) -> Result<Option<Overage>> {
135 let plan = self.plan_kind(workspace).await?;
136 if plan == PlanKind::Internal {
137 return Ok(None);
138 }
139 let (this_month, history) = self.months_charged(workspace).await?;
140 let usual = typical(&history);
141 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
142 let spike = self.latest_spike(workspace).await?.filter(|s| s.detected_at.as_str() >= month_start.as_str());
143 let open_request = self.requests_of(workspace).await?.into_iter().find(|r| r.kind == "overage" && r.status == "open");
144 if !force && !is_overage(this_month, usual) && spike.is_none() && open_request.is_none() {
145 return Ok(None);
146 }
147 #[derive(Deserialize)]
148 struct Cost {
149 cost: Option<i64>,
150 }
151 let cost = self
152 .db
153 .prepare(
154 "SELECT SUM(cost_micros) AS cost FROM ledger
155 WHERE workspace = ? AND kind = 'usage' AND COALESCE(billed_to, 'g1t') = 'g1t' AND created_at >= ?",
156 )
157 .bind(&[workspace.into(), month_start.as_str().into()])?
158 .first::<Cost>(None)
159 .await?
160 .and_then(|c| c.cost)
161 .unwrap_or(0);
162 let top_entries = self
163 .db
164 .prepare(
165 "SELECT * FROM ledger WHERE workspace = ? AND kind = 'usage' AND created_at >= ?
166 ORDER BY (-amount_micros + credit_micros + trial_micros + oss_micros + given_micros) DESC LIMIT 10",
167 )
168 .bind(&[workspace.into(), month_start.as_str().into()])?
169 .all()
170 .await?
171 .results::<LedgerRow>()?
172 .into_iter()
173 .map(LedgerEntry::from)
174 .collect();
175 let last = self.last_goodwill(workspace).await?;
176 Ok(Some(Overage {
177 workspace: workspace.to_owned(),
178 plan,
179 typical_month_micros: usual,
180 this_month_micros: this_month,
181 cost_micros: cost,
182 margin_micros: this_month - cost.min(this_month),
183 spike,
184 top_entries,
185 goodwill: goodwill(this_month, usual, self.margin_percent, self.plans.forgive_cost_micros),
186 goodwill_available: last.is_none(),
187 last_goodwill_at: last,
188 request: open_request,
189 }))
190 }
191
192 /// `admin_overages`: the Overages queue, biggest overage first.
193 pub(crate) async fn admin_overages(&self, _: AdminOveragesArgs) -> Result<Vec<Overage>> {
194 #[derive(Deserialize)]
195 struct Active {
196 workspace: String,
197 }
198 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
199 let active = self
200 .db
201 .prepare(
202 "SELECT DISTINCT workspace FROM ledger WHERE kind = 'usage' AND created_at >= ?1
203 UNION SELECT workspace FROM spikes WHERE detected_at >= ?1
204 UNION SELECT workspace FROM limit_requests WHERE kind = 'overage' AND status = 'open'
205 LIMIT 500",
206 )
207 .bind(&[month_start.into()])?
208 .all()
209 .await?
210 .results::<Active>()?;
211 let mut queue = vec![];
212 for Active { workspace } in active {
213 if let Some(overage) = self.overage_of(&workspace, false).await? {
214 queue.push(overage);
215 }
216 }
217 queue.sort_by(|a, b| b.goodwill.overage_micros.cmp(&a.goodwill.overage_micros));
218 Ok(queue)
219 }
220
221 /// `admin_goodwill`: a credit for accidental usage.
222 pub(crate) async fn admin_goodwill(&self, a: AdminGoodwillArgs) -> Result<Outcome<LedgerEntry>> {
223 let workspace = a.workspace.trim().to_lowercase();
224 if workspace.is_empty() || a.by.trim().is_empty() {
225 return Ok(Outcome::fail(FailureCode::Invalid, "Name the workspace, and who is giving the credit."));
226 }
227 let Some(overage) = self.overage_of(&workspace, true).await? else {
228 return Ok(Outcome::fail(FailureCode::Conflict, "g1t's own workspaces are not charged, so there is nothing to credit."));
229 };
230 let quote = overage.goodwill;
231 let amount = a.amount_micros.unwrap_or(quote.credit_micros);
232 if amount <= 0 {
233 return Ok(Outcome::fail(FailureCode::Invalid, "This month is not above the workspace's typical month, so the one-click credit is $0. Give an amount, with a reason."));
234 }
235 if amount > 10_000 * g1t_contracts::billing::MICROS_PER_DOLLAR {
236 return Ok(Outcome::fail(FailureCode::Invalid, "A goodwill credit is at most $10,000."));
237 }
238 let reason = a.reason.trim();
239 if needs_reason(amount, &quote, overage.last_goodwill_at.is_some()) && reason.chars().count() < 10 {
240 return Ok(Outcome::fail(
241 FailureCode::Invalid,
242 if overage.last_goodwill_at.is_some() {
243 "This workspace had a goodwill credit in the last 12 months: say why another, in a sentence."
244 } else {
245 "This is more than the one-click credit: say why, in a sentence."
246 },
247 ));
248 }
249 let day = a
250 .day
251 .as_deref()
252 .filter(|d| d.len() == 10 && d.chars().all(|c| c.is_ascii_digit() || c == '-'))
253 .map(str::to_owned)
254 .or_else(|| overage.spike.as_ref().map(|s| s.detected_at[..10].to_owned()))
255 .unwrap_or_else(|| rfc3339(now_ms())[..10].to_owned());
256 let reference = format!("{GOODWILL_PREFIX}{}", new_id("gw", now_ms()));
257 let description = format!("Credit from g1t: accidental usage on {day}");
258 self.enter(&workspace, EntryKind::TopUp, amount, &description, &reference, None, None, Some(a.by.trim()), None)
259 .await?;
260 let absorbed = absorbed_by(amount, &quote);
261 let account = self.account_of(&workspace).await?;
262 self.audit(
263 &account.id,
264 "goodwill",
265 &format!(
266 "{} to {workspace} for accidental usage on {day} (typical month {}, this month {}); g1t absorbs {} of real cost{}",
267 dollars(amount),
268 dollars(overage.typical_month_micros),
269 dollars(overage.this_month_micros),
270 dollars(absorbed),
271 if reason.is_empty() { String::new() } else { format!(": {reason}") }
272 ),
273 a.by.trim(),
274 )
275 .await?;
276 // An open overage request is answered by the credit.
277 if let Some(request) = &overage.request {
278 self.db
279 .prepare(
280 "UPDATE limit_requests SET status = 'approved', decided_by = ?1, decided_at = ?2, answer = ?3
281 WHERE id = ?4 AND status = 'open'",
282 )
283 .bind(&[
284 a.by.trim().into(),
285 rfc3339(now_ms()).into(),
286 format!("g1t credited {} for accidental usage on {day}. It is on this month's statement.", dollars(amount)).into(),
287 request.id.as_str().into(),
288 ])?
289 .run()
290 .await?;
291 }
292 let row = self
293 .db
294 .prepare("SELECT * FROM ledger WHERE reference = ?")
295 .bind(&[reference.as_str().into()])?
296 .first::<LedgerRow>(None)
297 .await?;
298 Ok(match row {
299 Some(row) => Outcome::Ok(LedgerEntry::from(row)),
300 None => Outcome::fail(FailureCode::NotFound, "The credit was not saved."),
301 })
302 }
303
304 /// `admin_velocity`: workspaces spending in the last day, fastest first.
305 pub(crate) async fn admin_velocity(&self, _: AdminVelocityArgs) -> Result<Vec<Velocity>> {
306 #[derive(Deserialize)]
307 struct Active {
308 workspace: String,
309 first_seen: Option<String>,
310 }
311 let day_ago = rfc3339(now_ms() - 24 * 60 * 60 * 1000);
312 let active = self
313 .db
314 .prepare(
315 "SELECT workspace, (SELECT MIN(created_at) FROM ledger l WHERE l.workspace = ledger.workspace) AS first_seen
316 FROM ledger WHERE kind = 'usage' AND created_at >= ? GROUP BY workspace LIMIT 200",
317 )
318 .bind(&[day_ago.into()])?
319 .all()
320 .await?
321 .results::<Active>()?;
322 let mut list = vec![];
323 for Active { workspace, first_seen } in active {
324 let pace = self.pace(&workspace).await?;
325 let (this_month, _) = self.months_charged(&workspace).await?;
326 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
327 list.push(Velocity {
328 plan: self.plan_kind(&workspace).await?,
329 last_hour_micros: pace.last_hour,
330 average_hour_micros: pace.usual_hour,
331 last_day_micros: pace.last_day,
332 this_month_micros: this_month,
333 ratio: if pace.usual_hour > 0 { pace.last_hour as f64 / pace.usual_hour as f64 } else { 0.0 },
334 spike: self.latest_spike(&workspace).await?.filter(|s| s.detected_at.as_str() >= month_start.as_str()),
335 first_seen,
336 workspace,
337 });
338 }
339 list.sort_by(|a, b| b.last_hour_micros.cmp(&a.last_hour_micros).then(b.last_day_micros.cmp(&a.last_day_micros)));
340 Ok(list)
341 }
342}
343
344#[cfg(test)]
345mod tests {
346 use super::*;
347
348 #[test]
349 fn a_typical_month_is_the_median() {
350 assert_eq!(typical(&[]), 0);
351 assert_eq!(typical(&[30_000_000]), 30_000_000);
352 assert_eq!(typical(&[30_000_000, 400_000_000, 20_000_000]), 30_000_000);
353 assert_eq!(typical(&[20_000_000, 40_000_000]), 30_000_000);
354 }
355
356 #[test]
357 fn an_overage_is_twice_the_usual_and_ten_dollars_over() {
358 assert!(is_overage(70_000_000, 30_000_000));
359 assert!(!is_overage(50_000_000, 30_000_000));
360 // A new workspace with no history: $10 is enough to look at.
361 assert!(is_overage(10_000_000, 0));
362 assert!(!is_overage(9_000_000, 0));
363 }
364
365 #[test]
366 fn goodwill_always_returns_the_margin_and_caps_the_cost() {
367 // $120 over a $30 typical month: $90 over, of which $75 cost g1t and
368 // $15 is margin. Under the $50 cap: the margin plus $50.
369 let quote = goodwill(120_000_000, 30_000_000, 20, 50_000_000);
370 assert_eq!(quote.overage_micros, 90_000_000);
371 assert_eq!(quote.cost_micros, 75_000_000);
372 assert_eq!(quote.margin_micros, 15_000_000);
373 assert_eq!(quote.absorbed_micros, 50_000_000);
374 assert_eq!(quote.credit_micros, 65_000_000);
375 // A small overage is credited in full: margin and cost.
376 let small = goodwill(42_000_000, 30_000_000, 20, 50_000_000);
377 assert_eq!(small.overage_micros, 12_000_000);
378 assert_eq!(small.credit_micros, 12_000_000);
379 assert_eq!(small.absorbed_micros, 10_000_000);
380 // A huge one never costs g1t more than the cap.
381 let huge = goodwill(10_030_000_000, 30_000_000, 20, 50_000_000);
382 assert_eq!(huge.absorbed_micros, 50_000_000);
383 assert_eq!(huge.credit_micros, huge.margin_micros + 50_000_000);
384 assert!(huge.credit_micros < huge.overage_micros);
385 // Nothing over the typical month: nothing to credit.
386 assert_eq!(goodwill(20_000_000, 30_000_000, 20, 50_000_000), Goodwill::default());
387 // The split never loses a millionth.
388 let odd = goodwill(1_000_001, 0, 20, 50_000_000);
389 assert_eq!(odd.cost_micros + odd.margin_micros, odd.overage_micros);
390 }
391
392 #[test]
393 fn more_than_one_click_needs_a_reason_and_says_what_g1t_absorbs() {
394 let quote = goodwill(120_000_000, 30_000_000, 20, 50_000_000);
395 assert!(!needs_reason(quote.credit_micros, &quote, false));
396 assert!(needs_reason(quote.credit_micros + 1, &quote, false));
397 // A second within 12 months, even the same amount.
398 assert!(needs_reason(quote.credit_micros, &quote, true));
399 // Crediting the whole $90 overage: g1t absorbs the whole $75 cost.
400 assert_eq!(absorbed_by(90_000_000, &quote), 75_000_000);
401 assert_eq!(absorbed_by(quote.credit_micros, &quote), 50_000_000);
402 assert_eq!(absorbed_by(10_000_000, &quote), 0);
403 }
404}