Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
486 linesCodeBlameRaw
1//! Accidental overages: protecting g1t without punishing customers.
2//!
3//! Spikes pause new compute before a runaway gets far (see `compute`).
4//! When a month still goes well past a workspace's usual, or hits a spike,
5//! it shows in sudo's Overages queue with its typical month, this month,
6//! what it cost g1t, the margin, and the runs that caused it.
7//!
8//! **Goodwill credit.** One click, once per workspace in 12 months: a
9//! credit for the overage above its typical month. It always includes g1t's
10//! margin on that overage; of what the overage cost g1t, it covers at most
11//! `OVERAGE_FORGIVE_COST_MICROS` ($50), so one forgiveness never costs g1t
12//! more than that. More than that, or a second within 12 months, needs a
13//! typed reason, and sudo shows what g1t absorbs in real cost. Every credit
14//! is in `admin_actions` with who and why, and on the customer's statement
15//! as "Credit from g1t: accidental usage on <date>".
16
17use g1t_contracts::billing::{
18 AdminGoodwillArgs, AdminOveragesArgs, AdminVelocityArgs, CreditKind, Goodwill, LedgerEntry, Overage, PlanKind, Velocity,
19};
20use g1t_contracts::time::rfc3339;
21use g1t_contracts::{FailureCode, Outcome};
22use g1t_kit::now_ms;
23use serde::Deserialize;
24use worker::Result;
25
26use crate::features::dollars;
27use crate::limits::previous_month;
28use crate::{Billing, LedgerRow};
29
30/// A month counts as an overage at twice the typical month, and at least
31/// this much above it.
32const OVERAGE_MIN_MICROS: i64 = 10_000_000;
33/// How long one goodwill credit lasts before another is one click again.
34const GOODWILL_DAYS: u64 = 365;
35/// Reference prefix of goodwill credits. Starts `crd`, so like every
36/// credit from g1t it never counts as a payment.
37pub(crate) const GOODWILL_PREFIX: &str = "crd_goodwill_";
38
39/// A goodwill credit without a request key, to the same workspace this
40/// soon after another, is taken as the same click again: two minutes.
41const GOODWILL_REPEAT_MS: u64 = 2 * 60 * 1000;
42
43/// A goodwill credit's ledger reference: from the request's key when it
44/// has one (letters, digits, `-` and `_`, up to 64), else from the
45/// workspace and the two minutes it was asked in, so that a request sent
46/// twice gives one credit.
47pub(crate) fn goodwill_reference(workspace: &str, request_key: Option<&str>, now_ms: u64) -> String {
48 let key: String = request_key
49 .unwrap_or_default()
50 .chars()
51 .filter(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
52 .take(64)
53 .collect();
54 if key.is_empty() {
55 format!("{GOODWILL_PREFIX}{workspace}_{}", now_ms / GOODWILL_REPEAT_MS)
56 } else {
57 format!("{GOODWILL_PREFIX}{workspace}_k{key}")
58 }
59}
60
61/// The median of the months given; 0 with none.
62pub(crate) fn typical(months: &[i64]) -> i64 {
63 let mut sorted: Vec<i64> = months.iter().map(|m| (*m).max(0)).collect();
64 if sorted.is_empty() {
65 return 0;
66 }
67 sorted.sort_unstable();
68 let middle = sorted.len() / 2;
69 if sorted.len().is_multiple_of(2) { (sorted[middle - 1] + sorted[middle]) / 2 } else { sorted[middle] }
70}
71
72/// Whether this month is well past the typical one.
73pub(crate) fn is_overage(this_month: i64, typical: i64) -> bool {
74 this_month - typical >= OVERAGE_MIN_MICROS && this_month >= typical * 2
75}
76
77/// The one-click goodwill credit for a month: the overage above the
78/// typical month, split into g1t's margin and what g1t paid for it. The
79/// credit is the margin, always, plus the cost up to `cap`.
80pub(crate) fn goodwill(this_month: i64, typical: i64, margin_percent: u32, cap: i64) -> Goodwill {
81 let overage = (this_month - typical).max(0);
82 // What g1t paid of it: the charge less the margin, rounded so the
83 // margin is never overstated.
84 let cost = (overage * 100 + i64::from(100 + margin_percent) - 1) / i64::from(100 + margin_percent);
85 let cost = cost.min(overage);
86 let margin = overage - cost;
87 let absorbed = cost.min(cap.max(0));
88 Goodwill { overage_micros: overage, margin_micros: margin, cost_micros: cost, credit_micros: margin + absorbed, absorbed_micros: absorbed }
89}
90
91/// What a credit of `amount` costs g1t in real money, given the overage's
92/// margin: whatever of it is not margin.
93pub(crate) fn absorbed_by(amount: i64, quote: &Goodwill) -> i64 {
94 (amount - quote.margin_micros).max(0)
95}
96
97/// Whether a goodwill credit of `amount` needs a typed reason: past the
98/// one-click credit, or a second within 12 months.
99pub(crate) fn needs_reason(amount: i64, quote: &Goodwill, given_within_year: bool) -> bool {
100 amount > quote.credit_micros || given_within_year
101}
102
103impl Billing {
104 /// A workspace's charges this month, and its last three months'.
105 async fn months_charged(&self, workspace: &str) -> Result<(i64, Vec<i64>)> {
106 let now = rfc3339(now_ms());
107 let this = now[..7].to_owned();
108 let mut earlier = vec![];
109 let mut month = previous_month(&this);
110 for _ in 0..3 {
111 earlier.push(month.clone());
112 month = previous_month(&month);
113 }
114 #[derive(Deserialize)]
115 struct Row {
116 month: String,
117 charged: Option<i64>,
118 }
119 let rows = self
120 .db
121 .prepare(
122 "SELECT substr(created_at, 1, 7) AS month, -SUM(amount_micros) AS charged FROM ledger
123 WHERE workspace = ? AND kind = 'usage' AND created_at >= ? GROUP BY 1",
124 )
125 .bind(&[workspace.into(), format!("{}-01", earlier[2]).into()])?
126 .all()
127 .await?
128 .results::<Row>()?;
129 let get = |m: &str| rows.iter().find(|r| r.month == m).and_then(|r| r.charged).unwrap_or(0).max(0);
130 // Months before the workspace's first are not "typical" zeros.
131 let first = rows.iter().map(|r| r.month.clone()).min();
132 let history: Vec<i64> = earlier
133 .iter()
134 .filter(|m| first.as_deref().is_some_and(|first| m.as_str() >= first))
135 .map(|m| get(m))
136 .collect();
137 Ok((get(&this), history))
138 }
139
140 /// A goodwill credit this request already gave: the entry with its
141 /// reference, or, for a request without a key (`recent`), any goodwill
142 /// credit to the workspace in the last `GOODWILL_REPEAT_MS`.
143 async fn goodwill_given(&self, workspace: &str, reference: &str, recent: bool, now: u64) -> Result<Option<LedgerRow>> {
144 let since = rfc3339(now.saturating_sub(GOODWILL_REPEAT_MS));
145 self.db
146 .prepare(
147 "SELECT * FROM ledger WHERE reference = ?1
148 OR (?2 = 1 AND workspace = ?3 AND reference LIKE ?4 AND created_at >= ?5)
149 ORDER BY reference = ?1 DESC, created_at DESC LIMIT 1",
150 )
151 .bind(&[
152 reference.into(),
153 (u8::from(recent) as f64).into(),
154 workspace.into(),
155 format!("{GOODWILL_PREFIX}%").into(),
156 since.into(),
157 ])?
158 .first::<LedgerRow>(None)
159 .await
160 }
161
162 /// When the workspace last had a goodwill credit, if in the last year.
163 async fn last_goodwill(&self, workspace: &str) -> Result<Option<String>> {
164 #[derive(Deserialize)]
165 struct Row {
166 at: Option<String>,
167 }
168 let since = rfc3339(now_ms() - GOODWILL_DAYS * 24 * 60 * 60 * 1000);
169 Ok(self
170 .db
171 .prepare("SELECT MAX(created_at) AS at FROM ledger WHERE workspace = ? AND reference LIKE ? AND created_at >= ?")
172 .bind(&[workspace.into(), format!("{GOODWILL_PREFIX}%").into(), since.into()])?
173 .first::<Row>(None)
174 .await?
175 .and_then(|r| r.at))
176 }
177
178 async fn overage_of(&self, workspace: &str, force: bool) -> Result<Option<Overage>> {
179 let plan = self.plan_kind(workspace).await?;
180 if plan == PlanKind::Internal {
181 return Ok(None);
182 }
183 let (this_month, history) = self.months_charged(workspace).await?;
184 let usual = typical(&history);
185 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
186 let spike = self.latest_spike(workspace).await?.filter(|s| s.detected_at.as_str() >= month_start.as_str());
187 let open_request = self.requests_of(workspace).await?.into_iter().find(|r| r.kind == "overage" && r.status == "open");
188 if !force && !is_overage(this_month, usual) && spike.is_none() && open_request.is_none() {
189 return Ok(None);
190 }
191 #[derive(Deserialize)]
192 struct Cost {
193 cost: Option<i64>,
194 }
195 let cost = self
196 .db
197 .prepare(
198 "SELECT SUM(cost_micros) AS cost FROM ledger
199 WHERE workspace = ? AND kind = 'usage' AND COALESCE(billed_to, 'g1t') = 'g1t' AND created_at >= ?",
200 )
201 .bind(&[workspace.into(), month_start.as_str().into()])?
202 .first::<Cost>(None)
203 .await?
204 .and_then(|c| c.cost)
205 .unwrap_or(0);
206 let top_entries = self
207 .db
208 .prepare(
209 "SELECT * FROM ledger WHERE workspace = ? AND kind = 'usage' AND created_at >= ?
210 ORDER BY (-amount_micros + credit_micros + trial_micros + oss_micros + given_micros) DESC LIMIT 10",
211 )
212 .bind(&[workspace.into(), month_start.as_str().into()])?
213 .all()
214 .await?
215 .results::<LedgerRow>()?
216 .into_iter()
217 .map(LedgerEntry::from)
218 .collect();
219 let last = self.last_goodwill(workspace).await?;
220 Ok(Some(Overage {
221 workspace: workspace.to_owned(),
222 plan,
223 typical_month_micros: usual,
224 this_month_micros: this_month,
225 cost_micros: cost,
226 margin_micros: this_month - cost.min(this_month),
227 spike,
228 top_entries,
229 goodwill: goodwill(this_month, usual, self.margin_percent, self.plans.forgive_cost_micros),
230 goodwill_available: last.is_none(),
231 last_goodwill_at: last,
232 request: open_request,
233 }))
234 }
235
236 /// `admin_overages`: the Overages queue, biggest overage first.
237 pub(crate) async fn admin_overages(&self, _: AdminOveragesArgs) -> Result<Vec<Overage>> {
238 #[derive(Deserialize)]
239 struct Active {
240 workspace: String,
241 }
242 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
243 let active = self
244 .db
245 .prepare(
246 "SELECT DISTINCT workspace FROM ledger WHERE kind = 'usage' AND created_at >= ?1
247 UNION SELECT workspace FROM spikes WHERE detected_at >= ?1
248 UNION SELECT workspace FROM limit_requests WHERE kind = 'overage' AND status = 'open'
249 LIMIT 500",
250 )
251 .bind(&[month_start.into()])?
252 .all()
253 .await?
254 .results::<Active>()?;
255 let mut queue = vec![];
256 for Active { workspace } in active {
257 if let Some(overage) = self.overage_of(&workspace, false).await? {
258 queue.push(overage);
259 }
260 }
261 queue.sort_by_key(|a| std::cmp::Reverse(a.goodwill.overage_micros));
262 Ok(queue)
263 }
264
265 /// `admin_goodwill`: a credit for accidental usage.
266 pub(crate) async fn admin_goodwill(&self, a: AdminGoodwillArgs) -> Result<Outcome<LedgerEntry>> {
267 let workspace = a.workspace.trim().to_lowercase();
268 if workspace.is_empty() || a.by.trim().is_empty() {
269 return Ok(Outcome::fail(FailureCode::Invalid, "Name the workspace, and who is giving the credit."));
270 }
271 // Once per request: the same key again (a double click, a retry)
272 // answers with the credit already given rather than a second one.
273 let now = now_ms();
274 let reference = goodwill_reference(&workspace, a.request_key.as_deref(), now);
275 let keyless = a.request_key.as_deref().map(str::trim).is_none_or(str::is_empty);
276 if let Some(given) = self.goodwill_given(&workspace, &reference, keyless, now).await? {
277 return Ok(Outcome::Ok(LedgerEntry::from(given)));
278 }
279 let Some(overage) = self.overage_of(&workspace, true).await? else {
280 return Ok(Outcome::fail(FailureCode::Conflict, "g1t's own workspaces are not charged, so there is nothing to credit."));
281 };
282 let quote = overage.goodwill;
283 let amount = a.amount_micros.unwrap_or(quote.credit_micros);
284 if amount <= 0 {
285 return Ok(Outcome::fail(FailureCode::Invalid, "This month is not above the workspace's typical month, so the one-click credit is $0. Give an amount, with a reason."));
286 }
287 if amount > 10_000 * g1t_contracts::billing::MICROS_PER_DOLLAR {
288 return Ok(Outcome::fail(FailureCode::Invalid, "A goodwill credit is at most $10,000."));
289 }
290 let reason = a.reason.trim();
291 if needs_reason(amount, &quote, overage.last_goodwill_at.is_some()) && reason.chars().count() < 10 {
292 return Ok(Outcome::fail(
293 FailureCode::Invalid,
294 if overage.last_goodwill_at.is_some() {
295 "This workspace had a goodwill credit in the last 12 months: say why another, in a sentence."
296 } else {
297 "This is more than the one-click credit: say why, in a sentence."
298 },
299 ));
300 }
301 let day = a
302 .day
303 .as_deref()
304 .filter(|d| d.len() == 10 && d.chars().all(|c| c.is_ascii_digit() || c == '-'))
305 .map(str::to_owned)
306 .or_else(|| overage.spike.as_ref().map(|s| s.detected_at[..10].to_owned()))
307 .unwrap_or_else(|| rfc3339(now_ms())[..10].to_owned());
308 let description = format!("Credit from g1t: accidental usage on {day}");
309 let note = if reason.is_empty() { format!("Accidental usage on {day}") } else { format!("Accidental usage on {day}: {reason}") };
310 // Two at once with the same reference: the grant's own key lets one
311 // in, and the other answers with it.
312 if let Err(error) = self
313 .grant_credit(&workspace, CreditKind::Goodwill, amount, &note, a.by.trim(), None, None, None, Some(reference.clone()), Some(description))
314 .await
315 {
316 return match self.goodwill_given(&workspace, &reference, false, now).await? {
317 Some(given) => Ok(Outcome::Ok(LedgerEntry::from(given))),
318 None => Err(error),
319 };
320 }
321 let absorbed = absorbed_by(amount, &quote);
322 let account = self.account_of(&workspace).await?;
323 self.audit(
324 &account.id,
325 "goodwill",
326 &format!(
327 "{} to {workspace} for accidental usage on {day} (typical month {}, this month {}); g1t absorbs {} of real cost{}",
328 dollars(amount),
329 dollars(overage.typical_month_micros),
330 dollars(overage.this_month_micros),
331 dollars(absorbed),
332 if reason.is_empty() { String::new() } else { format!(": {reason}") }
333 ),
334 a.by.trim(),
335 )
336 .await?;
337 self.tell_owners_of_credit(&workspace, CreditKind::Goodwill, amount, &format!("It is for the accidental usage on {day}."), None, None)
338 .await;
339 // An open overage request is answered by the credit.
340 if let Some(request) = &overage.request {
341 self.db
342 .prepare(
343 "UPDATE limit_requests SET status = 'approved', decided_by = ?1, decided_at = ?2, answer = ?3
344 WHERE id = ?4 AND status = 'open'",
345 )
346 .bind(&[
347 a.by.trim().into(),
348 rfc3339(now_ms()).into(),
349 format!("g1t credited {} for accidental usage on {day}. It is on this month's statement.", dollars(amount)).into(),
350 request.id.as_str().into(),
351 ])?
352 .run()
353 .await?;
354 }
355 let row = self
356 .db
357 .prepare("SELECT * FROM ledger WHERE reference = ?")
358 .bind(&[reference.as_str().into()])?
359 .first::<LedgerRow>(None)
360 .await?;
361 Ok(match row {
362 Some(row) => Outcome::Ok(LedgerEntry::from(row)),
363 None => Outcome::fail(FailureCode::NotFound, "The credit was not saved."),
364 })
365 }
366
367 /// `admin_velocity`: workspaces spending in the last day, fastest first.
368 pub(crate) async fn admin_velocity(&self, _: AdminVelocityArgs) -> Result<Vec<Velocity>> {
369 #[derive(Deserialize)]
370 struct Active {
371 workspace: String,
372 first_seen: Option<String>,
373 }
374 let day_ago = rfc3339(now_ms() - 24 * 60 * 60 * 1000);
375 let active = self
376 .db
377 .prepare(
378 "SELECT workspace, (SELECT MIN(created_at) FROM ledger l WHERE l.workspace = ledger.workspace) AS first_seen
379 FROM ledger WHERE kind = 'usage' AND created_at >= ? GROUP BY workspace LIMIT 200",
380 )
381 .bind(&[day_ago.into()])?
382 .all()
383 .await?
384 .results::<Active>()?;
385 let mut list = vec![];
386 for Active { workspace, first_seen } in active {
387 let pace = self.pace(&workspace).await?;
388 let (this_month, _) = self.months_charged(&workspace).await?;
389 let month_start = format!("{}-01", &rfc3339(now_ms())[..7]);
390 list.push(Velocity {
391 plan: self.plan_kind(&workspace).await?,
392 last_hour_micros: pace.last_hour,
393 average_hour_micros: pace.usual_hour,
394 last_day_micros: pace.last_day,
395 this_month_micros: this_month,
396 ratio: if pace.usual_hour > 0 { pace.last_hour as f64 / pace.usual_hour as f64 } else { 0.0 },
397 spike: self.latest_spike(&workspace).await?.filter(|s| s.detected_at.as_str() >= month_start.as_str()),
398 first_seen,
399 workspace,
400 });
401 }
402 list.sort_by(|a, b| b.last_hour_micros.cmp(&a.last_hour_micros).then(b.last_day_micros.cmp(&a.last_day_micros)));
403 Ok(list)
404 }
405}
406
407#[cfg(test)]
408mod tests {
409 use super::*;
410
411 #[test]
412 fn a_goodwill_request_sent_twice_is_one_credit() {
413 let now = 1_791_000_000_000;
414 // The same key: the same reference, whenever it comes.
415 let keyed = goodwill_reference("acme", Some("f3a9-1"), now);
416 assert_eq!(keyed, "crd_goodwill_acme_kf3a9-1");
417 assert_eq!(goodwill_reference("acme", Some("f3a9-1"), now + 3_600_000), keyed);
418 // Odd characters never reach the reference.
419 assert_eq!(goodwill_reference("acme", Some("a b'%;"), now), "crd_goodwill_acme_kab");
420 // No key: the same two minutes are the same credit; later, another.
421 let bare = goodwill_reference("acme", None, now);
422 assert!(bare.starts_with(GOODWILL_PREFIX));
423 assert_eq!(goodwill_reference("acme", Some(" "), now), bare);
424 let window_start = now - now % GOODWILL_REPEAT_MS;
425 assert_eq!(goodwill_reference("acme", None, window_start + GOODWILL_REPEAT_MS - 1), goodwill_reference("acme", None, window_start));
426 assert_ne!(goodwill_reference("acme", None, window_start + GOODWILL_REPEAT_MS), goodwill_reference("acme", None, window_start));
427 assert_ne!(goodwill_reference("other", None, now), bare);
428 }
429
430 #[test]
431 fn a_typical_month_is_the_median() {
432 assert_eq!(typical(&[]), 0);
433 assert_eq!(typical(&[30_000_000]), 30_000_000);
434 assert_eq!(typical(&[30_000_000, 400_000_000, 20_000_000]), 30_000_000);
435 assert_eq!(typical(&[20_000_000, 40_000_000]), 30_000_000);
436 }
437
438 #[test]
439 fn an_overage_is_twice_the_usual_and_ten_dollars_over() {
440 assert!(is_overage(70_000_000, 30_000_000));
441 assert!(!is_overage(50_000_000, 30_000_000));
442 // A new workspace with no history: $10 is enough to look at.
443 assert!(is_overage(10_000_000, 0));
444 assert!(!is_overage(9_000_000, 0));
445 }
446
447 #[test]
448 fn goodwill_always_returns_the_margin_and_caps_the_cost() {
449 // $120 over a $30 typical month: $90 over, of which $75 cost g1t and
450 // $15 is margin. Under the $50 cap: the margin plus $50.
451 let quote = goodwill(120_000_000, 30_000_000, 20, 50_000_000);
452 assert_eq!(quote.overage_micros, 90_000_000);
453 assert_eq!(quote.cost_micros, 75_000_000);
454 assert_eq!(quote.margin_micros, 15_000_000);
455 assert_eq!(quote.absorbed_micros, 50_000_000);
456 assert_eq!(quote.credit_micros, 65_000_000);
457 // A small overage is credited in full: margin and cost.
458 let small = goodwill(42_000_000, 30_000_000, 20, 50_000_000);
459 assert_eq!(small.overage_micros, 12_000_000);
460 assert_eq!(small.credit_micros, 12_000_000);
461 assert_eq!(small.absorbed_micros, 10_000_000);
462 // A huge one never costs g1t more than the cap.
463 let huge = goodwill(10_030_000_000, 30_000_000, 20, 50_000_000);
464 assert_eq!(huge.absorbed_micros, 50_000_000);
465 assert_eq!(huge.credit_micros, huge.margin_micros + 50_000_000);
466 assert!(huge.credit_micros < huge.overage_micros);
467 // Nothing over the typical month: nothing to credit.
468 assert_eq!(goodwill(20_000_000, 30_000_000, 20, 50_000_000), Goodwill::default());
469 // The split never loses a millionth.
470 let odd = goodwill(1_000_001, 0, 20, 50_000_000);
471 assert_eq!(odd.cost_micros + odd.margin_micros, odd.overage_micros);
472 }
473
474 #[test]
475 fn more_than_one_click_needs_a_reason_and_says_what_g1t_absorbs() {
476 let quote = goodwill(120_000_000, 30_000_000, 20, 50_000_000);
477 assert!(!needs_reason(quote.credit_micros, &quote, false));
478 assert!(needs_reason(quote.credit_micros + 1, &quote, false));
479 // A second within 12 months, even the same amount.
480 assert!(needs_reason(quote.credit_micros, &quote, true));
481 // Crediting the whole $90 overage: g1t absorbs the whole $75 cost.
482 assert_eq!(absorbed_by(90_000_000, &quote), 75_000_000);
483 assert_eq!(absorbed_by(quote.credit_micros, &quote), 50_000_000);
484 assert_eq!(absorbed_by(10_000_000, &quote), 0);
485 }
486}