| 1 | //! Whether a workspace may start compute, and holding what it may cost. |
| 2 | //! |
| 3 | //! Every service that starts something that costs g1t real money asks |
| 4 | //! here first (see `g1t_contracts::billing::ReserveArgs`): |
| 5 | //! |
| 6 | //! - **`entitlements`**: the workspace's plan, whether it may start compute |
| 7 | //! at all, its caps (agents at once, a run's time and spend), its |
| 8 | //! ceiling and exposure, and whether compute is paused. |
| 9 | //! - **`reserve`**: holds the work's estimated cost against what may pay |
| 10 | //! for it (the plan's included usage, the trial, the open-source pool, |
| 11 | //! then on-demand room under the ceiling and the spend limit), so starts |
| 12 | //! at the same moment cannot overshoot together. Answers who pays first, |
| 13 | //! or refuses with a stable code and a message for the owner. |
| 14 | //! - **`settle`**: releases the hold. The charge itself goes on the ledger |
| 15 | //! the usual way; a hold never settled lapses after three hours. |
| 16 | //! |
| 17 | //! **No card, no compute.** A free workspace's forge is free, but compute |
| 18 | //! needs the plan, or a card check: it unlocks the one-time trial and g1t's |
| 19 | //! open-source pool (checks, workflows and the merge queue on public |
| 20 | //! repositories). The card check is what keeps g1t's free compute from |
| 21 | //! being mined: one trial per card, and a real person behind each. |
| 22 | //! |
| 23 | //! **Spikes.** An hour's spend above `SPIKE_FACTOR` (5) times the |
| 24 | //! workspace's usual hour over the last week, and at least |
| 25 | //! `SPIKE_FLOOR_MICROS` ($5), pauses new compute until an owner answers: |
| 26 | //! keep going (for 24 hours, or until the hour's spend doubles again) or |
| 27 | //! stop. Runs already under way finish. g1t's own workspaces are watched |
| 28 | //! but never paused. |
| 29 | |
| 30 | use futures_util::future::{try_join, try_join4}; |
| 31 | use g1t_contracts::billing::{ |
| 32 | BillingAccount, ComputeKind, ConfirmSpikeArgs, Entitlements, EntitlementsArgs, LimitState, PaidBy, PlanKind, Reservation, |
| 33 | ReserveArgs, SettleArgs, Spike, UNLIMITED_MICROS, UsageAlert, RESERVATION_HOURS, |
| 34 | }; |
| 35 | use g1t_contracts::time::rfc3339; |
| 36 | use g1t_contracts::{FailureCode, Outcome, new_id}; |
| 37 | use g1t_kit::now_ms; |
| 38 | use serde::Deserialize; |
| 39 | use worker::Result; |
| 40 | use worker::wasm_bindgen::JsValue; |
| 41 | |
| 42 | use crate::Billing; |
| 43 | use crate::credits::{self, left}; |
| 44 | use crate::features::dollars; |
| 45 | use crate::limits::alert_level; |
| 46 | |
| 47 | /// Sandboxes (workflow jobs, checks, the queue, deploy builds) a workspace |
| 48 | /// runs at once in the first month, on the free plan or the trial, and |
| 49 | /// after: room for a workflow's matrix and a deploy side by side, where 2 |
| 50 | /// and 10, sized for g1t's former hosted agents, held CI behind one another. |
| 51 | pub(crate) const FIRST_MONTH_AGENTS: u32 = 20; |
| 52 | pub(crate) const AGENTS: u32 = 40; |
| 53 | /// The longest run in the first month or on the trial, in minutes. |
| 54 | pub(crate) const FIRST_MONTH_MINUTES: u32 = 60; |
| 55 | /// How long "keep going" lifts a spike's pause. |
| 56 | const KEEP_GOING_MS: u64 = 24 * 60 * 60 * 1000; |
| 57 | /// The week a usual hour is measured over. |
| 58 | const WEEK_HOURS: i64 = 7 * 24; |
| 59 | |
| 60 | /// What may pay for reserved work, at price, in the order it pays. |
| 61 | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] |
| 62 | pub(crate) struct Room { |
| 63 | pub credit: i64, |
| 64 | pub trial: i64, |
| 65 | pub oss: i64, |
| 66 | /// Under the ceiling and the spend limit; None: no bound (g1t's own). |
| 67 | pub on_demand: Option<i64>, |
| 68 | } |
| 69 | |
| 70 | /// Why `place` could not hold an estimate. |
| 71 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 72 | pub(crate) enum Short { |
| 73 | /// Nothing left that pays for it. |
| 74 | Empty, |
| 75 | /// Some room, but less than a paid workspace's whole estimate. |
| 76 | TooSmall, |
| 77 | } |
| 78 | |
| 79 | /// Where a new hold of `estimate` goes, given what open holds take |
| 80 | /// (`held`): the source that pays first, and what to hold. Holds fill the |
| 81 | /// sources in order, so the first source with room after them pays first. |
| 82 | /// A paid workspace's whole estimate must fit, so the ceiling cannot be |
| 83 | /// overshot; a free workspace may use its last bit of trial, and what the |
| 84 | /// run costs past it is g1t's. |
| 85 | pub(crate) fn place(room: &Room, held: i64, estimate: i64, whole: bool) -> std::result::Result<(PaidBy, i64), Short> { |
| 86 | let sources = [ |
| 87 | (PaidBy::Credit, room.credit.max(0)), |
| 88 | (PaidBy::Trial, room.trial.max(0)), |
| 89 | (PaidBy::Oss, room.oss.max(0)), |
| 90 | ]; |
| 91 | let pools: i64 = sources.iter().map(|(_, room)| room).sum(); |
| 92 | let remaining = match room.on_demand { |
| 93 | None => i64::MAX, |
| 94 | Some(on_demand) => pools + on_demand.max(0) - held.max(0), |
| 95 | }; |
| 96 | if remaining <= 0 { |
| 97 | return Err(Short::Empty); |
| 98 | } |
| 99 | let estimate = estimate.max(0); |
| 100 | if whole && estimate > remaining { |
| 101 | return Err(Short::TooSmall); |
| 102 | } |
| 103 | let hold = estimate.min(remaining); |
| 104 | let mut end = 0; |
| 105 | for (source, size) in sources { |
| 106 | end += size; |
| 107 | if held.max(0) < end { |
| 108 | return Ok((source, hold)); |
| 109 | } |
| 110 | } |
| 111 | Ok((PaidBy::OnDemand, hold)) |
| 112 | } |
| 113 | |
| 114 | /// Whether the last hour is a spike: above `factor` times the usual hour, |
| 115 | /// and at least `floor`. |
| 116 | pub(crate) fn is_spike(last_hour: i64, usual_hour: i64, factor: i64, floor: i64) -> bool { |
| 117 | last_hour >= floor.max(1) && last_hour > usual_hour.max(0) * factor |
| 118 | } |
| 119 | |
| 120 | /// Whether a spike an owner said to keep going on still lets work start: |
| 121 | /// within its 24 hours, and the hour's spend not doubled again. |
| 122 | pub(crate) fn still_continued(until: Option<&str>, now: &str, hour_at_spike: i64, last_hour: i64) -> bool { |
| 123 | until.is_some_and(|until| now < until) && last_hour < hour_at_spike.max(1) * 2 |
| 124 | } |
| 125 | |
| 126 | /// A workspace's caps, from its plan. |
| 127 | pub(crate) fn caps(plan: PlanKind, first_month: bool, on_trial: bool, agents: Option<u32>) -> (u32, u32) { |
| 128 | let tight = first_month || (plan == PlanKind::Free && on_trial) || plan == PlanKind::Free; |
| 129 | let default_agents = if tight { FIRST_MONTH_AGENTS } else { AGENTS }; |
| 130 | let minutes = if tight { FIRST_MONTH_MINUTES } else { g1t_contracts::guardrails::MAX_MINUTES }; |
| 131 | (agents.unwrap_or(default_agents), minutes) |
| 132 | } |
| 133 | |
| 134 | /// The refusal for a start that cannot be held, with what to do. |
| 135 | pub(crate) fn refusal(code: FailureCode, workspace: &str, kind: ComputeKind, detail: &str) -> Outcome<Reservation> { |
| 136 | let link = format!("/{workspace}/-/billing"); |
| 137 | let what = match kind { |
| 138 | ComputeKind::Agent => "Agents", |
| 139 | ComputeKind::Check => "Checks", |
| 140 | ComputeKind::Workflow => "Workflows", |
| 141 | ComputeKind::Queue => "The merge queue", |
| 142 | ComputeKind::Deploy => "Deployments", |
| 143 | ComputeKind::Embedding => "Semantic search", |
| 144 | }; |
| 145 | let message = match code { |
| 146 | FailureCode::NotPaid if kind.open_source_pool() => format!( |
| 147 | "{what} run in g1t's sandboxes, which cost real money, so they need the g1t plan ($21 a month) or a card check. A card check gives public repositories g1t's open-source pool and starts the $5 trial; it is never charged. Both are at {link}." |
| 148 | ), |
| 149 | FailureCode::NotPaid => format!( |
| 150 | "{what} cost real money to run, so they need the g1t plan ($21 a month, with $10 of usage included) or the one-time $5 trial, which starts with a card check that is never charged. Both are at {link}." |
| 151 | ), |
| 152 | FailureCode::TrialUsed => format!( |
| 153 | "This workspace has used its $5 trial. Start the g1t plan ($21 a month, with $10 of usage included) to keep going: {link}." |
| 154 | ), |
| 155 | FailureCode::OssPoolEmpty => format!( |
| 156 | "g1t's open-source pool for this month is used up{detail}, so checks and workflows on public repositories wait until the 1st. The g1t plan runs them now: {link}." |
| 157 | ), |
| 158 | FailureCode::Limit => format!("{detail} An owner can raise the limit, prepay, or ask g1t for more at {link}."), |
| 159 | FailureCode::Paused => format!("New compute is paused: {detail} An owner can see why and answer at {link}."), |
| 160 | _ => detail.to_owned(), |
| 161 | }; |
| 162 | Outcome::fail(code, message) |
| 163 | } |
| 164 | |
| 165 | #[derive(Deserialize)] |
| 166 | struct SpikeRow { |
| 167 | id: String, |
| 168 | status: String, |
| 169 | hour_micros: i64, |
| 170 | average_micros: i64, |
| 171 | detected_at: String, |
| 172 | decided_by: Option<String>, |
| 173 | decided_at: Option<String>, |
| 174 | until: Option<String>, |
| 175 | } |
| 176 | |
| 177 | impl From<SpikeRow> for Spike { |
| 178 | fn from(row: SpikeRow) -> Self { |
| 179 | Spike { |
| 180 | id: row.id, |
| 181 | status: row.status, |
| 182 | hour_micros: row.hour_micros, |
| 183 | average_micros: row.average_micros, |
| 184 | detected_at: row.detected_at, |
| 185 | decided_by: row.decided_by, |
| 186 | decided_at: row.decided_at, |
| 187 | until: row.until, |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | /// A workspace's pace: the last hour, the usual hour over the last week, |
| 193 | /// the last day, at price (what was charged plus what paid for it first). |
| 194 | #[derive(Clone, Copy, Debug, Default)] |
| 195 | pub(crate) struct Pace { |
| 196 | pub last_hour: i64, |
| 197 | pub usual_hour: i64, |
| 198 | pub last_day: i64, |
| 199 | } |
| 200 | |
| 201 | impl Billing { |
| 202 | /// Spend at price over the last hour, day and week. |
| 203 | pub(crate) async fn pace(&self, workspace: &str) -> Result<Pace> { |
| 204 | #[derive(Deserialize)] |
| 205 | struct Row { |
| 206 | hour: Option<i64>, |
| 207 | day: Option<i64>, |
| 208 | week: Option<i64>, |
| 209 | } |
| 210 | let now = now_ms(); |
| 211 | let hour_ago = rfc3339(now - 60 * 60 * 1000); |
| 212 | let day_ago = rfc3339(now - 24 * 60 * 60 * 1000); |
| 213 | let week_ago = rfc3339(now - 7 * 24 * 60 * 60 * 1000); |
| 214 | let gross = "(-amount_micros + credit_micros + gift_micros + trial_micros + oss_micros + given_micros)"; |
| 215 | let row = self |
| 216 | .db |
| 217 | .prepare(format!( |
| 218 | "SELECT SUM(CASE WHEN created_at >= ?2 THEN {gross} END) AS hour, |
| 219 | SUM(CASE WHEN created_at >= ?3 THEN {gross} END) AS day, |
| 220 | SUM(CASE WHEN created_at < ?2 THEN {gross} END) AS week |
| 221 | FROM ledger WHERE workspace = ?1 AND kind = 'usage' AND created_at >= ?4" |
| 222 | )) |
| 223 | .bind(&[workspace.into(), hour_ago.into(), day_ago.into(), week_ago.into()])? |
| 224 | .first::<Row>(None) |
| 225 | .await?; |
| 226 | Ok(row.map_or_else(Pace::default, |r| Pace { |
| 227 | last_hour: r.hour.unwrap_or(0).max(0), |
| 228 | usual_hour: r.week.unwrap_or(0).max(0) / (WEEK_HOURS - 1), |
| 229 | last_day: r.day.unwrap_or(0).max(0), |
| 230 | })) |
| 231 | } |
| 232 | |
| 233 | /// The workspace's latest spike, if any. |
| 234 | pub(crate) async fn latest_spike(&self, workspace: &str) -> Result<Option<Spike>> { |
| 235 | Ok(self |
| 236 | .db |
| 237 | .prepare( |
| 238 | "SELECT id, status, hour_micros, average_micros, detected_at, decided_by, decided_at, until |
| 239 | FROM spikes WHERE workspace = ? ORDER BY detected_at DESC LIMIT 1", |
| 240 | ) |
| 241 | .bind(&[workspace.into()])? |
| 242 | .first::<SpikeRow>(None) |
| 243 | .await? |
| 244 | .map(Spike::from)) |
| 245 | } |
| 246 | |
| 247 | /// The spike pausing the workspace now, found or new. Never for g1t's |
| 248 | /// own workspaces, which are watched in sudo but never paused. |
| 249 | async fn spike_pause(&self, workspace: &str, plan: PlanKind) -> Result<Option<Spike>> { |
| 250 | let latest = self.latest_spike(workspace).await?; |
| 251 | if let Some(spike) = &latest |
| 252 | && (spike.status == "open" || spike.status == "stopped") { |
| 253 | return Ok(latest); |
| 254 | } |
| 255 | if plan == PlanKind::Internal || !self.charging { |
| 256 | return Ok(None); |
| 257 | } |
| 258 | let pace = self.pace(workspace).await?; |
| 259 | let now = rfc3339(now_ms()); |
| 260 | if let Some(spike) = &latest |
| 261 | && spike.status == "continued" && still_continued(spike.until.as_deref(), &now, spike.hour_micros, pace.last_hour) { |
| 262 | return Ok(None); |
| 263 | } |
| 264 | if !is_spike(pace.last_hour, pace.usual_hour, self.plans.spike_factor, self.plans.spike_floor_micros) { |
| 265 | return Ok(None); |
| 266 | } |
| 267 | let id = new_id("spk", now_ms()); |
| 268 | self.db |
| 269 | .prepare( |
| 270 | "INSERT INTO spikes (id, workspace, status, hour_micros, average_micros, detected_at) |
| 271 | SELECT ?1, ?2, 'open', ?3, ?4, ?5 |
| 272 | WHERE NOT EXISTS (SELECT 1 FROM spikes WHERE workspace = ?2 AND status = 'open')", |
| 273 | ) |
| 274 | .bind(&[id.as_str().into(), workspace.into(), (pace.last_hour as f64).into(), (pace.usual_hour as f64).into(), now.as_str().into()])? |
| 275 | .run() |
| 276 | .await?; |
| 277 | self.latest_spike(workspace).await |
| 278 | } |
| 279 | |
| 280 | /// The alerts a workspace has reached this month: its plan's included |
| 281 | /// usage, its spend limit and g1t's ceiling, from 50%. |
| 282 | pub(crate) async fn alerts_for(&self, workspace: &str) -> Result<Vec<UsageAlert>> { |
| 283 | let account = self.account_of(workspace).await?; |
| 284 | let plan = self.plan_kind_for(workspace, &account).await?; |
| 285 | let has_plan = plan != PlanKind::Free; |
| 286 | let month = credits::month_of(&rfc3339(now_ms())); |
| 287 | let (limit, used) = try_join(self.limit_with(workspace, &account, plan), async { |
| 288 | if has_plan { self.allowance_used("plan_credit", workspace, &month).await } else { Ok(0) } |
| 289 | }) |
| 290 | .await?; |
| 291 | Ok(alerts_from(workspace, &limit, has_plan, used, self.plans.plan_included_micros)) |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | /// The alerts reached, from a limit already worked out, whether the |
| 296 | /// workspace has the plan, and what of its included usage it has used. |
| 297 | fn alerts_from( |
| 298 | workspace: &str, |
| 299 | limit: &g1t_contracts::billing::Limit, |
| 300 | has_plan: bool, |
| 301 | used: i64, |
| 302 | included: i64, |
| 303 | ) -> Vec<UsageAlert> { |
| 304 | let mut alerts = vec![]; |
| 305 | if has_plan && limit.trust != g1t_contracts::billing::Trust::Internal { |
| 306 | let level = alert_level(used, included); |
| 307 | if level > 0 { |
| 308 | alerts.push(UsageAlert { |
| 309 | meter: "included".into(), |
| 310 | level, |
| 311 | used_micros: used, |
| 312 | limit_micros: included, |
| 313 | message: if level >= 100 { |
| 314 | format!("{workspace} has used all {} of this month's included usage. Usage from here is charged at cost plus 20%, up to your spend limit.", dollars(included)) |
| 315 | } else { |
| 316 | format!("{workspace} has used {} of this month's {} included usage ({level}%). Past it, usage is charged at cost plus 20%, up to your spend limit.", dollars(used), dollars(included)) |
| 317 | }, |
| 318 | }); |
| 319 | } |
| 320 | } |
| 321 | if let Some(spend_limit) = limit.spend_limit_micros { |
| 322 | let level = crate::limits::alert_level_in(limit.spent_micros, spend_limit, &limit.alert_levels); |
| 323 | if level > 0 { |
| 324 | alerts.push(UsageAlert { |
| 325 | meter: "spend_limit".into(), |
| 326 | level, |
| 327 | used_micros: limit.spent_micros, |
| 328 | limit_micros: spend_limit, |
| 329 | message: format!( |
| 330 | "{workspace} has spent {} of its {} monthly spend limit ({level}%). At the limit, new sandboxes, builds and agents stop until the month turns or an owner raises it.", |
| 331 | dollars(limit.spent_micros), |
| 332 | dollars(spend_limit) |
| 333 | ), |
| 334 | }); |
| 335 | } |
| 336 | } |
| 337 | if let Some(ceiling) = limit.ceiling_micros.filter(|_| limit.trust != g1t_contracts::billing::Trust::New) { |
| 338 | let level = alert_level(limit.exposure_micros, ceiling); |
| 339 | if level > 0 { |
| 340 | alerts.push(UsageAlert { |
| 341 | meter: "ceiling".into(), |
| 342 | level, |
| 343 | used_micros: limit.exposure_micros, |
| 344 | limit_micros: ceiling, |
| 345 | message: format!( |
| 346 | "{workspace} has {} of usage not yet paid for, of the {} g1t allows ({level}%). With a card on file g1t charges it as the limit nears; prepaying raises it at once.", |
| 347 | dollars(limit.exposure_micros), |
| 348 | dollars(ceiling) |
| 349 | ), |
| 350 | }); |
| 351 | } |
| 352 | } |
| 353 | alerts |
| 354 | } |
| 355 | |
| 356 | impl Billing { |
| 357 | /// Whether a card check was done for the workspace. |
| 358 | pub(crate) async fn card_checked(&self, workspace: &str) -> Result<bool> { |
| 359 | Ok(self |
| 360 | .db |
| 361 | .prepare("SELECT workspace FROM card_checks WHERE workspace = ?") |
| 362 | .bind(&[workspace.into()])? |
| 363 | .first::<serde_json::Value>(None) |
| 364 | .await? |
| 365 | .is_some()) |
| 366 | } |
| 367 | |
| 368 | /// What open reservations hold across `members`, at price. |
| 369 | async fn held(&self, members: &[String]) -> Result<i64> { |
| 370 | #[derive(Deserialize)] |
| 371 | struct Row { |
| 372 | held: Option<i64>, |
| 373 | } |
| 374 | let marks = vec!["?"; members.len().max(1)].join(", "); |
| 375 | let mut values: Vec<JsValue> = members.iter().map(|m| JsValue::from(m.as_str())).collect(); |
| 376 | if values.is_empty() { |
| 377 | values.push("".into()); |
| 378 | } |
| 379 | values.push(rfc3339(now_ms()).into()); |
| 380 | Ok(self |
| 381 | .db |
| 382 | .prepare(format!( |
| 383 | "SELECT SUM(hold_micros) AS held FROM reservations |
| 384 | WHERE workspace IN ({marks}) AND settled_at IS NULL AND expires_at > ?" |
| 385 | )) |
| 386 | .bind(&values)? |
| 387 | .first::<Row>(None) |
| 388 | .await? |
| 389 | .and_then(|r| r.held) |
| 390 | .unwrap_or(0)) |
| 391 | } |
| 392 | |
| 393 | /// Why new compute is paused, if it is: a staff hold, a spike waiting |
| 394 | /// for an owner (or stopped by one), or the limit reached. |
| 395 | async fn pause_reason( |
| 396 | &self, |
| 397 | workspace: &str, |
| 398 | account: &BillingAccount, |
| 399 | plan: PlanKind, |
| 400 | limit: Option<&g1t_contracts::billing::Limit>, |
| 401 | ) -> Result<(Option<String>, Option<Spike>, Option<FailureCode>)> { |
| 402 | if let Some(hold) = account.allowances.hold.as_deref().filter(|h| !h.trim().is_empty()) { |
| 403 | return Ok((Some(format!("g1t staff put a hold on new compute ({}).", hold.trim())), None, Some(FailureCode::Paused))); |
| 404 | } |
| 405 | // A comped account past its monthly budget (`budget`). |
| 406 | if let Some(why) = self.comped_stop(account).await? { |
| 407 | return Ok((Some(why), None, Some(FailureCode::Paused))); |
| 408 | } |
| 409 | let spike = self.spike_pause(workspace, plan).await?; |
| 410 | if let Some(spike) = &spike { |
| 411 | let why = if spike.status == "stopped" { |
| 412 | format!( |
| 413 | "an owner stopped new compute after a spend spike ({} in an hour). An owner can choose Keep going.", |
| 414 | dollars(spike.hour_micros) |
| 415 | ) |
| 416 | } else { |
| 417 | format!( |
| 418 | "{} was spent in an hour, more than {} times the usual {} an hour, so new compute waits for an owner to confirm.", |
| 419 | dollars(spike.hour_micros), |
| 420 | self.plans.spike_factor, |
| 421 | dollars(spike.average_micros) |
| 422 | ) |
| 423 | }; |
| 424 | return Ok((Some(why), Some(spike.clone()), Some(FailureCode::Paused))); |
| 425 | } |
| 426 | let latest = self.latest_spike(workspace).await?; |
| 427 | // A complimentary workspace that stops at its allowance has no |
| 428 | // on-demand usage to limit: the allowance is the bound (`reserve`, |
| 429 | // which passes no limit for it). One charged past it has the plan's. |
| 430 | if plan != PlanKind::Internal && (plan != PlanKind::Complimentary || limit.is_some()) && self.charging { |
| 431 | let worked_out; |
| 432 | let limit = match limit { |
| 433 | Some(limit) => limit, |
| 434 | None => { |
| 435 | worked_out = self.limit_of(workspace).await?; |
| 436 | &worked_out |
| 437 | } |
| 438 | }; |
| 439 | if limit.state == LimitState::Stopped { |
| 440 | return Ok((limit.message.clone(), latest, Some(FailureCode::Limit))); |
| 441 | } |
| 442 | } |
| 443 | Ok((None, latest, None)) |
| 444 | } |
| 445 | |
| 446 | /// `entitlements`: what the workspace may do now. |
| 447 | pub(crate) async fn entitlements(&self, a: EntitlementsArgs) -> Result<Entitlements> { |
| 448 | let workspace = a.workspace.to_lowercase(); |
| 449 | let account = self.account_of(&workspace).await?; |
| 450 | let plan = self.plan_kind_for(&workspace, &account).await?; |
| 451 | let has_plan = plan != PlanKind::Free; |
| 452 | let now = rfc3339(now_ms()); |
| 453 | let month = credits::month_of(&now); |
| 454 | // Every signed-in page asks for this, so what does not need another |
| 455 | // answer is read at once: the limit (and the pause, from it), the |
| 456 | // trial, and the month's counts. |
| 457 | let standing = async { |
| 458 | let limit = self.limit_with(&workspace, &account, plan).await?; |
| 459 | // An enrollment has the plan's limit only when its own card is |
| 460 | // charged past the allowance, as `reserve` decides it. |
| 461 | let limited = plan != PlanKind::Complimentary || match self.complimentary_active(&workspace).await? { |
| 462 | Some(row) if row.charges_past() => self.has_card(&workspace).await?, |
| 463 | _ => false, |
| 464 | }; |
| 465 | let pause = self.pause_reason(&workspace, &account, plan, limited.then_some(&limit)).await?; |
| 466 | Ok::<_, worker::Error>((limit, pause)) |
| 467 | }; |
| 468 | let trial = async { |
| 469 | let (checked, grant) = try_join( |
| 470 | async { |
| 471 | if matches!(plan, PlanKind::Internal | PlanKind::Enterprise) { Ok(true) } else { self.card_checked(&workspace).await } |
| 472 | }, |
| 473 | self.grant_of(&workspace), |
| 474 | ) |
| 475 | .await?; |
| 476 | // A card checked while the month's pool was empty: granted once |
| 477 | // it has room. |
| 478 | let grant = match grant { |
| 479 | Some(grant) => Some(grant), |
| 480 | None if checked && plan == PlanKind::Free && self.trial_allowed(&workspace).await? => self.ensure_grant(&workspace).await?, |
| 481 | None => None, |
| 482 | }; |
| 483 | Ok::<_, worker::Error>((checked, grant)) |
| 484 | }; |
| 485 | let counts = try_join4( |
| 486 | self.private_storage(&workspace), |
| 487 | self.oss_paid(&workspace, &month), |
| 488 | self.held(&account.workspaces), |
| 489 | async { if has_plan { self.allowance_used("plan_credit", &workspace, &month).await } else { Ok(0) } }, |
| 490 | ); |
| 491 | let more = try_join(self.allowance_used("build_seconds", &workspace, &month), self.git_operations_this_month(&workspace)); |
| 492 | let ((limit, (paused, spike, _)), (verified, grant), (stored, oss, held, included_used), (build_seconds, git_operations)) = |
| 493 | try_join4(standing, trial, counts, more).await?; |
| 494 | let trial_left = grant.as_ref().map_or(0, |g| left(g.granted_micros, g.used_micros)); |
| 495 | let first_month = limit.first_month; |
| 496 | let (max_agents, max_minutes) = caps(plan, first_month, trial_left > 0, account.allowances.max_concurrent_agents); |
| 497 | let ceiling = match plan { |
| 498 | PlanKind::Free => 0, |
| 499 | PlanKind::Internal => UNLIMITED_MICROS, |
| 500 | _ => limit.ceiling_micros.unwrap_or(UNLIMITED_MICROS), |
| 501 | }; |
| 502 | // A complimentary enrollment's month (g1t's own may have one too). |
| 503 | let complimentary = match plan { |
| 504 | PlanKind::Complimentary | PlanKind::Internal => match self.complimentary_active(&workspace).await? { |
| 505 | Some(row) => Some(self.complimentary_month(&row, Some(held)).await?), |
| 506 | None => None, |
| 507 | }, |
| 508 | _ => None, |
| 509 | }; |
| 510 | let enrolled = complimentary.as_ref().filter(|_| plan == PlanKind::Complimentary); |
| 511 | // Past its allowance, nothing more that costs money starts (no |
| 512 | // compute; storage and caches past the free amounts refused), unless |
| 513 | // the workspace's own card is charged past it, under its limits. |
| 514 | let charged_past = enrolled.is_some_and(|m| m.charge_past_allowance && m.has_card); |
| 515 | let room = enrolled.is_none_or(|m| m.unlimited || m.remaining_micros > 0) || charged_past; |
| 516 | let alerts = match enrolled { |
| 517 | Some(month) => { |
| 518 | let mut alerts: Vec<UsageAlert> = crate::complimentary::alert(&workspace, month).into_iter().collect(); |
| 519 | if charged_past { |
| 520 | alerts.extend(alerts_from(&workspace, &limit, false, 0, 0)); |
| 521 | } |
| 522 | alerts |
| 523 | } |
| 524 | None => alerts_from(&workspace, &limit, has_plan, included_used, self.plans.plan_included_micros), |
| 525 | }; |
| 526 | let paused = paused.or_else(|| { |
| 527 | enrolled.filter(|_| !room).map(|m| { |
| 528 | if m.charge_past_allowance { |
| 529 | "This month's complimentary allowance is used up. Add a card on Billing to keep going: usage past it is charged to the card at the usual prices.".to_owned() |
| 530 | } else { |
| 531 | format!("This month's complimentary allowance is used up. New work that costs money starts again on {}.", &m.resets_at[..10]) |
| 532 | } |
| 533 | }) |
| 534 | }); |
| 535 | let ceiling = match enrolled { |
| 536 | Some(m) if m.unlimited => UNLIMITED_MICROS, |
| 537 | Some(_) if charged_past => limit.ceiling_micros.unwrap_or(UNLIMITED_MICROS), |
| 538 | Some(_) => 0, |
| 539 | None => ceiling, |
| 540 | }; |
| 541 | let (included_micros, included_used) = match enrolled { |
| 542 | Some(m) => (if m.unlimited { UNLIMITED_MICROS } else { m.allowance_micros }, m.used_micros), |
| 543 | None => (if has_plan { self.plans.plan_included_micros } else { 0 }, included_used), |
| 544 | }; |
| 545 | let has_plan = has_plan && room; |
| 546 | Ok(Entitlements { |
| 547 | plan, |
| 548 | compute: has_plan || trial_left > 0, |
| 549 | trial_micros_left: trial_left, |
| 550 | trial_verified: verified, |
| 551 | first_month, |
| 552 | max_concurrent_agents: max_agents, |
| 553 | max_run_minutes: max_minutes, |
| 554 | ceiling_micros: ceiling, |
| 555 | exposure_micros: limit.exposure_micros, |
| 556 | paused, |
| 557 | held: account.allowances.hold.as_deref().is_some_and(|hold| !hold.trim().is_empty()), |
| 558 | held_micros: held, |
| 559 | prepaid_micros: limit.prepaid_micros, |
| 560 | included_micros, |
| 561 | included_used_micros: included_used, |
| 562 | audit_retention_days: crate::retention::effective_days(&self.plans, plan, account.allowances.audit_retention_days), |
| 563 | audit_retention_custom: account.allowances.audit_retention_days.is_some(), |
| 564 | free_private_storage_bytes: self.plans.free_storage_bytes, |
| 565 | // For the packages service: on the plan nothing is refused; without it, |
| 566 | // pushes past these amounts are. |
| 567 | has_plan, |
| 568 | package_public_free_bytes: self.plans.public_package_free_bytes, |
| 569 | package_private_free_bytes: self.plans.private_package_free_bytes, |
| 570 | private_storage_bytes: stored, |
| 571 | oss_paid_micros: oss, |
| 572 | build_seconds_used: build_seconds.max(0) as u32, |
| 573 | git_operations, |
| 574 | git_operations_included: self.plans.git_included, |
| 575 | min_charge_micros: self.plans.min_charge_micros, |
| 576 | spike, |
| 577 | alerts, |
| 578 | complimentary, |
| 579 | workspace, |
| 580 | }) |
| 581 | } |
| 582 | |
| 583 | /// `reserve`: holds a start's estimated cost, or says why not. |
| 584 | pub(crate) async fn reserve(&self, a: ReserveArgs) -> Result<Outcome<Reservation>> { |
| 585 | let workspace = a.workspace.to_lowercase(); |
| 586 | let now = now_ms(); |
| 587 | let expires_at = rfc3339(now + RESERVATION_HOURS * 60 * 60 * 1000); |
| 588 | let repo = format!("{}/{}", a.repo.namespace, a.repo.name).to_lowercase(); |
| 589 | // A platform pause holds for everyone, a g1t that does not charge |
| 590 | // included (platform.rs): kept 30 seconds in the isolate. |
| 591 | if let Some(why) = self.platform_refuses(a.kind).await { |
| 592 | return Ok(Outcome::fail(FailureCode::Paused, why)); |
| 593 | } |
| 594 | // A g1t that does not charge holds nothing. |
| 595 | if !self.charging { |
| 596 | return Ok(Outcome::Ok(Reservation { id: new_id("rsv", now), paid_by: PaidBy::OnDemand, held_micros: 0, expires_at })); |
| 597 | } |
| 598 | let account = self.account_of(&workspace).await?; |
| 599 | let plan = self.plan_kind_for(&workspace, &account).await?; |
| 600 | // g1t's own caps (`budget`), in their own words: a comped account's |
| 601 | // monthly budget, and the daily breaker. |
| 602 | if let Some(why) = self.comped_stop(&account).await? { |
| 603 | return Ok(Outcome::fail(FailureCode::Paused, why)); |
| 604 | } |
| 605 | if let Some(why) = self.breaker_refuses(plan, &account, a.kind).await? { |
| 606 | return Ok(Outcome::fail(FailureCode::Paused, why)); |
| 607 | } |
| 608 | // An enrollment: what pays first, and whether its own card is charged |
| 609 | // past it (`charge`), with a card to charge. |
| 610 | let enrollment = if plan == PlanKind::Complimentary { self.complimentary_active(&workspace).await? } else { None }; |
| 611 | let charges_past = match &enrollment { |
| 612 | Some(row) if row.charges_past() => self.has_card(&workspace).await?, |
| 613 | _ => false, |
| 614 | }; |
| 615 | let limit = if plan == PlanKind::Internal || (plan == PlanKind::Complimentary && !charges_past) { |
| 616 | None |
| 617 | } else { |
| 618 | Some(self.limit_with(&workspace, &account, plan).await?) |
| 619 | }; |
| 620 | let (paused, _, code) = self.pause_reason(&workspace, &account, plan, limit.as_ref()).await?; |
| 621 | if let (Some(why), Some(code)) = (paused, code) { |
| 622 | return Ok(refusal(code, &workspace, a.kind, &why)); |
| 623 | } |
| 624 | let month = credits::month_of(&rfc3339(now)); |
| 625 | let estimate = credits::with_margin(a.estimate_micros, self.margin_percent); |
| 626 | let verified = matches!(plan, PlanKind::Internal | PlanKind::Enterprise | PlanKind::Paid | PlanKind::Complimentary) |
| 627 | || self.card_checked(&workspace).await?; |
| 628 | let has_plan = plan != PlanKind::Free; |
| 629 | // A complimentary allowance is the only thing that pays: what is |
| 630 | // left after this month's use and its metered usage not on the |
| 631 | // ledger yet; `place` takes off what running work holds. |
| 632 | let enrolled = match &enrollment { |
| 633 | Some(row) => Some(self.complimentary_month(row, Some(0)).await?), |
| 634 | None => None, |
| 635 | }; |
| 636 | let credit = if let Some(month) = &enrolled { |
| 637 | if month.unlimited { 0 } else { month.remaining_micros } |
| 638 | } else if has_plan { |
| 639 | left(self.plans.plan_included_micros, self.allowance_used("plan_credit", &workspace, &month).await?) |
| 640 | } else { |
| 641 | 0 |
| 642 | }; |
| 643 | let trial = if enrolled.is_some() || a.kind == ComputeKind::Deploy || !verified { |
| 644 | 0 |
| 645 | } else { |
| 646 | self.grant_of(&workspace).await?.map_or(0, |g| left(g.granted_micros, g.used_micros)) |
| 647 | }; |
| 648 | let oss_eligible = enrolled.is_none() && a.public && a.kind.open_source_pool() && verified; |
| 649 | let oss = if oss_eligible { self.oss_left(&workspace, &repo, &month).await? } else { 0 }; |
| 650 | let on_demand = match plan { |
| 651 | PlanKind::Free => Some(0), |
| 652 | PlanKind::Internal => None, |
| 653 | // Unlimited is unbounded; one that stops has nothing past the |
| 654 | // allowance; one charged past it has the plan's limits. |
| 655 | PlanKind::Complimentary if enrolled.as_ref().is_some_and(|m| m.unlimited) => None, |
| 656 | PlanKind::Complimentary if !charges_past => Some(0), |
| 657 | _ => { |
| 658 | let Some(limit) = &limit else { unreachable!("only g1t's own and complimentary workspaces skip the limit") }; |
| 659 | let under_ceiling = limit.ceiling_micros.map(|c| (c - limit.exposure_micros).max(0)); |
| 660 | let under_spend = limit.spend_limit_micros.map(|s| (s - limit.spent_micros).max(0)); |
| 661 | match (under_ceiling, under_spend) { |
| 662 | (Some(c), Some(s)) => Some(c.min(s)), |
| 663 | (c, s) => c.or(s), |
| 664 | } |
| 665 | } |
| 666 | }; |
| 667 | let room = Room { credit, trial, oss, on_demand }; |
| 668 | // Optimistic: what was held is read, and the hold is written only if |
| 669 | // nothing was held meanwhile; otherwise read again. |
| 670 | for _ in 0..4 { |
| 671 | let held = self.held(&account.workspaces).await?; |
| 672 | let (paid_by, hold) = match place(&room, held, estimate, has_plan) { |
| 673 | Ok(placed) => placed, |
| 674 | Err(short) => return self.short(&workspace, plan, &a, verified, &room, short).await, |
| 675 | }; |
| 676 | let id = new_id("rsv", now); |
| 677 | let mut values: Vec<JsValue> = vec![ |
| 678 | id.as_str().into(), |
| 679 | workspace.as_str().into(), |
| 680 | repo.as_str().into(), |
| 681 | a.kind.as_str().into(), |
| 682 | u8::from(a.public).into(), |
| 683 | (a.estimate_micros.max(0) as f64).into(), |
| 684 | (hold as f64).into(), |
| 685 | paid_by_text(paid_by).into(), |
| 686 | rfc3339(now).into(), |
| 687 | expires_at.as_str().into(), |
| 688 | (held as f64).into(), |
| 689 | ]; |
| 690 | values.extend(account.workspaces.iter().map(|w| JsValue::from(w.as_str()))); |
| 691 | if account.workspaces.is_empty() { |
| 692 | values.push("".into()); |
| 693 | } |
| 694 | let placed = self |
| 695 | .db |
| 696 | .prepare(format!( |
| 697 | "INSERT INTO reservations (id, workspace, repo, kind, public, estimate_micros, hold_micros, paid_by, created_at, expires_at) |
| 698 | SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10 |
| 699 | WHERE (SELECT COALESCE(SUM(hold_micros), 0) FROM reservations |
| 700 | WHERE workspace IN ({marks}) AND settled_at IS NULL AND expires_at > ?9) = ?11 |
| 701 | RETURNING id", |
| 702 | marks = (12..12 + account.workspaces.len().max(1)).map(|i| format!("?{i}")).collect::<Vec<_>>().join(", ") |
| 703 | )) |
| 704 | .bind(&values)? |
| 705 | .first::<serde_json::Value>(None) |
| 706 | .await?; |
| 707 | if placed.is_some() { |
| 708 | // What is held, at cost, as it was asked. |
| 709 | let held_cost = if hold >= estimate { a.estimate_micros.max(0) } else { hold * 100 / i64::from(100 + self.margin_percent) }; |
| 710 | return Ok(Outcome::Ok(Reservation { id, paid_by, held_micros: held_cost, expires_at })); |
| 711 | } |
| 712 | } |
| 713 | Ok(refusal(FailureCode::Limit, &workspace, a.kind, "Too many starts at once to hold this one; try again in a moment.")) |
| 714 | } |
| 715 | |
| 716 | /// The refusal for a start nothing pays for. |
| 717 | async fn short( |
| 718 | &self, |
| 719 | workspace: &str, |
| 720 | plan: PlanKind, |
| 721 | a: &ReserveArgs, |
| 722 | verified: bool, |
| 723 | room: &Room, |
| 724 | short: Short, |
| 725 | ) -> Result<Outcome<Reservation>> { |
| 726 | let enrollment = if plan == PlanKind::Complimentary { self.complimentary_active(workspace).await? } else { None }; |
| 727 | if enrollment.as_ref().is_some_and(|row| row.charges_past()) && self.has_card(workspace).await? { |
| 728 | // Past the allowance on its own card: the plan's limit refuses. |
| 729 | let limit = self.limit_of(workspace).await?; |
| 730 | let detail = limit.message.unwrap_or_else(|| "The workspace reached its spend limit for this month.".to_owned()); |
| 731 | return Ok(refusal(FailureCode::Limit, workspace, a.kind, &format!("Its complimentary allowance is used up and {detail}"))); |
| 732 | } |
| 733 | if enrollment.as_ref().is_some_and(|row| row.charges_past()) { |
| 734 | return Ok(refusal( |
| 735 | FailureCode::Limit, |
| 736 | workspace, |
| 737 | a.kind, |
| 738 | "This month's complimentary allowance is used up. Add a card on Billing to keep going: usage past the allowance is charged to it at the usual prices, up to your spend limit.", |
| 739 | )); |
| 740 | } |
| 741 | if plan == PlanKind::Complimentary { |
| 742 | let month = credits::month_of(&rfc3339(now_ms())); |
| 743 | let detail = match short { |
| 744 | Short::TooSmall => format!( |
| 745 | "About {} of this month's complimentary allowance is left after running work, less than this would cost.", |
| 746 | dollars(room.credit) |
| 747 | ), |
| 748 | Short::Empty => "This month's complimentary allowance is used up.".to_owned(), |
| 749 | }; |
| 750 | return Ok(refusal( |
| 751 | FailureCode::Limit, |
| 752 | workspace, |
| 753 | a.kind, |
| 754 | &format!( |
| 755 | "{detail} It starts again on {}. An owner can start the g1t plan from Billing to keep going now.", |
| 756 | &credits::next_month_start(&month)[..10] |
| 757 | ), |
| 758 | )); |
| 759 | } |
| 760 | if plan != PlanKind::Free { |
| 761 | let limit = self.limit_of(workspace).await?; |
| 762 | let detail = match short { |
| 763 | Short::TooSmall => format!( |
| 764 | "This would take the workspace past its limit: about {} more could start now ({} spent of a {} spend limit, {} not yet paid of the {} g1t allows).", |
| 765 | dollars(room.credit + room.trial + room.oss + room.on_demand.unwrap_or(0)), |
| 766 | dollars(limit.spent_micros), |
| 767 | dollars(limit.spend_limit_micros.unwrap_or_default()), |
| 768 | dollars(limit.exposure_micros), |
| 769 | dollars(limit.ceiling_micros.unwrap_or_default()), |
| 770 | ), |
| 771 | Short::Empty => limit.message.unwrap_or_else(|| "The workspace reached its limit for this month.".to_owned()), |
| 772 | }; |
| 773 | return Ok(refusal(FailureCode::Limit, workspace, a.kind, &detail)); |
| 774 | } |
| 775 | if !verified { |
| 776 | return Ok(refusal(FailureCode::NotPaid, workspace, a.kind, "")); |
| 777 | } |
| 778 | let oss_eligible = a.public && a.kind.open_source_pool(); |
| 779 | let trial = self.grant_of(workspace).await?; |
| 780 | if oss_eligible && room.oss == 0 { |
| 781 | let month = credits::month_of(&rfc3339(now_ms())); |
| 782 | let repo = format!("{}/{}", a.repo.namespace, a.repo.name).to_lowercase(); |
| 783 | let pool = left(self.plans.oss_pool_micros, self.allowance_used("oss_pool", "", &month).await?); |
| 784 | let detail = if pool > 0 { format!(" for {repo} (its share is {})", dollars(self.oss_repo_cap(workspace).await?)) } else { String::new() }; |
| 785 | if trial.as_ref().is_none_or(|g| g.used_micros >= g.granted_micros) { |
| 786 | return Ok(refusal(FailureCode::OssPoolEmpty, workspace, a.kind, &detail)); |
| 787 | } |
| 788 | } |
| 789 | match trial { |
| 790 | Some(grant) if grant.used_micros >= grant.granted_micros => Ok(refusal(FailureCode::TrialUsed, workspace, a.kind, "")), |
| 791 | _ => Ok(refusal(FailureCode::NotPaid, workspace, a.kind, "")), |
| 792 | } |
| 793 | } |
| 794 | |
| 795 | /// `settle`: releases a hold. |
| 796 | pub(crate) async fn settle_reservation(&self, a: SettleArgs) -> Result<Outcome<bool>> { |
| 797 | let now = rfc3339(now_ms()); |
| 798 | let settled = self |
| 799 | .db |
| 800 | .prepare( |
| 801 | "UPDATE reservations SET settled_at = ?1, actual_micros = ?2 |
| 802 | WHERE id = ?3 AND settled_at IS NULL AND expires_at > ?1 RETURNING id", |
| 803 | ) |
| 804 | .bind(&[now.as_str().into(), (a.actual_micros.max(0) as f64).into(), a.reservation_id.as_str().into()])? |
| 805 | .first::<serde_json::Value>(None) |
| 806 | .await?; |
| 807 | Ok(Outcome::Ok(settled.is_some())) |
| 808 | } |
| 809 | |
| 810 | /// Clears reservations long settled or lapsed. |
| 811 | pub(crate) async fn sweep_reservations(&self) -> Result<()> { |
| 812 | let week_ago = rfc3339(now_ms() - 7 * 24 * 60 * 60 * 1000); |
| 813 | self.db |
| 814 | .prepare("DELETE FROM reservations WHERE expires_at < ?1") |
| 815 | .bind(&[week_ago.into()])? |
| 816 | .run() |
| 817 | .await?; |
| 818 | Ok(()) |
| 819 | } |
| 820 | |
| 821 | /// `confirm_spike`: an owner keeps going, or stops. |
| 822 | pub(crate) async fn confirm_spike(&self, a: ConfirmSpikeArgs) -> Result<Outcome<Entitlements>> { |
| 823 | let workspace = a.workspace.to_lowercase(); |
| 824 | if !a.actor.manages_billing(&workspace) { |
| 825 | return Ok(Outcome::fail(FailureCode::Forbidden, "Only an owner or a billing manager can answer a spend spike.")); |
| 826 | } |
| 827 | let Some(spike) = self.latest_spike(&workspace).await? else { |
| 828 | return Ok(Outcome::fail(FailureCode::NotFound, "There is no spend spike to answer.")); |
| 829 | }; |
| 830 | let now = now_ms(); |
| 831 | let (status, until) = if a.keep_going { ("continued", Some(rfc3339(now + KEEP_GOING_MS))) } else { ("stopped", None) }; |
| 832 | self.db |
| 833 | .prepare("UPDATE spikes SET status = ?1, decided_by = ?2, decided_at = ?3, until = ?4 WHERE id = ?5") |
| 834 | .bind(&[ |
| 835 | status.into(), |
| 836 | a.actor.username.as_str().into(), |
| 837 | rfc3339(now).into(), |
| 838 | crate::optional(until.as_deref()), |
| 839 | spike.id.as_str().into(), |
| 840 | ])? |
| 841 | .run() |
| 842 | .await?; |
| 843 | let account = self.account_of(&workspace).await?; |
| 844 | self.audit( |
| 845 | &account.id, |
| 846 | "spike", |
| 847 | &format!("{workspace}: {} after {} in an hour", if a.keep_going { "kept going" } else { "stopped" }, dollars(spike.hour_micros)), |
| 848 | &a.actor.username, |
| 849 | ) |
| 850 | .await?; |
| 851 | Ok(Outcome::Ok(self.entitlements(EntitlementsArgs { workspace }).await?)) |
| 852 | } |
| 853 | |
| 854 | /// Emails owners about spikes that paused their workspace, once each. |
| 855 | pub(crate) async fn tell_spikes(&self, identity: &worker::Fetcher) -> Result<()> { |
| 856 | #[derive(Deserialize)] |
| 857 | struct Open { |
| 858 | id: String, |
| 859 | workspace: String, |
| 860 | hour_micros: i64, |
| 861 | average_micros: i64, |
| 862 | } |
| 863 | let open = self |
| 864 | .db |
| 865 | .prepare("SELECT id, workspace, hour_micros, average_micros FROM spikes WHERE status = 'open' AND told_at IS NULL LIMIT 20") |
| 866 | .all() |
| 867 | .await? |
| 868 | .results::<Open>()?; |
| 869 | for spike in open { |
| 870 | let workspace = &spike.workspace; |
| 871 | let intro = format!( |
| 872 | "{workspace} spent {} in the last hour, more than {} times its usual {} an hour, so g1t paused new sandboxes, agents and builds until an owner confirms. Runs already going finish. If this was meant, choose Keep going and nothing pauses for 24 hours unless the hour's spend doubles again. If not, choose Stop; and if it was a mistake, tell g1t from the billing page.", |
| 873 | dollars(spike.hour_micros), |
| 874 | self.plans.spike_factor, |
| 875 | dollars(spike.average_micros) |
| 876 | ); |
| 877 | let link = format!("https://g1t.sh/{workspace}/-/billing"); |
| 878 | if crate::limits::notify(identity, workspace, &format!("g1t: spending on {workspace} spiked, so new work is paused"), &intro, "Keep going or stop", &link).await { |
| 879 | self.db |
| 880 | .prepare("UPDATE spikes SET told_at = ? WHERE id = ?") |
| 881 | .bind(&[rfc3339(now_ms()).into(), spike.id.as_str().into()])? |
| 882 | .run() |
| 883 | .await?; |
| 884 | } |
| 885 | } |
| 886 | Ok(()) |
| 887 | } |
| 888 | |
| 889 | /// What the workspace's private repositories held at the last measure. |
| 890 | pub(crate) async fn private_storage(&self, workspace: &str) -> Result<i64> { |
| 891 | #[derive(Deserialize)] |
| 892 | struct Stored { |
| 893 | private_bytes: Option<i64>, |
| 894 | } |
| 895 | Ok(self |
| 896 | .db |
| 897 | .prepare("SELECT private_bytes FROM storage_days WHERE workspace = ? ORDER BY day DESC LIMIT 1") |
| 898 | .bind(&[workspace.into()])? |
| 899 | .first::<Stored>(None) |
| 900 | .await? |
| 901 | .and_then(|s| s.private_bytes) |
| 902 | .unwrap_or(0)) |
| 903 | } |
| 904 | |
| 905 | /// What g1t's open-source pool paid for the workspace in `month`. |
| 906 | async fn oss_paid(&self, workspace: &str, month: &str) -> Result<i64> { |
| 907 | #[derive(Deserialize)] |
| 908 | struct Sum { |
| 909 | micros: Option<i64>, |
| 910 | } |
| 911 | Ok(self |
| 912 | .db |
| 913 | .prepare("SELECT SUM(oss_micros) AS micros FROM ledger WHERE workspace = ? AND created_at >= ?") |
| 914 | .bind(&[workspace.into(), format!("{month}-01").into()])? |
| 915 | .first::<Sum>(None) |
| 916 | .await? |
| 917 | .and_then(|s| s.micros) |
| 918 | .unwrap_or(0)) |
| 919 | } |
| 920 | } |
| 921 | |
| 922 | pub(crate) fn paid_by_text(paid_by: PaidBy) -> &'static str { |
| 923 | match paid_by { |
| 924 | PaidBy::Credit => "credit", |
| 925 | PaidBy::Trial => "trial", |
| 926 | PaidBy::Oss => "oss", |
| 927 | PaidBy::OnDemand => "on_demand", |
| 928 | } |
| 929 | } |
| 930 | |
| 931 | #[cfg(test)] |
| 932 | mod tests { |
| 933 | use super::*; |
| 934 | |
| 935 | fn limit(trust: g1t_contracts::billing::Trust, spent: i64, spend_limit: Option<i64>, exposure: i64, ceiling: Option<i64>) -> g1t_contracts::billing::Limit { |
| 936 | g1t_contracts::billing::Limit { |
| 937 | workspace: "acme".into(), |
| 938 | account: "acc_acme".into(), |
| 939 | account_name: "acme".into(), |
| 940 | trust, |
| 941 | exposure_micros: exposure, |
| 942 | ceiling_micros: ceiling, |
| 943 | trust_ceiling_micros: ceiling, |
| 944 | spend_limit_micros: spend_limit, |
| 945 | state: LimitState::Ok, |
| 946 | message: None, |
| 947 | spent_micros: spent, |
| 948 | default_spend_limit: false, |
| 949 | available_micros: None, |
| 950 | growth: None, |
| 951 | prepaid_micros: 0, |
| 952 | max_ceiling_micros: None, |
| 953 | raise_once_micros: None, |
| 954 | raised_at: None, |
| 955 | first_month: false, |
| 956 | alert_levels: crate::limits::ALERT_LEVELS.to_vec(), |
| 957 | pause_at_limit: true, |
| 958 | budget_webhook: None, |
| 959 | } |
| 960 | } |
| 961 | |
| 962 | #[test] |
| 963 | fn alerts_come_from_the_answers_already_read() { |
| 964 | use g1t_contracts::billing::Trust; |
| 965 | let meters = |alerts: Vec<UsageAlert>| alerts.into_iter().map(|a| (a.meter, a.level)).collect::<Vec<_>>(); |
| 966 | // On the plan: included usage at 90%, the spend limit at 50%, the ceiling at 75%. |
| 967 | let paid = limit(Trust::Paid, 100_000_000, Some(200_000_000), 75_000_000, Some(100_000_000)); |
| 968 | assert_eq!( |
| 969 | meters(alerts_from("acme", &paid, true, 9_000_000, 10_000_000)), |
| 970 | vec![("included".to_owned(), 90), ("spend_limit".to_owned(), 50), ("ceiling".to_owned(), 75)] |
| 971 | ); |
| 972 | // Without the plan, what was used of the included usage is not an alert. |
| 973 | assert_eq!(meters(alerts_from("acme", &paid, false, 9_000_000, 10_000_000)).len(), 2); |
| 974 | // g1t's own workspaces have no included usage to warn of, and a new |
| 975 | // workspace's ceiling is not one either. |
| 976 | let internal = limit(Trust::Internal, 0, None, 0, None); |
| 977 | assert!(alerts_from("acme", &internal, true, 10_000_000, 10_000_000).is_empty()); |
| 978 | let new = limit(Trust::New, 0, None, 3_000_000, Some(3_000_000)); |
| 979 | assert!(alerts_from("acme", &new, false, 0, 10_000_000).is_empty()); |
| 980 | } |
| 981 | |
| 982 | fn paid(credit: i64, on_demand: i64) -> Room { |
| 983 | Room { credit, trial: 0, oss: 0, on_demand: Some(on_demand) } |
| 984 | } |
| 985 | |
| 986 | #[test] |
| 987 | fn included_usage_pays_first_then_on_demand() { |
| 988 | // $10 included, $100 under the ceiling, nothing held: included pays first. |
| 989 | assert_eq!(place(&paid(10_000_000, 100_000_000), 0, 2_400_000, true), Ok((PaidBy::Credit, 2_400_000))); |
| 990 | // Holds already cover the included usage: on demand. |
| 991 | assert_eq!(place(&paid(10_000_000, 100_000_000), 10_000_000, 2_400_000, true), Ok((PaidBy::OnDemand, 2_400_000))); |
| 992 | // A free workspace on its trial. |
| 993 | let trial = Room { trial: 5_000_000, on_demand: Some(0), ..Room::default() }; |
| 994 | assert_eq!(place(&trial, 0, 2_400_000, false), Ok((PaidBy::Trial, 2_400_000))); |
| 995 | // A public repository's checks, from the pool. |
| 996 | let pool = Room { oss: 2_000_000, on_demand: Some(0), ..Room::default() }; |
| 997 | assert_eq!(place(&pool, 0, 600_000, false), Ok((PaidBy::Oss, 600_000))); |
| 998 | } |
| 999 | |
| 1000 | #[test] |
| 1001 | fn a_paid_workspace_is_stopped_only_by_its_limit_never_by_a_count() { |
| 1002 | // The included $10 is gone and the workspace has already built, |
| 1003 | // served and stored far past what used to be quotas: the next start |
| 1004 | // still goes on demand, as long as its spend limit has room. |
| 1005 | let room = paid(0, 250_000_000); |
| 1006 | let held = 0; |
| 1007 | for start in 0..1_000 { |
| 1008 | let placed = place(&room, held + start * 100_000, 100_000, true); |
| 1009 | assert_eq!(placed, Ok((PaidBy::OnDemand, 100_000))); |
| 1010 | } |
| 1011 | // Only at its limit does it stop, with the limit's refusal. |
| 1012 | assert_eq!(place(&room, 250_000_000, 100_000, true), Err(Short::Empty)); |
| 1013 | // A free workspace has no on-demand room at all: with no trial or |
| 1014 | // pool left, nothing starts (`short` says NotPaid or TrialUsed). |
| 1015 | let free = Room { on_demand: Some(0), ..Room::default() }; |
| 1016 | assert_eq!(place(&free, 0, 100_000, false), Err(Short::Empty)); |
| 1017 | // g1t's own workspaces: no limit. |
| 1018 | let internal = Room { on_demand: None, ..Room::default() }; |
| 1019 | assert_eq!(place(&internal, i64::MAX / 2, 100_000, true), Ok((PaidBy::OnDemand, 100_000))); |
| 1020 | } |
| 1021 | |
| 1022 | #[test] |
| 1023 | fn concurrent_starts_cannot_overshoot_the_ceiling() { |
| 1024 | // $5 of room in all, and each start may cost up to $2.40. |
| 1025 | let room = paid(0, 5_000_000); |
| 1026 | let mut held = 0; |
| 1027 | let mut started = 0; |
| 1028 | for _ in 0..5 { |
| 1029 | match place(&room, held, 2_400_000, true) { |
| 1030 | Ok((_, hold)) => { |
| 1031 | held += hold; |
| 1032 | started += 1; |
| 1033 | } |
| 1034 | Err(short) => assert_eq!(short, Short::TooSmall), |
| 1035 | } |
| 1036 | } |
| 1037 | assert_eq!(started, 2); |
| 1038 | assert!(held <= 5_000_000); |
| 1039 | // Once the first settles, a third fits. |
| 1040 | assert!(place(&room, held - 2_400_000, 2_400_000, true).is_ok()); |
| 1041 | // Nothing left at all. |
| 1042 | assert_eq!(place(&room, 5_000_000, 1, true), Err(Short::Empty)); |
| 1043 | } |
| 1044 | |
| 1045 | #[test] |
| 1046 | fn a_free_workspace_may_use_its_last_bit_of_trial() { |
| 1047 | let room = Room { trial: 300_000, on_demand: Some(0), ..Room::default() }; |
| 1048 | // The whole estimate does not fit, but what is left is held. |
| 1049 | assert_eq!(place(&room, 0, 2_400_000, false), Ok((PaidBy::Trial, 300_000))); |
| 1050 | // Once it is held, nothing more starts. |
| 1051 | assert_eq!(place(&room, 300_000, 2_400_000, false), Err(Short::Empty)); |
| 1052 | // Nothing at all: no plan, no trial, no pool. |
| 1053 | assert_eq!(place(&Room { on_demand: Some(0), ..Room::default() }, 0, 1, false), Err(Short::Empty)); |
| 1054 | } |
| 1055 | |
| 1056 | #[test] |
| 1057 | fn g1ts_own_workspaces_are_never_short() { |
| 1058 | let room = Room { credit: 10_000_000, on_demand: None, ..Room::default() }; |
| 1059 | assert_eq!(place(&room, 50_000_000_000, 2_400_000, true), Ok((PaidBy::OnDemand, 2_400_000))); |
| 1060 | assert_eq!(place(&room, 0, 2_400_000, true), Ok((PaidBy::Credit, 2_400_000))); |
| 1061 | } |
| 1062 | |
| 1063 | #[test] |
| 1064 | fn a_spike_is_five_times_the_usual_hour_and_at_least_five_dollars() { |
| 1065 | let (factor, floor) = (5, 5_000_000); |
| 1066 | // A new workspace with no history: $5 in an hour is a spike, $4 is not. |
| 1067 | assert!(is_spike(5_000_000, 0, factor, floor)); |
| 1068 | assert!(!is_spike(4_000_000, 0, factor, floor)); |
| 1069 | // Usually $2 an hour: $10 is not above five times, $10.01 is. |
| 1070 | assert!(!is_spike(10_000_000, 2_000_000, factor, floor)); |
| 1071 | assert!(is_spike(10_010_000, 2_000_000, factor, floor)); |
| 1072 | // A busy workspace at its usual pace is never a spike. |
| 1073 | assert!(!is_spike(40_000_000, 30_000_000, factor, floor)); |
| 1074 | } |
| 1075 | |
| 1076 | #[test] |
| 1077 | fn keep_going_lasts_a_day_or_until_spend_doubles() { |
| 1078 | let until = "2026-10-06T12:00:00Z"; |
| 1079 | assert!(still_continued(Some(until), "2026-10-06T11:00:00Z", 6_000_000, 8_000_000)); |
| 1080 | // Doubled again: paused again. |
| 1081 | assert!(!still_continued(Some(until), "2026-10-06T11:00:00Z", 6_000_000, 12_000_000)); |
| 1082 | // A day later: watched afresh. |
| 1083 | assert!(!still_continued(Some(until), "2026-10-06T12:00:01Z", 6_000_000, 1_000_000)); |
| 1084 | assert!(!still_continued(None, "2026-10-06T11:00:00Z", 6_000_000, 1)); |
| 1085 | } |
| 1086 | |
| 1087 | #[test] |
| 1088 | fn caps_are_tight_in_the_first_month_and_on_the_trial() { |
| 1089 | assert_eq!(caps(PlanKind::Paid, true, false, None), (20, 60)); |
| 1090 | assert_eq!(caps(PlanKind::Free, false, true, None), (20, 60)); |
| 1091 | assert_eq!(caps(PlanKind::Paid, false, false, None), (40, g1t_contracts::guardrails::MAX_MINUTES)); |
| 1092 | assert_eq!(caps(PlanKind::Internal, false, false, None).0, 40); |
| 1093 | // Staff can set agents at once. |
| 1094 | assert_eq!(caps(PlanKind::Paid, true, false, Some(6)).0, 6); |
| 1095 | } |
| 1096 | |
| 1097 | #[test] |
| 1098 | fn every_refusal_says_what_to_do_and_where() { |
| 1099 | for code in [FailureCode::NotPaid, FailureCode::TrialUsed, FailureCode::OssPoolEmpty, FailureCode::Limit, FailureCode::Paused] { |
| 1100 | let Outcome::Fail(failure) = refusal(code, "acme", ComputeKind::Check, "Detail.") else { panic!() }; |
| 1101 | assert_eq!(failure.code, code); |
| 1102 | assert!(failure.message.contains("/acme/-/billing"), "{}", failure.message); |
| 1103 | } |
| 1104 | let Outcome::Fail(failure) = refusal(FailureCode::NotPaid, "acme", ComputeKind::Agent, "") else { panic!() }; |
| 1105 | assert!(failure.message.contains("$5 trial") && failure.message.contains("never charged")); |
| 1106 | } |
| 1107 | } |