flagon-io/g1t

public

970 lines45,314 bytesCode Blame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1d ago	1	//! Whether a workspace may start compute, and holding what it may cost.
	2	//!
	3	//! Every service that starts something that costs g1t real money asks
	4	//! here first (see `g1t_contracts::billing::ReserveArgs`):
	5	//!
	6	//! - `entitlements`: the workspace's plan, whether it may start compute
	7	//! at all, its caps (agents at once, a run's time and spend, an issue's
	8	//! spend), its ceiling and exposure, and whether compute is paused.
	9	//! - `reserve`: holds the work's estimated cost against what may pay
	10	//! for it (the plan's included usage, the trial, the open-source pool,
	11	//! then on-demand room under the ceiling and the spend limit), so starts
	12	//! at the same moment cannot overshoot together. Answers who pays first,
	13	//! or refuses with a stable code and a message for the owner.
	14	//! - `settle`: releases the hold. The charge itself goes on the ledger
	15	//! the usual way; a hold never settled lapses after three hours.
	16	//!
	17	//! No card, no compute. A free workspace's forge is free, but compute
	18	//! needs the plan, or a card check: it unlocks the one-time trial and g1t's
	19	//! open-source pool (checks, workflows and the merge queue on public
	20	//! repositories). The card check is what keeps g1t's free compute from
	21	//! being mined: one trial per card, and a real person behind each.
	22	//!
	23	//! Spikes. An hour's spend above `SPIKE_FACTOR` (5) times the
	24	//! workspace's usual hour over the last week, and at least
	25	//! `SPIKE_FLOOR_MICROS` ($5), pauses new compute until an owner answers:
	26	//! keep going (for 24 hours, or until the hour's spend doubles again) or
	27	//! stop. Runs already under way finish. g1t's own workspaces are watched
	28	//! but never paused.
	29
	30	use g1t_contracts::billing::{
	31	ComputeKind, ConfirmSpikeArgs, SetCapsArgs, Entitlements, EntitlementsArgs, LimitState, PaidBy, PlanKind, Reservation,
	32	ReserveArgs, SettleArgs, Spike, UNLIMITED_MICROS, UsageAlert, RESERVATION_HOURS,
	33	};
	34	use g1t_contracts::time::rfc3339;
	35	use g1t_contracts::{FailureCode, Outcome, Role, new_id};
	36	use g1t_kit::now_ms;
	37	use serde::Deserialize;
	38	use worker::Result;
	39	use worker::wasm_bindgen::JsValue;
	40
	41	use crate::Billing;
	42	use crate::credits::{self, left};
	43	use crate::features::dollars;
	44	use crate::limits::alert_level;
	45
	46	/// Agents at once in the first month or on the trial, and after.
	47	pub(crate) const FIRST_MONTH_AGENTS: u32 = 2;
	48	pub(crate) const AGENTS: u32 = 10;
	49	/// The longest run in the first month or on the trial, in minutes.
	50	pub(crate) const FIRST_MONTH_MINUTES: u32 = 60;
	51	/// How long "keep going" lifts a spike's pause.
	52	const KEEP_GOING_MS: u64 = 24 * 60 * 60 * 1000;
	53	/// The week a usual hour is measured over.
	54	const WEEK_HOURS: i64 = 7 * 24;
	55
	56	/// What may pay for reserved work, at price, in the order it pays.
	57	#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
	58	pub(crate) struct Room {
	59	pub credit: i64,
	60	pub trial: i64,
	61	pub oss: i64,
	62	/// Under the ceiling and the spend limit; None: no bound (g1t's own).
	63	pub on_demand: Option<i64>,
	64	}
	65
	66	/// Why `place` could not hold an estimate.
	67	#[derive(Clone, Copy, Debug, PartialEq, Eq)]
	68	pub(crate) enum Short {
	69	/// Nothing left that pays for it.
	70	Empty,
	71	/// Some room, but less than a paid workspace's whole estimate.
	72	TooSmall,
	73	}
	74
	75	/// Where a new hold of `estimate` goes, given what open holds take
	76	/// (`held`): the source that pays first, and what to hold. Holds fill the
	77	/// sources in order, so the first source with room after them pays first.
	78	/// A paid workspace's whole estimate must fit, so the ceiling cannot be
	79	/// overshot; a free workspace may use its last bit of trial, and what the
	80	/// run costs past it is g1t's.
	81	pub(crate) fn place(room: &Room, held: i64, estimate: i64, whole: bool) -> std::result::Result<(PaidBy, i64), Short> {
	82	let sources = [
	83	(PaidBy::Credit, room.credit.max(0)),
	84	(PaidBy::Trial, room.trial.max(0)),
	85	(PaidBy::Oss, room.oss.max(0)),
	86	];
	87	let pools: i64 = sources.iter().map(\|(_, room)\| room).sum();
	88	let remaining = match room.on_demand {
	89	None => i64::MAX,
	90	Some(on_demand) => pools + on_demand.max(0) - held.max(0),
	91	};
	92	if remaining <= 0 {
	93	return Err(Short::Empty);
	94	}
	95	let estimate = estimate.max(0);
	96	if whole && estimate > remaining {
	97	return Err(Short::TooSmall);
	98	}
	99	let hold = estimate.min(remaining);
	100	let mut end = 0;
	101	for (source, size) in sources {
	102	end += size;
	103	if held.max(0) < end {
	104	return Ok((source, hold));
	105	}
	106	}
	107	Ok((PaidBy::OnDemand, hold))
	108	}
	109
	110	/// Whether the last hour is a spike: above `factor` times the usual hour,
	111	/// and at least `floor`.
	112	pub(crate) fn is_spike(last_hour: i64, usual_hour: i64, factor: i64, floor: i64) -> bool {
	113	last_hour >= floor.max(1) && last_hour > usual_hour.max(0) * factor
	114	}
	115
	116	/// Whether a spike an owner said to keep going on still lets work start:
	117	/// within its 24 hours, and the hour's spend not doubled again.
	118	pub(crate) fn still_continued(until: Option<&str>, now: &str, hour_at_spike: i64, last_hour: i64) -> bool {
	119	until.is_some_and(\|until\| now < until) && last_hour < hour_at_spike.max(1) * 2
	120	}
	121
	122	/// A workspace's caps, from its plan.
	123	pub(crate) fn caps(plan: PlanKind, first_month: bool, on_trial: bool, agents: Option<u32>) -> (u32, u32) {
	124	let tight = first_month \|\| (plan == PlanKind::Free && on_trial) \|\| plan == PlanKind::Free;
	125	let default_agents = if tight { FIRST_MONTH_AGENTS } else { AGENTS };
	126	let minutes = if tight { FIRST_MONTH_MINUTES } else { g1t_contracts::guardrails::MAX_MINUTES };
	127	(agents.unwrap_or(default_agents), minutes)
	128	}
	129
	130	/// The refusal for a start that cannot be held, with what to do.
	131	pub(crate) fn refusal(code: FailureCode, workspace: &str, kind: ComputeKind, detail: &str) -> Outcome<Reservation> {
	132	let link = format!("/{workspace}/-/billing");
	133	let what = match kind {
	134	ComputeKind::Agent => "Agents",
	135	ComputeKind::Check => "Checks",
	136	ComputeKind::Workflow => "Workflows",
	137	ComputeKind::Queue => "The merge queue",
	138	ComputeKind::Deploy => "Deployments",
	139	ComputeKind::Embedding => "Semantic search",
	140	};
	141	let message = match code {
	142	FailureCode::NotPaid if kind.open_source_pool() => format!(
	143	"{what} run in g1t's sandboxes, which cost real money, so they need the g1t plan ($20 a month) or a card check. A card check gives public repositories g1t's open-source pool and starts the $5 trial; it is never charged. Both are at {link}."
	144	),
	145	FailureCode::NotPaid => format!(
	146	"{what} cost real money to run, so they need the g1t plan ($20 a month, with $10 of usage included) or the one-time $5 trial, which starts with a card check that is never charged. Both are at {link}."
	147	),
	148	FailureCode::TrialUsed => format!(
	149	"This workspace has used its $5 trial. Start the g1t plan ($20 a month, with $10 of usage included) to keep going: {link}."
	150	),
	151	FailureCode::OssPoolEmpty => format!(
	152	"g1t's open-source pool for this month is used up{detail}, so checks and workflows on public repositories wait until the 1st. The g1t plan runs them now: {link}."
	153	),
	154	FailureCode::Limit => format!("{detail} An owner can raise the limit, prepay, or ask g1t for more at {link}."),
	155	FailureCode::Paused => format!("New compute is paused: {detail} An owner can see why and answer at {link}."),
	156	_ => detail.to_owned(),
	157	};
	158	Outcome::fail(code, message)
	159	}
	160
	161	#[derive(Deserialize)]
	162	struct SpikeRow {
	163	id: String,
	164	status: String,
	165	hour_micros: i64,
	166	average_micros: i64,
	167	detected_at: String,
	168	decided_by: Option<String>,
	169	decided_at: Option<String>,
	170	until: Option<String>,
	171	}
	172
	173	impl From<SpikeRow> for Spike {
	174	fn from(row: SpikeRow) -> Self {
	175	Spike {
	176	id: row.id,
	177	status: row.status,
	178	hour_micros: row.hour_micros,
	179	average_micros: row.average_micros,
	180	detected_at: row.detected_at,
	181	decided_by: row.decided_by,
	182	decided_at: row.decided_at,
	183	until: row.until,
	184	}
	185	}
	186	}
	187
	188	/// A workspace's pace: the last hour, the usual hour over the last week,
	189	/// the last day, at price (what was charged plus what paid for it first).
	190	#[derive(Clone, Copy, Debug, Default)]
	191	pub(crate) struct Pace {
	192	pub last_hour: i64,
	193	pub usual_hour: i64,
	194	pub last_day: i64,
	195	}
	196
	197	impl Billing {
	198	/// Spend at price over the last hour, day and week.
	199	pub(crate) async fn pace(&self, workspace: &str) -> Result<Pace> {
	200	#[derive(Deserialize)]
	201	struct Row {
	202	hour: Option<i64>,
	203	day: Option<i64>,
	204	week: Option<i64>,
	205	}
	206	let now = now_ms();
	207	let hour_ago = rfc3339(now - 60 * 60 * 1000);
	208	let day_ago = rfc3339(now - 24 * 60 * 60 * 1000);
	209	let week_ago = rfc3339(now - 7 * 24 * 60 * 60 * 1000);
	210	let gross = "(-amount_micros + credit_micros + trial_micros + oss_micros + given_micros)";
	211	let row = self
	212	.db
	213	.prepare(format!(
	214	"SELECT SUM(CASE WHEN created_at >= ?2 THEN {gross} END) AS hour,
	215	SUM(CASE WHEN created_at >= ?3 THEN {gross} END) AS day,
	216	SUM(CASE WHEN created_at < ?2 THEN {gross} END) AS week
	217	FROM ledger WHERE workspace = ?1 AND kind = 'usage' AND created_at >= ?4"
	218	))
	219	.bind(&[workspace.into(), hour_ago.into(), day_ago.into(), week_ago.into()])?
	220	.first::<Row>(None)
	221	.await?;
	222	Ok(row.map_or_else(Pace::default, \|r\| Pace {
	223	last_hour: r.hour.unwrap_or(0).max(0),
	224	usual_hour: r.week.unwrap_or(0).max(0) / (WEEK_HOURS - 1),
	225	last_day: r.day.unwrap_or(0).max(0),
	226	}))
	227	}
	228
	229	/// The workspace's latest spike, if any.
	230	pub(crate) async fn latest_spike(&self, workspace: &str) -> Result<Option<Spike>> {
	231	Ok(self
	232	.db
	233	.prepare(
	234	"SELECT id, status, hour_micros, average_micros, detected_at, decided_by, decided_at, until
	235	FROM spikes WHERE workspace = ? ORDER BY detected_at DESC LIMIT 1",
	236	)
	237	.bind(&[workspace.into()])?
	238	.first::<SpikeRow>(None)
	239	.await?
	240	.map(Spike::from))
	241	}
	242
	243	/// The spike pausing the workspace now, found or new. Never for g1t's
	244	/// own workspaces, which are watched in sudo but never paused.
	245	async fn spike_pause(&self, workspace: &str, plan: PlanKind) -> Result<Option<Spike>> {
	246	let latest = self.latest_spike(workspace).await?;
	247	if let Some(spike) = &latest {
	248	if spike.status == "open" \|\| spike.status == "stopped" {
	249	return Ok(latest);
	250	}
	251	}
	252	if plan == PlanKind::Internal \|\| self.stripe.is_none() {
	253	return Ok(None);
	254	}
	255	let pace = self.pace(workspace).await?;
	256	let now = rfc3339(now_ms());
	257	if let Some(spike) = &latest {
	258	if spike.status == "continued" && still_continued(spike.until.as_deref(), &now, spike.hour_micros, pace.last_hour) {
	259	return Ok(None);
	260	}
	261	}
	262	if !is_spike(pace.last_hour, pace.usual_hour, self.plans.spike_factor, self.plans.spike_floor_micros) {
	263	return Ok(None);
	264	}
	265	let id = new_id("spk", now_ms());
	266	self.db
	267	.prepare(
	268	"INSERT INTO spikes (id, workspace, status, hour_micros, average_micros, detected_at)
	269	SELECT ?1, ?2, 'open', ?3, ?4, ?5
	270	WHERE NOT EXISTS (SELECT 1 FROM spikes WHERE workspace = ?2 AND status = 'open')",
	271	)
	272	.bind(&[id.as_str().into(), workspace.into(), (pace.last_hour as f64).into(), (pace.usual_hour as f64).into(), now.as_str().into()])?
	273	.run()
	274	.await?;
	275	self.latest_spike(workspace).await
	276	}
	277
	278	/// The alerts a workspace has reached this month: its plan's included
	279	/// usage, its spend limit and g1t's ceiling, from 50%.
	280	pub(crate) async fn alerts_for(&self, workspace: &str) -> Result<Vec<UsageAlert>> {
	281	let limit = self.limit_of(workspace).await?;
	282	self.alerts_from(workspace, &limit).await
	283	}
	284
	285	/// The same, from a limit already worked out.
	286	async fn alerts_from(&self, workspace: &str, limit: &g1t_contracts::billing::Limit) -> Result<Vec<UsageAlert>> {
	287	let month = credits::month_of(&rfc3339(now_ms()));
	288	let mut alerts = vec![];
	289	if self.has_plan(workspace).await? && limit.trust != g1t_contracts::billing::Trust::Internal {
	290	let used = self.allowance_used("plan_credit", workspace, &month).await?;
	291	let included = self.plans.plan_included_micros;
	292	let level = alert_level(used, included);
	293	if level > 0 {
	294	alerts.push(UsageAlert {
	295	meter: "included".into(),
	296	level,
	297	used_micros: used,
	298	limit_micros: included,
	299	message: if level >= 100 {
	300	format!("{workspace} has used all {} of this month's included usage. Usage from here is charged at cost plus 20%, up to your spend limit.", dollars(included))
	301	} else {
	302	format!("{workspace} has used {} of this month's {} included usage ({level}%). Past it, usage is charged at cost plus 20%, up to your spend limit.", dollars(used), dollars(included))
	303	},
	304	});
	305	}
	306	}
	307	if let Some(spend_limit) = limit.spend_limit_micros {
	308	let level = alert_level(limit.spent_micros, spend_limit);
	309	if level > 0 {
	310	alerts.push(UsageAlert {
	311	meter: "spend_limit".into(),
	312	level,
	313	used_micros: limit.spent_micros,
	314	limit_micros: spend_limit,
	315	message: format!(
	316	"{workspace} has spent {} of its {} monthly spend limit ({level}%). At the limit, new sandboxes, builds and agents stop until the month turns or an owner raises it.",
	317	dollars(limit.spent_micros),
	318	dollars(spend_limit)
	319	),
	320	});
	321	}
	322	}
	323	if let Some(ceiling) = limit.ceiling_micros.filter(\|_\| limit.trust != g1t_contracts::billing::Trust::New) {
	324	let level = alert_level(limit.exposure_micros, ceiling);
	325	if level > 0 {
	326	alerts.push(UsageAlert {
	327	meter: "ceiling".into(),
	328	level,
	329	used_micros: limit.exposure_micros,
	330	limit_micros: ceiling,
	331	message: format!(
	332	"{workspace} has {} of usage not yet paid for, of the {} g1t allows ({level}%). With a card on file g1t charges it as the limit nears; prepaying raises it at once.",
	333	dollars(limit.exposure_micros),
	334	dollars(ceiling)
	335	),
	336	});
	337	}
	338	}
	339	Ok(alerts)
	340	}
	341
	342	/// Whether a card check was done for the workspace.
	343	pub(crate) async fn card_checked(&self, workspace: &str) -> Result<bool> {
	344	Ok(self
	345	.db
	346	.prepare("SELECT workspace FROM card_checks WHERE workspace = ?")
	347	.bind(&[workspace.into()])?
	348	.first::<serde_json::Value>(None)
	349	.await?
	350	.is_some())
	351	}
	352
	353	/// What open reservations hold across `members`, at price.
	354	async fn held(&self, members: &[String]) -> Result<i64> {
	355	#[derive(Deserialize)]
	356	struct Row {
	357	held: Option<i64>,
	358	}
	359	let marks = vec!["?"; members.len().max(1)].join(", ");
	360	let mut values: Vec<JsValue> = members.iter().map(\|m\| JsValue::from(m.as_str())).collect();
	361	if values.is_empty() {
	362	values.push("".into());
	363	}
	364	values.push(rfc3339(now_ms()).into());
	365	Ok(self
	366	.db
	367	.prepare(format!(
	368	"SELECT SUM(hold_micros) AS held FROM reservations
	369	WHERE workspace IN ({marks}) AND settled_at IS NULL AND expires_at > ?"
	370	))
	371	.bind(&values)?
	372	.first::<Row>(None)
	373	.await?
	374	.and_then(\|r\| r.held)
	375	.unwrap_or(0))
	376	}
	377
	378	/// Why new compute is paused, if it is: a staff hold, a spike waiting
	379	/// for an owner (or stopped by one), or the limit reached.
	380	async fn pause_reason(
	381	&self,
	382	workspace: &str,
	383	plan: PlanKind,
	384	limit: Option<&g1t_contracts::billing::Limit>,
	385	) -> Result<(Option<String>, Option<Spike>, Option<FailureCode>)> {
	386	let account = self.account_of(workspace).await?;
	387	if let Some(hold) = account.allowances.hold.as_deref().filter(\|h\| !h.trim().is_empty()) {
	388	return Ok((Some(format!("g1t staff put a hold on new compute ({}).", hold.trim())), None, Some(FailureCode::Paused)));
	389	}
Spend caps: a monthly budget for comped workspaces and a daily breaker on what g1t pays1d ago	390	// A comped account past its monthly budget (`budget`).
	391	if let Some(why) = self.comped_stop(&account).await? {
	392	return Ok((Some(why), None, Some(FailureCode::Paused)));
	393	}
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1d ago	394	let spike = self.spike_pause(workspace, plan).await?;
	395	if let Some(spike) = &spike {
	396	let why = if spike.status == "stopped" {
	397	format!(
	398	"an owner stopped new compute after a spend spike ({} in an hour). An owner can choose Keep going.",
	399	dollars(spike.hour_micros)
	400	)
	401	} else {
	402	format!(
	403	"{} was spent in an hour, more than {} times the usual {} an hour, so new compute waits for an owner to confirm.",
	404	dollars(spike.hour_micros),
	405	self.plans.spike_factor,
	406	dollars(spike.average_micros)
	407	)
	408	};
	409	return Ok((Some(why), Some(spike.clone()), Some(FailureCode::Paused)));
	410	}
	411	let latest = self.latest_spike(workspace).await?;
	412	if plan != PlanKind::Internal && self.stripe.is_some() {
	413	let worked_out;
	414	let limit = match limit {
	415	Some(limit) => limit,
	416	None => {
	417	worked_out = self.limit_of(workspace).await?;
	418	&worked_out
	419	}
	420	};
	421	if limit.state == LimitState::Stopped {
	422	return Ok((limit.message.clone(), latest, Some(FailureCode::Limit)));
	423	}
	424	}
	425	Ok((None, latest, None))
	426	}
	427
	428	/// `entitlements`: what the workspace may do now.
	429	pub(crate) async fn entitlements(&self, a: EntitlementsArgs) -> Result<Entitlements> {
	430	let workspace = a.workspace.to_lowercase();
	431	let plan = self.plan_kind(&workspace).await?;
	432	let account = self.account_of(&workspace).await?;
	433	let limit = self.limit_of(&workspace).await?;
	434	let now = rfc3339(now_ms());
	435	let month = credits::month_of(&now);
	436	let verified = matches!(plan, PlanKind::Internal \| PlanKind::Enterprise) \|\| self.card_checked(&workspace).await?;
	437	// A card checked while the month's pool was empty: granted once it
	438	// has room.
	439	let grant = match self.grant_of(&workspace).await? {
	440	Some(grant) => Some(grant),
	441	None if verified && plan == PlanKind::Free && self.trial_allowed(&workspace).await? => self.ensure_grant(&workspace).await?,
	442	None => None,
	443	};
	444	let trial_left = grant.as_ref().map_or(0, \|g\| left(g.granted_micros, g.used_micros));
	445	let first_month = limit.first_month;
	446	let (max_agents, max_minutes) = caps(plan, first_month, trial_left > 0, account.allowances.max_concurrent_agents);
	447	let (paused, spike, _) = self.pause_reason(&workspace, plan, Some(&limit)).await?;
	448	let ceiling = match plan {
	449	PlanKind::Free => 0,
	450	PlanKind::Internal => UNLIMITED_MICROS,
	451	_ => limit.ceiling_micros.unwrap_or(UNLIMITED_MICROS),
	452	};
	453	let has_plan = plan != PlanKind::Free;
	454	let owners = self.owner_caps(&workspace).await?;
	455	let stored = self.private_storage(&workspace).await?;
	456	let oss = self.oss_paid(&workspace, &month).await?;
	457	Ok(Entitlements {
	458	plan,
	459	compute: has_plan \|\| trial_left > 0,
	460	trial_micros_left: trial_left,
	461	trial_verified: verified,
	462	first_month,
	463	max_concurrent_agents: max_agents,
	464	max_run_minutes: max_minutes,
	465	run_cap_micros: account.allowances.run_cap_micros.or(owners.0).unwrap_or(self.plans.run_cap_micros),
	466	issue_cap_micros: account.allowances.issue_cap_micros.or(owners.1).unwrap_or(self.plans.issue_cap_micros),
	467	ceiling_micros: ceiling,
	468	exposure_micros: limit.exposure_micros,
	469	paused,
	470	held_micros: self.held(&account.workspaces).await?,
	471	prepaid_micros: limit.prepaid_micros,
	472	included_micros: if has_plan { self.plans.plan_included_micros } else { 0 },
	473	included_used_micros: if has_plan { self.allowance_used("plan_credit", &workspace, &month).await? } else { 0 },
	474	audit_retention_days: self.plans.audit_days,
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1d ago	475	free_private_storage_bytes: self.plans.free_storage_bytes,
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1d ago	476	private_storage_bytes: stored,
	477	oss_paid_micros: oss,
	478	build_seconds_used: self.allowance_used("build_seconds", &workspace, &month).await?.max(0) as u32,
	479	git_operations: self.git_operations_this_month(&workspace).await?,
	480	git_operations_included: self.plans.git_included,
	481	min_charge_micros: self.plans.min_charge_micros,
	482	spike,
	483	alerts: self.alerts_from(&workspace, &limit).await?,
	484	workspace,
	485	})
	486	}
	487
	488	/// `reserve`: holds a start's estimated cost, or says why not.
	489	pub(crate) async fn reserve(&self, a: ReserveArgs) -> Result<Outcome<Reservation>> {
	490	let workspace = a.workspace.to_lowercase();
	491	let now = now_ms();
	492	let expires_at = rfc3339(now + RESERVATION_HOURS * 60 * 60 * 1000);
	493	let repo = format!("{}/{}", a.repo.namespace, a.repo.name).to_lowercase();
	494	// A g1t that does not charge holds nothing.
	495	if self.stripe.is_none() {
	496	return Ok(Outcome::Ok(Reservation { id: new_id("rsv", now), paid_by: PaidBy::OnDemand, held_micros: 0, expires_at }));
	497	}
	498	let plan = self.plan_kind(&workspace).await?;
Spend caps: a monthly budget for comped workspaces and a daily breaker on what g1t pays1d ago	499	let account = self.account_of(&workspace).await?;
	500	// g1t's own caps (`budget`), in their own words: a comped account's
	501	// monthly budget, and the daily breaker.
	502	if let Some(why) = self.comped_stop(&account).await? {
	503	return Ok(Outcome::fail(FailureCode::Paused, why));
	504	}
	505	if let Some(why) = self.breaker_refuses(plan, &account, a.kind, a.hosted_model).await? {
	506	return Ok(Outcome::fail(FailureCode::Paused, why));
	507	}
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1d ago	508	let limit = if plan == PlanKind::Internal { None } else { Some(self.limit_of(&workspace).await?) };
	509	let (paused, _, code) = self.pause_reason(&workspace, plan, limit.as_ref()).await?;
	510	if let (Some(why), Some(code)) = (paused, code) {
	511	return Ok(refusal(code, &workspace, a.kind, &why));
	512	}
	513	let month = credits::month_of(&rfc3339(now));
	514	let estimate = credits::with_margin(a.estimate_micros, self.margin_percent);
	515	let verified = matches!(plan, PlanKind::Internal \| PlanKind::Enterprise \| PlanKind::Paid) \|\| self.card_checked(&workspace).await?;
	516	let has_plan = plan != PlanKind::Free;
	517	let credit = if has_plan {
	518	left(self.plans.plan_included_micros, self.allowance_used("plan_credit", &workspace, &month).await?)
	519	} else {
	520	0
	521	};
	522	let trial = if a.kind == ComputeKind::Deploy \|\| !verified {
	523	0
	524	} else {
	525	self.grant_of(&workspace).await?.map_or(0, \|g\| left(g.granted_micros, g.used_micros))
	526	};
	527	let oss_eligible = a.public && a.kind.open_source_pool() && verified;
	528	let oss = if oss_eligible { self.oss_left(&workspace, &repo, &month).await? } else { 0 };
	529	let on_demand = match plan {
	530	PlanKind::Free => Some(0),
	531	PlanKind::Internal => None,
	532	_ => {
	533	let Some(limit) = &limit else { unreachable!("only g1t's own workspaces skip the limit") };
	534	let under_ceiling = limit.ceiling_micros.map(\|c\| (c - limit.exposure_micros).max(0));
	535	let under_spend = limit.spend_limit_micros.map(\|s\| (s - limit.spent_micros).max(0));
	536	match (under_ceiling, under_spend) {
	537	(Some(c), Some(s)) => Some(c.min(s)),
	538	(c, s) => c.or(s),
	539	}
	540	}
	541	};
	542	let room = Room { credit, trial, oss, on_demand };
	543	// Optimistic: what was held is read, and the hold is written only if
	544	// nothing was held meanwhile; otherwise read again.
	545	for _ in 0..4 {
	546	let held = self.held(&account.workspaces).await?;
	547	let (paid_by, hold) = match place(&room, held, estimate, has_plan) {
	548	Ok(placed) => placed,
	549	Err(short) => return Ok(self.short(&workspace, plan, &a, verified, &room, short).await?),
	550	};
	551	let id = new_id("rsv", now);
	552	let mut values: Vec<JsValue> = vec![
	553	id.as_str().into(),
	554	workspace.as_str().into(),
	555	repo.as_str().into(),
	556	a.kind.as_str().into(),
	557	u8::from(a.public).into(),
	558	(a.estimate_micros.max(0) as f64).into(),
	559	(hold as f64).into(),
	560	paid_by_text(paid_by).into(),
	561	rfc3339(now).into(),
	562	expires_at.as_str().into(),
	563	(held as f64).into(),
	564	];
	565	values.extend(account.workspaces.iter().map(\|w\| JsValue::from(w.as_str())));
	566	if account.workspaces.is_empty() {
	567	values.push("".into());
	568	}
	569	let placed = self
	570	.db
	571	.prepare(format!(
	572	"INSERT INTO reservations (id, workspace, repo, kind, public, estimate_micros, hold_micros, paid_by, created_at, expires_at)
	573	SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10
	574	WHERE (SELECT COALESCE(SUM(hold_micros), 0) FROM reservations
	575	WHERE workspace IN ({marks}) AND settled_at IS NULL AND expires_at > ?9) = ?11
	576	RETURNING id",
	577	marks = (12..12 + account.workspaces.len().max(1)).map(\|i\| format!("?{i}")).collect::<Vec<_>>().join(", ")
	578	))
	579	.bind(&values)?
	580	.first::<serde_json::Value>(None)
	581	.await?;
	582	if placed.is_some() {
	583	// What is held, at cost, as it was asked.
	584	let held_cost = if hold >= estimate { a.estimate_micros.max(0) } else { hold * 100 / i64::from(100 + self.margin_percent) };
	585	return Ok(Outcome::Ok(Reservation { id, paid_by, held_micros: held_cost, expires_at }));
	586	}
	587	}
	588	Ok(refusal(FailureCode::Limit, &workspace, a.kind, "Too many starts at once to hold this one; try again in a moment."))
	589	}
	590
	591	/// The refusal for a start nothing pays for.
	592	async fn short(
	593	&self,
	594	workspace: &str,
	595	plan: PlanKind,
	596	a: &ReserveArgs,
	597	verified: bool,
	598	room: &Room,
	599	short: Short,
	600	) -> Result<Outcome<Reservation>> {
	601	if plan != PlanKind::Free {
	602	let limit = self.limit_of(workspace).await?;
	603	let detail = match short {
	604	Short::TooSmall => format!(
	605	"This would take the workspace past its limit: about {} more could start now ({} spent of a {} spend limit, {} not yet paid of the {} g1t allows).",
	606	dollars(room.credit + room.trial + room.oss + room.on_demand.unwrap_or(0)),
	607	dollars(limit.spent_micros),
	608	dollars(limit.spend_limit_micros.unwrap_or_default()),
	609	dollars(limit.exposure_micros),
	610	dollars(limit.ceiling_micros.unwrap_or_default()),
	611	),
	612	Short::Empty => limit.message.unwrap_or_else(\|\| "The workspace reached its limit for this month.".to_owned()),
	613	};
	614	return Ok(refusal(FailureCode::Limit, workspace, a.kind, &detail));
	615	}
	616	if !verified {
	617	return Ok(refusal(FailureCode::NotPaid, workspace, a.kind, ""));
	618	}
	619	let oss_eligible = a.public && a.kind.open_source_pool();
	620	let trial = self.grant_of(workspace).await?;
	621	if oss_eligible && room.oss == 0 {
	622	let month = credits::month_of(&rfc3339(now_ms()));
	623	let repo = format!("{}/{}", a.repo.namespace, a.repo.name).to_lowercase();
	624	let pool = left(self.plans.oss_pool_micros, self.allowance_used("oss_pool", "", &month).await?);
	625	let detail = if pool > 0 { format!(" for {repo} (its share is {})", dollars(self.oss_repo_cap(workspace).await?)) } else { String::new() };
	626	if trial.as_ref().is_none_or(\|g\| g.used_micros >= g.granted_micros) {
	627	return Ok(refusal(FailureCode::OssPoolEmpty, workspace, a.kind, &detail));
	628	}
	629	}
	630	match trial {
	631	Some(grant) if grant.used_micros >= grant.granted_micros => Ok(refusal(FailureCode::TrialUsed, workspace, a.kind, "")),
	632	_ => Ok(refusal(FailureCode::NotPaid, workspace, a.kind, "")),
	633	}
	634	}
	635
	636	/// `settle`: releases a hold.
	637	pub(crate) async fn settle_reservation(&self, a: SettleArgs) -> Result<Outcome<bool>> {
	638	let now = rfc3339(now_ms());
	639	let settled = self
	640	.db
	641	.prepare(
	642	"UPDATE reservations SET settled_at = ?1, actual_micros = ?2
	643	WHERE id = ?3 AND settled_at IS NULL AND expires_at > ?1 RETURNING id",
	644	)
	645	.bind(&[now.as_str().into(), (a.actual_micros.max(0) as f64).into(), a.reservation_id.as_str().into()])?
	646	.first::<serde_json::Value>(None)
	647	.await?;
	648	Ok(Outcome::Ok(settled.is_some()))
	649	}
	650
	651	/// Clears reservations long settled or lapsed.
	652	pub(crate) async fn sweep_reservations(&self) -> Result<()> {
	653	let week_ago = rfc3339(now_ms() - 7 * 24 * 60 * 60 * 1000);
	654	self.db
	655	.prepare("DELETE FROM reservations WHERE expires_at < ?1")
	656	.bind(&[week_ago.into()])?
	657	.run()
	658	.await?;
	659	Ok(())
	660	}
	661
	662	/// `confirm_spike`: an owner keeps going, or stops.
	663	pub(crate) async fn confirm_spike(&self, a: ConfirmSpikeArgs) -> Result<Outcome<Entitlements>> {
	664	let workspace = a.workspace.to_lowercase();
	665	if a.actor.role_in(&workspace) != Some(Role::Owner) {
	666	return Ok(Outcome::fail(FailureCode::Forbidden, "Only an owner can answer a spend spike."));
	667	}
	668	let Some(spike) = self.latest_spike(&workspace).await? else {
	669	return Ok(Outcome::fail(FailureCode::NotFound, "There is no spend spike to answer."));
	670	};
	671	let now = now_ms();
	672	let (status, until) = if a.keep_going { ("continued", Some(rfc3339(now + KEEP_GOING_MS))) } else { ("stopped", None) };
	673	self.db
	674	.prepare("UPDATE spikes SET status = ?1, decided_by = ?2, decided_at = ?3, until = ?4 WHERE id = ?5")
	675	.bind(&[
	676	status.into(),
	677	a.actor.username.as_str().into(),
	678	rfc3339(now).into(),
	679	crate::optional(until.as_deref()),
	680	spike.id.as_str().into(),
	681	])?
	682	.run()
	683	.await?;
	684	let account = self.account_of(&workspace).await?;
	685	self.audit(
	686	&account.id,
	687	"spike",
	688	&format!("{workspace}: {} after {} in an hour", if a.keep_going { "kept going" } else { "stopped" }, dollars(spike.hour_micros)),
	689	&a.actor.username,
	690	)
	691	.await?;
	692	Ok(Outcome::Ok(self.entitlements(EntitlementsArgs { workspace }).await?))
	693	}
	694
	695	/// The owners' own run and issue caps, if they set them.
	696	async fn owner_caps(&self, workspace: &str) -> Result<(Option<i64>, Option<i64>)> {
	697	#[derive(Deserialize)]
	698	struct Row {
	699	run_cap_micros: Option<i64>,
	700	issue_cap_micros: Option<i64>,
	701	}
	702	Ok(self
	703	.db
	704	.prepare("SELECT run_cap_micros, issue_cap_micros FROM limits WHERE workspace = ?")
	705	.bind(&[workspace.into()])?
	706	.first::<Row>(None)
	707	.await?
	708	.map_or((None, None), \|r\| (r.run_cap_micros, r.issue_cap_micros)))
	709	}
	710
	711	/// `set_caps`: the owners' own run and issue caps.
	712	pub(crate) async fn set_caps(&self, a: SetCapsArgs) -> Result<Outcome<Entitlements>> {
	713	let workspace = a.workspace.to_lowercase();
	714	if a.actor.role_in(&workspace) != Some(Role::Owner) {
	715	return Ok(Outcome::fail(FailureCode::Forbidden, "Only an owner can set the workspace's caps."));
	716	}
	717	if let Err(why) = cap_bounds(a.run_cap_micros, a.issue_cap_micros) {
	718	return Ok(Outcome::fail(FailureCode::Invalid, why));
	719	}
	720	let opt = \|m: Option<i64>\| m.map_or(JsValue::NULL, \|m\| (m as f64).into());
	721	let now = rfc3339(now_ms());
	722	self.db
	723	.prepare(
	724	"INSERT INTO limits (workspace, run_cap_micros, issue_cap_micros, updated_at) VALUES (?1, ?2, ?3, ?4)
	725	ON CONFLICT (workspace) DO UPDATE SET run_cap_micros = ?2, issue_cap_micros = ?3, updated_at = ?4",
	726	)
	727	.bind(&[workspace.as_str().into(), opt(a.run_cap_micros), opt(a.issue_cap_micros), now.into()])?
	728	.run()
	729	.await?;
	730	let account = self.account_of(&workspace).await?;
	731	let shown = \|m: Option<i64>\| m.map_or_else(\|\| "the default".to_owned(), dollars);
	732	self.audit(
	733	&account.id,
	734	"caps",
	735	&format!("{workspace}: run cap {}, issue cap {}", shown(a.run_cap_micros), shown(a.issue_cap_micros)),
	736	&a.actor.username,
	737	)
	738	.await?;
	739	Ok(Outcome::Ok(self.entitlements(EntitlementsArgs { workspace }).await?))
	740	}
	741
	742	/// Emails owners about spikes that paused their workspace, once each.
	743	pub(crate) async fn tell_spikes(&self, identity: &worker::Fetcher) -> Result<()> {
	744	#[derive(Deserialize)]
	745	struct Open {
	746	id: String,
	747	workspace: String,
	748	hour_micros: i64,
	749	average_micros: i64,
	750	}
	751	let open = self
	752	.db
	753	.prepare("SELECT id, workspace, hour_micros, average_micros FROM spikes WHERE status = 'open' AND told_at IS NULL LIMIT 20")
	754	.all()
	755	.await?
	756	.results::<Open>()?;
	757	for spike in open {
	758	let workspace = &spike.workspace;
	759	let intro = format!(
	760	"{workspace} spent {} in the last hour, more than {} times its usual {} an hour, so g1t paused new sandboxes, agents and builds until an owner confirms. Runs already going finish. If this was meant, choose Keep going and nothing pauses for 24 hours unless the hour's spend doubles again. If not, choose Stop; and if it was a mistake, tell g1t from the billing page.",
	761	dollars(spike.hour_micros),
	762	self.plans.spike_factor,
	763	dollars(spike.average_micros)
	764	);
	765	let link = format!("https://g1t.sh/{workspace}/-/billing");
	766	if crate::limits::notify(identity, workspace, &format!("g1t: spending on {workspace} spiked, so new work is paused"), &intro, "Keep going or stop", &link).await {
	767	self.db
	768	.prepare("UPDATE spikes SET told_at = ? WHERE id = ?")
	769	.bind(&[rfc3339(now_ms()).into(), spike.id.as_str().into()])?
	770	.run()
	771	.await?;
	772	}
	773	}
	774	Ok(())
	775	}
	776
	777	/// What the workspace's private repositories held at the last measure.
	778	pub(crate) async fn private_storage(&self, workspace: &str) -> Result<i64> {
	779	#[derive(Deserialize)]
	780	struct Stored {
	781	private_bytes: Option<i64>,
	782	}
	783	Ok(self
	784	.db
	785	.prepare("SELECT private_bytes FROM storage_days WHERE workspace = ? ORDER BY day DESC LIMIT 1")
	786	.bind(&[workspace.into()])?
	787	.first::<Stored>(None)
	788	.await?
	789	.and_then(\|s\| s.private_bytes)
	790	.unwrap_or(0))
	791	}
	792
	793	/// What g1t's open-source pool paid for the workspace in `month`.
	794	async fn oss_paid(&self, workspace: &str, month: &str) -> Result<i64> {
	795	#[derive(Deserialize)]
	796	struct Sum {
	797	micros: Option<i64>,
	798	}
	799	Ok(self
	800	.db
	801	.prepare("SELECT SUM(oss_micros) AS micros FROM ledger WHERE workspace = ? AND created_at >= ?")
	802	.bind(&[workspace.into(), format!("{month}-01").into()])?
	803	.first::<Sum>(None)
	804	.await?
	805	.and_then(\|s\| s.micros)
	806	.unwrap_or(0))
	807	}
	808	}
	809
	810	/// Whether owners' caps are in bounds: a run $0.10 to $100 (the
	811	/// guardrails' most), an issue $1 to $1,000.
	812	pub(crate) fn cap_bounds(run: Option<i64>, issue: Option<i64>) -> std::result::Result<(), String> {
	813	if run.is_some_and(\|m\| !(100_000..=100_000_000).contains(&m)) {
	814	return Err("A run's cap is between $0.10 and $100.".to_owned());
	815	}
	816	if issue.is_some_and(\|m\| !(1_000_000..=1_000_000_000).contains(&m)) {
	817	return Err("An issue's cap is between $1 and $1,000.".to_owned());
	818	}
	819	Ok(())
	820	}
	821
	822	pub(crate) fn paid_by_text(paid_by: PaidBy) -> &'static str {
	823	match paid_by {
	824	PaidBy::Credit => "credit",
	825	PaidBy::Trial => "trial",
	826	PaidBy::Oss => "oss",
	827	PaidBy::OnDemand => "on_demand",
	828	}
	829	}
	830
	831	#[cfg(test)]
	832	mod tests {
	833	use super::*;
	834
	835	fn paid(credit: i64, on_demand: i64) -> Room {
	836	Room { credit, trial: 0, oss: 0, on_demand: Some(on_demand) }
	837	}
	838
	839	#[test]
	840	fn included_usage_pays_first_then_on_demand() {
	841	// $10 included, $100 under the ceiling, nothing held: included pays first.
	842	assert_eq!(place(&paid(10_000_000, 100_000_000), 0, 2_400_000, true), Ok((PaidBy::Credit, 2_400_000)));
	843	// Holds already cover the included usage: on demand.
	844	assert_eq!(place(&paid(10_000_000, 100_000_000), 10_000_000, 2_400_000, true), Ok((PaidBy::OnDemand, 2_400_000)));
	845	// A free workspace on its trial.
	846	let trial = Room { trial: 5_000_000, on_demand: Some(0), ..Room::default() };
	847	assert_eq!(place(&trial, 0, 2_400_000, false), Ok((PaidBy::Trial, 2_400_000)));
	848	// A public repository's checks, from the pool.
	849	let pool = Room { oss: 2_000_000, on_demand: Some(0), ..Room::default() };
	850	assert_eq!(place(&pool, 0, 600_000, false), Ok((PaidBy::Oss, 600_000)));
	851	}
	852
	853	#[test]
status.g1t.sh with incident management, invites that land you in the workspace, settings as pages, usage without quotas1d ago	854	fn a_paid_workspace_is_stopped_only_by_its_limit_never_by_a_count() {
	855	// The included $10 is gone and the workspace has already built,
	856	// served and stored far past what used to be quotas: the next start
	857	// still goes on demand, as long as its spend limit has room.
	858	let room = paid(0, 250_000_000);
	859	let held = 0;
	860	for start in 0..1_000 {
	861	let placed = place(&room, held + start * 100_000, 100_000, true);
	862	assert_eq!(placed, Ok((PaidBy::OnDemand, 100_000)));
	863	}
	864	// Only at its limit does it stop, with the limit's refusal.
	865	assert_eq!(place(&room, 250_000_000, 100_000, true), Err(Short::Empty));
	866	// A free workspace has no on-demand room at all: with no trial or
	867	// pool left, nothing starts (`short` says NotPaid or TrialUsed).
	868	let free = Room { on_demand: Some(0), ..Room::default() };
	869	assert_eq!(place(&free, 0, 100_000, false), Err(Short::Empty));
	870	// g1t's own workspaces: no limit.
	871	let internal = Room { on_demand: None, ..Room::default() };
	872	assert_eq!(place(&internal, i64::MAX / 2, 100_000, true), Ok((PaidBy::OnDemand, 100_000)));
	873	}
	874
	875	#[test]
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look1d ago	876	fn concurrent_starts_cannot_overshoot_the_ceiling() {
	877	// $5 of room in all, and each start may cost up to $2.40.
	878	let room = paid(0, 5_000_000);
	879	let mut held = 0;
	880	let mut started = 0;
	881	for _ in 0..5 {
	882	match place(&room, held, 2_400_000, true) {
	883	Ok((_, hold)) => {
	884	held += hold;
	885	started += 1;
	886	}
	887	Err(short) => assert_eq!(short, Short::TooSmall),
	888	}
	889	}
	890	assert_eq!(started, 2);
	891	assert!(held <= 5_000_000);
	892	// Once the first settles, a third fits.
	893	assert!(place(&room, held - 2_400_000, 2_400_000, true).is_ok());
	894	// Nothing left at all.
	895	assert_eq!(place(&room, 5_000_000, 1, true), Err(Short::Empty));
	896	}
	897
	898	#[test]
	899	fn a_free_workspace_may_use_its_last_bit_of_trial() {
	900	let room = Room { trial: 300_000, on_demand: Some(0), ..Room::default() };
	901	// The whole estimate does not fit, but what is left is held.
	902	assert_eq!(place(&room, 0, 2_400_000, false), Ok((PaidBy::Trial, 300_000)));
	903	// Once it is held, nothing more starts.
	904	assert_eq!(place(&room, 300_000, 2_400_000, false), Err(Short::Empty));
	905	// Nothing at all: no plan, no trial, no pool.
	906	assert_eq!(place(&Room { on_demand: Some(0), ..Room::default() }, 0, 1, false), Err(Short::Empty));
	907	}
	908
	909	#[test]
	910	fn g1ts_own_workspaces_are_never_short() {
	911	let room = Room { credit: 10_000_000, on_demand: None, ..Room::default() };
	912	assert_eq!(place(&room, 50_000_000_000, 2_400_000, true), Ok((PaidBy::OnDemand, 2_400_000)));
	913	assert_eq!(place(&room, 0, 2_400_000, true), Ok((PaidBy::Credit, 2_400_000)));
	914	}
	915
	916	#[test]
	917	fn a_spike_is_five_times_the_usual_hour_and_at_least_five_dollars() {
	918	let (factor, floor) = (5, 5_000_000);
	919	// A new workspace with no history: $5 in an hour is a spike, $4 is not.
	920	assert!(is_spike(5_000_000, 0, factor, floor));
	921	assert!(!is_spike(4_000_000, 0, factor, floor));
	922	// Usually $2 an hour: $10 is not above five times, $10.01 is.
	923	assert!(!is_spike(10_000_000, 2_000_000, factor, floor));
	924	assert!(is_spike(10_010_000, 2_000_000, factor, floor));
	925	// A busy workspace at its usual pace is never a spike.
	926	assert!(!is_spike(40_000_000, 30_000_000, factor, floor));
	927	}
	928
	929	#[test]
	930	fn keep_going_lasts_a_day_or_until_spend_doubles() {
	931	let until = "2026-10-06T12:00:00Z";
	932	assert!(still_continued(Some(until), "2026-10-06T11:00:00Z", 6_000_000, 8_000_000));
	933	// Doubled again: paused again.
	934	assert!(!still_continued(Some(until), "2026-10-06T11:00:00Z", 6_000_000, 12_000_000));
	935	// A day later: watched afresh.
	936	assert!(!still_continued(Some(until), "2026-10-06T12:00:01Z", 6_000_000, 1_000_000));
	937	assert!(!still_continued(None, "2026-10-06T11:00:00Z", 6_000_000, 1));
	938	}
	939
	940	#[test]
	941	fn owners_caps_stay_in_bounds() {
	942	assert!(cap_bounds(None, None).is_ok());
	943	assert!(cap_bounds(Some(5_000_000), Some(50_000_000)).is_ok());
	944	assert!(cap_bounds(Some(50_000), None).is_err());
	945	assert!(cap_bounds(Some(101_000_000), None).is_err());
	946	assert!(cap_bounds(None, Some(500_000)).is_err());
	947	assert!(cap_bounds(None, Some(2_000_000_000)).is_err());
	948	}
	949
	950	#[test]
	951	fn caps_are_tight_in_the_first_month_and_on_the_trial() {
	952	assert_eq!(caps(PlanKind::Paid, true, false, None), (2, 60));
	953	assert_eq!(caps(PlanKind::Free, false, true, None), (2, 60));
	954	assert_eq!(caps(PlanKind::Paid, false, false, None), (10, g1t_contracts::guardrails::MAX_MINUTES));
	955	assert_eq!(caps(PlanKind::Internal, false, false, None).0, 10);
	956	// Staff can set agents at once.
	957	assert_eq!(caps(PlanKind::Paid, true, false, Some(6)).0, 6);
	958	}
	959
	960	#[test]
	961	fn every_refusal_says_what_to_do_and_where() {
	962	for code in [FailureCode::NotPaid, FailureCode::TrialUsed, FailureCode::OssPoolEmpty, FailureCode::Limit, FailureCode::Paused] {
	963	let Outcome::Fail(failure) = refusal(code, "acme", ComputeKind::Check, "Detail.") else { panic!() };
	964	assert_eq!(failure.code, code);
	965	assert!(failure.message.contains("/acme/-/billing"), "{}", failure.message);
	966	}
	967	let Outcome::Fail(failure) = refusal(FailureCode::NotPaid, "acme", ComputeKind::Agent, "") else { panic!() };
	968	assert!(failure.message.contains("$5 trial") && failure.message.contains("never charged"));
	969	}
	970	}