Sign in

flagon-io/g1t

public

g1t/services/repos/src/git_ops.rs

419 lines19,062 bytesCodeBlame
1//! Git operations, counted per workspace.
2//!
3//! Cloudflare Artifacts charges g1t per operation from 2026-10-14 ($0.15
4//! per 1,000) without having said exactly which calls are operations. So
5//! every interaction with the store is metered by kind (meters.rs), and
6//! which meters a workspace is counted for, and how much each is worth, is
7//! data (`operation_mapping`). By default: each clone or fetch (an
8//! upload-pack request that fetches objects, not `ls-refs` and never an
9//! answer g1t served from its own cache), each push (receive-pack), and
10//! making, forking and deleting a repository. The counts go to
11//! `git_operations` by the hour, after answers have gone back. Billing
12//! reads the month's count each day and charges workspaces on the plan for
13//! what is past the amount that is free for everyone (50,000 a month), at
14//! cost plus 20%. A workspace on the plan is never slowed or refused for
15//! git operations or for storage: it pays for them as usage, up to its
16//! spend limit.
17//!
18//! A free workspace is never charged for git operations. Past
19//! `GIT_OPERATIONS_FREE_CAP` in a month (50,000, billing's
20//! `GIT_OPERATIONS_INCLUDED`), it is slowed down instead: at most
21//! `GIT_OPERATIONS_FREE_HOURLY` (60) an hour, answered 429 with when to try
22//! again. Whether it is past its cap is decided from counts this isolate
23//! read a moment ago and has added to since, never by asking the database
24//! on the way (63 to 98 ms a request, measured).
25//!
26//! Agents' sandboxes, checks, builds and workflow jobs clone, fetch and
27//! push through g1t's git endpoints (`https://g1t.sh/<path>.git`, with a
28//! run credential), never the store directly, so they are counted here
29//! like anyone's; so is git an agent runs itself in its sandbox. A pull
30//! request's working copy counts for the workspace of the repository it
31//! came from (meters.rs). The one sandbox that reads the store directly,
32//! a nightly backup, reports its clone, which is g1t's cost and never a
33//! workspace's (backups.rs). The limits here go by the path asked for, so
34//! a free workspace's requests to a working copy (`pulls/<pull id>`) are
35//! counted for it but never slowed down.
36
37use std::cell::RefCell;
38use std::collections::HashMap;
39
40use g1t_contracts::repos::{GitService, WorkspaceGitOperations};
41use serde::Deserialize;
42use worker::{D1Database, Env, Fetcher, Response, Result};
43
44/// What a request to g1t's git endpoints asks the store.
45#[derive(Clone, Copy, Debug, PartialEq, Eq)]
46pub enum GitCall {
47 /// `GET info/refs`: the refs, for a fetch or a push.
48 RefAdvertisement,
49 /// A protocol v2 `ls-refs`.
50 LsRefs,
51 /// An upload-pack request that fetches objects: a clone or fetch.
52 Fetch,
53 /// A push.
54 ReceivePack,
55}
56
57impl GitCall {
58 /// Its meter (meters.rs).
59 pub fn meter(self) -> &'static str {
60 match self {
61 GitCall::RefAdvertisement => "git.info_refs",
62 GitCall::LsRefs => "git.ls_refs",
63 GitCall::Fetch => "git.fetch",
64 GitCall::ReceivePack => "git.receive_pack",
65 }
66 }
67
68 /// The meter for an answer g1t served from its own cache, which never
69 /// reaches the store and is never an operation.
70 pub fn cached_meter(self) -> &'static str {
71 match self {
72 GitCall::RefAdvertisement => "cache.info_refs",
73 GitCall::LsRefs => "cache.ls_refs",
74 GitCall::Fetch => "cache.fetch",
75 GitCall::ReceivePack => "cache.receive_pack",
76 }
77 }
78}
79
80/// What a git request is. `body` is an upload-pack POST's, read already.
81pub fn classify(service: GitService, endpoint: &str, get: bool, body: Option<&[u8]>) -> GitCall {
82 if get || endpoint == "info/refs" {
83 return GitCall::RefAdvertisement;
84 }
85 if service == GitService::ReceivePack {
86 return GitCall::ReceivePack;
87 }
88 let ls_refs = body.is_some_and(|body| {
89 let (lines, _) = crate::land::read_pkt_lines(body);
90 lines.first().is_some_and(|line| line.strip_suffix(b"\n").unwrap_or(line) == b"command=ls-refs")
91 });
92 if ls_refs { GitCall::LsRefs } else { GitCall::Fetch }
93}
94
95/// The hour an operation is counted in: `YYYY-MM-DDTHH` of an RFC 3339 time.
96pub fn hour_key(timestamp: &str) -> String {
97 timestamp[..13].to_owned()
98}
99
100/// Whether a free workspace's operation should wait: past the month's cap,
101/// and past the hour's share.
102pub fn slow_down(month_ops: u64, hour_ops: u64, free_cap: u64, hourly: u64) -> bool {
103 month_ops > free_cap && hour_ops > hourly
104}
105
106/// The limits, from the repos service's variables.
107pub struct Limits {
108 pub free_cap: u64,
109 pub hourly: u64,
110}
111
112impl Limits {
113 pub fn from_env(env: &Env) -> Self {
114 let number = |name: &str, default: u64| env.var(name).ok().and_then(|v| v.to_string().parse().ok()).unwrap_or(default);
115 Limits { free_cap: number("GIT_OPERATIONS_FREE_CAP", 50_000), hourly: number("GIT_OPERATIONS_FREE_HOURLY", 60) }
116 }
117}
118
119#[derive(Deserialize)]
120struct Counts {
121 month: Option<f64>,
122 hour: Option<f64>,
123}
124
125/// Where a workspace stands this month and hour, as this isolate knows it.
126#[derive(Clone, Debug, Default, PartialEq)]
127pub struct Standing {
128 /// `YYYY-MM-DDTHH` the counts are for.
129 hour_key: String,
130 month: f64,
131 hour: f64,
132 /// When the database was last read for it.
133 read_at: u64,
134}
135
136impl Standing {
137 /// The month's and the hour's counts at `hour_key`, if read recently
138 /// enough to go by: an hour that has turned starts at nothing, a month
139 /// that has turned likewise.
140 pub fn at(&self, hour_key: &str, now: u64) -> Option<(u64, u64)> {
141 if now.saturating_sub(self.read_at) > STANDING_TTL_MS {
142 return None;
143 }
144 let month = if self.hour_key.get(..7) == hour_key.get(..7) { self.month } else { 0.0 };
145 let hour = if self.hour_key == hour_key { self.hour } else { 0.0 };
146 Some((month as u64, hour as u64))
147 }
148
149 /// Adds operations counted here, not yet written.
150 pub fn add(&mut self, hour_key: &str, operations: f64) {
151 if self.hour_key != hour_key {
152 if self.hour_key.get(..7) != hour_key.get(..7) {
153 self.month = 0.0;
154 }
155 self.hour = 0.0;
156 self.hour_key = hour_key.to_owned();
157 }
158 self.month += operations;
159 self.hour += operations;
160 }
161}
162
163/// How long counts read from the database are gone by. Every write of the
164/// meters reads them again (meters.rs), so a busy workspace's are seconds old.
165const STANDING_TTL_MS: u64 = 10 * 60 * 1000;
166/// How long billing's answer about a workspace's plan is kept.
167const PLAN_TTL_MS: u64 = 5 * 60 * 1000;
168
169thread_local! {
170 static STANDING: RefCell<HashMap<String, Standing>> = RefCell::new(HashMap::new());
171 static FREE: RefCell<HashMap<String, (bool, u64)>> = RefCell::new(HashMap::new());
172}
173
174/// Adds operations this isolate counted for `namespace` (meters.rs).
175pub fn note_local(namespace: &str, hour_key: &str, operations: f64) {
176 STANDING.with(|standing| {
177 if let Some(kept) = standing.borrow_mut().get_mut(namespace) {
178 kept.add(hour_key, operations);
179 }
180 });
181}
182
183/// The month's and the hour's counts for `namespace`, as last read and
184/// added to here; `None` when not read lately, which never slows anyone.
185pub fn standing(namespace: &str, hour_key: &str, now: u64) -> Option<(u64, u64)> {
186 STANDING.with(|standing| standing.borrow().get(namespace).and_then(|kept| kept.at(hour_key, now)))
187}
188
189/// Reads `namespace`'s counts again, after the meters were written.
190pub async fn refresh(db: &D1Database, namespace: &str) -> Result<()> {
191 let now = g1t_kit::now_ms();
192 let hour = hour_key(&g1t_contracts::time::rfc3339(now));
193 let counts = db
194 .prepare(
195 "SELECT SUM(operations) AS month, SUM(CASE WHEN hour = ?2 THEN operations END) AS hour
196 FROM git_operations WHERE namespace = ?1 AND substr(hour, 1, 7) = ?3",
197 )
198 .bind(&[namespace.into(), hour.as_str().into(), hour[..7].into()])?
199 .first::<Counts>(None)
200 .await?;
201 let (month, hour_count) = counts.map_or((0.0, 0.0), |c| (c.month.unwrap_or(0.0), c.hour.unwrap_or(0.0)));
202 STANDING.with(|standing| {
203 standing.borrow_mut().insert(namespace.to_owned(), Standing { hour_key: hour, month, hour: hour_count, read_at: now });
204 });
205 Ok(())
206}
207
208/// The hours of `month` (`YYYY-MM`) from `since` on, as the range
209/// `[from, until)` of hour keys; `None` for a month that is not one.
210/// A range on `hour` is what the table's key can find; `substr` is not.
211pub fn month_hours(month: &str, since: Option<&str>) -> Option<(String, String)> {
212 let year: u32 = month.get(..4)?.parse().ok()?;
213 let number: u32 = month.get(5..7)?.parse().ok()?;
214 if month.len() != 7 || &month[4..5] != "-" || !(1..=12).contains(&number) {
215 return None;
216 }
217 let until = if number == 12 { format!("{}-01-01", year + 1) } else { format!("{year}-{:02}-01", number + 1) };
218 let start = format!("{month}-01");
219 let from = match since {
220 Some(since) if since > start.as_str() => since.to_owned(),
221 _ => start,
222 };
223 Some((from, until))
224}
225
226/// Each workspace's operations in `month`, from `since` (an hour) on.
227pub async fn totals(db: &D1Database, month: &str, since: Option<&str>, namespace: Option<&str>) -> Result<Vec<WorkspaceGitOperations>> {
228 #[derive(Deserialize)]
229 struct Row {
230 namespace: String,
231 operations: Option<f64>,
232 }
233 let Some((from, until)) = month_hours(month, since) else { return Ok(vec![]) };
234 // One workspace's is read by the table's key, namespace first; every
235 // workspace's (billing's daily measure) reads the month's hours.
236 let statement = match namespace {
237 Some(namespace) => db
238 .prepare(
239 "SELECT namespace, SUM(operations) AS operations FROM git_operations
240 WHERE namespace = ?1 AND hour >= ?2 AND hour < ?3
241 GROUP BY namespace",
242 )
243 .bind(&[namespace.into(), from.as_str().into(), until.as_str().into()])?,
244 None => db
245 .prepare(
246 "SELECT namespace, SUM(operations) AS operations FROM git_operations
247 WHERE hour >= ?1 AND hour < ?2
248 GROUP BY namespace",
249 )
250 .bind(&[from.as_str().into(), until.as_str().into()])?,
251 };
252 Ok(statement
253 .all()
254 .await?
255 .results::<Row>()?
256 .into_iter()
257 .map(|row| WorkspaceGitOperations { namespace: row.namespace, operations: row.operations.unwrap_or(0.0) as u64 })
258 .collect())
259}
260
261/// Whether billing says the workspace is free. Unknown (billing not bound
262/// or not answering) counts as not free: nothing is slowed down on a guess.
263pub async fn is_free(billing: Option<&Fetcher>, namespace: &str) -> bool {
264 let Some(billing) = billing else { return false };
265 let args = g1t_contracts::billing::EntitlementsArgs { workspace: namespace.to_owned() };
266 match g1t_kit::call::<_, serde_json::Value>(billing, "entitlements", &args).await {
267 Ok(found) => found["plan"].as_str() == Some("free"),
268 Err(error) => {
269 worker::console_error!("could not ask billing about {namespace}: {error}");
270 false
271 }
272 }
273}
274
275/// [`is_free`], kept for a few minutes: asked only of a workspace past its
276/// cap, on each of its requests.
277pub async fn is_free_kept(billing: Option<&Fetcher>, namespace: &str) -> bool {
278 let now = g1t_kit::now_ms();
279 let kept = FREE.with(|free| {
280 free.borrow().get(namespace).filter(|(_, at)| now.saturating_sub(*at) < PLAN_TTL_MS).map(|(free, _)| *free)
281 });
282 if let Some(free) = kept {
283 return free;
284 }
285 let free = is_free(billing, namespace).await;
286 FREE.with(|kept| kept.borrow_mut().insert(namespace.to_owned(), (free, now)));
287 free
288}
289
290/// The private storage a free workspace may push to: 1 GB unless set.
291pub fn free_private_bytes(env: &worker::Env) -> i64 {
292 env.var("FREE_PRIVATE_STORAGE_BYTES")
293 .ok()
294 .and_then(|value| value.to_string().parse().ok())
295 .unwrap_or(1_000_000_000)
296}
297
298/// Whether a push to a private repository should be refused: a free
299/// workspace whose private repositories already hold its free amount. Free
300/// workspaces are never charged for storage; past it, pushes stop instead.
301/// Only ever asked for a free workspace: one on the plan pays for storage
302/// past the free amount and is never refused.
303pub fn storage_full(private_bytes: i64, free_bytes: i64) -> bool {
304 private_bytes >= free_bytes
305}
306
307/// The answer to a push a free workspace has no room for. Plain text on
308/// the push's first request, which git shows as the reason.
309pub fn storage_full_response(namespace: &str, private_bytes: i64, free_bytes: i64) -> Result<Response> {
310 let gb = |bytes: i64| bytes as f64 / 1_000_000_000.0;
311 let message = format!(
312 "{namespace}'s private repositories hold {:.2} GB, and a free workspace has {:.0} GB. Free workspaces are never charged for storage, so pushes to private repositories stop here. Make the repository public, delete what you no longer need, or start the g1t plan, where storage past it is usage at cost plus 20% and pushes never stop: https://g1t.sh/{namespace}/-/billing
313",
314 gb(private_bytes),
315 gb(free_bytes)
316 );
317 Response::error(message, 403)
318}
319
320/// The answer to a free workspace past its share: try again next hour.
321pub fn too_many(namespace: &str, free_cap: u64, hourly: u64) -> Result<Response> {
322 let message = format!(
323 "{namespace} has made more than {free_cap} git operations this month, so g1t allows {hourly} an hour until the month turns. Free workspaces are never charged for git operations. On the g1t plan they are never slowed: past {free_cap} a month they are usage at cost plus 20%: https://g1t.sh/{namespace}/-/billing\n"
324 );
325 let response = Response::error(message, 429)?;
326 response.headers().set("retry-after", "3600")?;
327 Ok(response)
328}
329
330#[cfg(test)]
331mod tests {
332 use super::*;
333
334 #[test]
335 fn operations_are_counted_by_the_hour() {
336 assert_eq!(hour_key("2026-10-14T09:59:59.000Z"), "2026-10-14T09");
337 }
338
339 #[test]
340 fn a_months_hours_are_a_range_on_the_key() {
341 let range = |month: &str, since: Option<&str>| month_hours(month, since);
342 assert_eq!(range("2026-10", None), Some(("2026-10-01".to_owned(), "2026-11-01".to_owned())));
343 assert_eq!(range("2026-12", None), Some(("2026-12-01".to_owned(), "2027-01-01".to_owned())));
344 // `since` narrows the start, never widens it past the month.
345 assert_eq!(range("2026-10", Some("2026-10-14T00")).map(|r| r.0), Some("2026-10-14T00".to_owned()));
346 assert_eq!(range("2026-10", Some("2026-09-30T23")).map(|r| r.0), Some("2026-10-01".to_owned()));
347 assert_eq!(range("2026-10", Some("")).map(|r| r.0), Some("2026-10-01".to_owned()));
348 // Every hour of the month is in it, and none of the next or last,
349 // as `substr(hour, 1, 7) = month` had it.
350 let (from, until) = range("2026-10", None).unwrap();
351 let inside = |hour: &str| hour >= from.as_str() && hour < until.as_str();
352 assert!(inside("2026-10-01T00") && inside("2026-10-31T23"));
353 assert!(!inside("2026-09-30T23") && !inside("2026-11-01T00"));
354 assert_eq!(range("2026-13", None), None);
355 assert_eq!(range("2026-1", None), None);
356 assert_eq!(range("", None), None);
357 }
358
359 fn pkt(payload: &str) -> Vec<u8> {
360 format!("{:04x}{payload}", payload.len() + 4).into_bytes()
361 }
362
363 #[test]
364 fn each_git_request_is_metered_by_what_it_asks() {
365 use GitService::{ReceivePack, UploadPack};
366 assert_eq!(classify(UploadPack, "info/refs", true, None), GitCall::RefAdvertisement);
367 assert_eq!(classify(ReceivePack, "info/refs", true, None), GitCall::RefAdvertisement);
368 let ls_refs = [pkt("command=ls-refs\n"), b"0001".to_vec(), pkt("peel\n"), b"0000".to_vec()].concat();
369 assert_eq!(classify(UploadPack, "git-upload-pack", false, Some(&ls_refs)), GitCall::LsRefs);
370 let fetch = [pkt("command=fetch\n"), b"0001".to_vec(), pkt("want 1111111111111111111111111111111111111111\n"), pkt("done\n"), b"0000".to_vec()].concat();
371 assert_eq!(classify(UploadPack, "git-upload-pack", false, Some(&fetch)), GitCall::Fetch);
372 let v0 = [pkt("want 1111111111111111111111111111111111111111 side-band-64k\n"), b"0000".to_vec(), pkt("done\n")].concat();
373 assert_eq!(classify(UploadPack, "git-upload-pack", false, Some(&v0)), GitCall::Fetch);
374 assert_eq!(classify(ReceivePack, "git-receive-pack", false, None), GitCall::ReceivePack);
375 // By default only fetches and pushes are operations; listing refs,
376 // and anything g1t answered from its cache, never are.
377 let mapping = crate::meters::Mapping::defaults();
378 assert_eq!(mapping.billable(GitCall::Fetch.meter()), 1.0);
379 assert_eq!(mapping.billable(GitCall::ReceivePack.meter()), 1.0);
380 assert_eq!(mapping.billable(GitCall::LsRefs.meter()), 0.0);
381 assert_eq!(mapping.billable(GitCall::RefAdvertisement.meter()), 0.0);
382 for call in [GitCall::RefAdvertisement, GitCall::LsRefs, GitCall::Fetch, GitCall::ReceivePack] {
383 assert_eq!(mapping.billable(call.cached_meter()), 0.0);
384 assert_eq!(mapping.cost(call.cached_meter()), 0.0);
385 }
386 }
387
388 #[test]
389 fn the_standing_kept_here_moves_with_local_counts_and_turns_with_the_hour() {
390 let mut standing = Standing { hour_key: "2026-10-14T09".into(), month: 50_000.0, hour: 59.0, read_at: 1_000 };
391 assert_eq!(standing.at("2026-10-14T09", 1_000), Some((50_000, 59)));
392 standing.add("2026-10-14T09", 2.0);
393 assert_eq!(standing.at("2026-10-14T09", 2_000), Some((50_002, 61)));
394 // A new hour starts at nothing; the month goes on.
395 assert_eq!(standing.at("2026-10-14T10", 2_000), Some((50_002, 0)));
396 standing.add("2026-10-14T10", 1.0);
397 assert_eq!(standing.at("2026-10-14T10", 2_000), Some((50_003, 1)));
398 // A new month too.
399 assert_eq!(standing.at("2026-11-01T00", 2_000), Some((0, 0)));
400 // Read too long ago: not gone by, so nobody is slowed on old news.
401 assert_eq!(standing.at("2026-10-14T10", 1_000 + STANDING_TTL_MS + 1), None);
402 }
403
404 #[test]
405 fn a_free_workspace_pushes_until_its_private_storage_is_full() {
406 assert!(!storage_full(999_999_999, 1_000_000_000));
407 assert!(storage_full(1_000_000_000, 1_000_000_000));
408 assert!(storage_full(3_000_000_000, 1_000_000_000));
409 }
410
411 #[test]
412 fn a_free_workspace_is_slowed_only_past_its_monthly_cap() {
413 // Under the cap: never slowed, however busy the hour.
414 assert!(!slow_down(49_999, 5_000, 50_000, 60));
415 // Past it: 60 an hour, then wait.
416 assert!(!slow_down(50_001, 60, 50_000, 60));
417 assert!(slow_down(50_001, 61, 50_000, 60));
418 }
419}