Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
486 linesCodeBlameRaw
1//! What a repository's About reads from its files and history: its
2//! license, its security policy, its languages and its contributors.
3//!
4//! Worked out for the default branch's head in the background, never on
5//! the way to a page: a request that finds the kept answer behind the head
6//! (or none at all) answers with what is kept and starts the work in its
7//! `wait_until` (`lib.rs`, `refresh`). One run at a time per repository,
8//! by a lease in `repo_stats.started_ms`. What it reads is kept by hash in
9//! the store's cache (trees, blob sizes, history), so working out the next
10//! commit reads only what changed. A run that runs out of time keeps what
11//! it read, marked `partial`, and the next view starts another.
12
13use std::collections::{HashMap, HashSet};
14
15use g1t_contracts::about::{ABOUT_CONTRIBUTORS, Contributor, Contributors, Freshness, LanguageShare, Languages, License, WeekCommits};
16use g1t_contracts::account_deletion::GHOST_USERNAME;
17use g1t_contracts::accounts::{EmailOwner, EmailOwnersArgs};
18use g1t_contracts::repos::{FileEntry, MAX_LISTED_FILES, Repo};
19use g1t_contracts::time::rfc3339;
20use g1t_kit::now_ms;
21use serde::{Deserialize, Serialize};
22use worker::wasm_bindgen::JsValue;
23use worker::{D1Database, Fetcher, Result};
24
25use crate::contributors::{Authored, tally};
26use crate::languages::{self, Attributes};
27use crate::license;
28use crate::store::GitRepo;
29
30/// A run older than this is taken to have died.
31pub const LEASE_MS: u64 = 120_000;
32/// How long a run reads before keeping what it has, at most.
33pub const BUDGET_MS: u64 = 22_000;
34/// The most commits of history read for contributors.
35pub const MAX_HISTORY: usize = 3_000;
36/// Commits read at a time. A history is kept by the commit it starts from
37/// (store.rs), and a page from a new head is the few commits since the last
38/// one, spliced onto the page kept from it; but each later page starts
39/// wherever the one before it ended, which moves with every push, and so is
40/// read from the store again. Pages as long as the store reads in one go
41/// (as `MAX_ANCESTRY` in lib.rs does) leave most repositories one page: 200
42/// made g1t's own history four, three of them read again after each push.
43const LOG_PAGE: u32 = 1_000;
44/// The most blobs sized for languages.
45const MAX_SIZED: usize = 6_000;
46/// Blobs sized at once: each is a round trip to the store.
47const SIZE_BATCH: usize = 32;
48/// The most distinct addresses matched to accounts.
49const MAX_MATCHED: usize = 1_000;
50/// Directories never read: nothing in them ever counts.
51const NEVER_READ: [&str; 3] = ["node_modules", "bower_components", "jspm_packages"];
52/// Where a security policy may be.
53const POLICY_DIRS: [&str; 4] = ["", ".g1t/", ".github/", "docs/"];
54
55/// How the About is worked out (2: coding agents are contributors of their own; 3: TSX counts as TypeScript); a row kept by an older version is worked
56/// out again. 1: g1t's older commit addresses count as g1t.
57pub const STATS_VERSION: f64 = 3.0;
58
59/// What `repo_stats` keeps for the About, without the full contributors.
60#[derive(Debug, Default, Deserialize)]
61pub struct Kept {
62 pub commit_hash: Option<String>,
63 pub computed_at: Option<String>,
64 pub started_ms: Option<f64>,
65 pub partial: f64,
66 pub license: Option<String>,
67 pub security_policy: Option<String>,
68 pub languages: Option<String>,
69 pub contributors_total: f64,
70 pub contributors_top: Option<String>,
71 /// [`STATS_VERSION`] when it was worked out.
72 #[serde(default)]
73 pub version: f64,
74}
75
76impl Kept {
77 /// Whether a run is under way now.
78 pub fn running(&self, now: u64) -> bool {
79 self.started_ms.is_some_and(|started| now.saturating_sub(started as u64) < LEASE_MS)
80 }
81
82 /// Whether a new run should start: the head moved past what is kept,
83 /// or what is kept is partial, and none is under way.
84 pub fn wants_run(&self, head: Option<&str>, now: u64) -> bool {
85 let Some(head) = head else { return false };
86 let behind = self.commit_hash.as_deref() != Some(head) || self.partial > 0.0 || self.version < STATS_VERSION;
87 // A partial answer is tried again at most every lease.
88 let rested = self.partial == 0.0 || self.started_ms.is_none_or(|started| now.saturating_sub(started as u64) >= LEASE_MS);
89 behind && rested && !self.running(now)
90 }
91
92 pub fn freshness(&self, head: Option<String>) -> Freshness {
93 Freshness {
94 pending: head.is_some() && self.commit_hash.is_none(),
95 head,
96 commit: self.commit_hash.clone(),
97 computed_at: self.computed_at.clone(),
98 partial: self.partial > 0.0,
99 }
100 }
101
102 pub fn license(&self) -> Option<License> {
103 self.license.as_deref().and_then(|text| serde_json::from_str(text).ok())
104 }
105
106 pub fn languages(&self) -> Vec<LanguageShare> {
107 self.languages.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
108 }
109
110 pub fn top_contributors(&self) -> Vec<Contributor> {
111 self.contributors_top.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
112 }
113}
114
115/// What `repo_stats` keeps for a repository; the default when nothing is.
116pub async fn kept(db: &D1Database, repo_id: &str) -> Result<Kept> {
117 Ok(db
118 .prepare(
119 "SELECT commit_hash, computed_at, started_ms, partial, license, security_policy, languages, contributors_total, contributors_top, version
120 FROM repo_stats WHERE repo_id = ?",
121 )
122 .bind(&[repo_id.into()])?
123 .first::<Kept>(None)
124 .await?
125 .unwrap_or_default())
126}
127
128/// The whole contributors answer kept for a repository.
129#[derive(Debug, Default, Serialize, Deserialize)]
130struct KeptContributors {
131 commits: u32,
132 contributors: Vec<Contributor>,
133 weeks: Vec<WeekCommits>,
134}
135
136/// The Contributors page's answer, as kept.
137pub async fn contributors(db: &D1Database, repo_id: &str, head: Option<String>) -> Result<Contributors> {
138 #[derive(Deserialize)]
139 struct Row {
140 contributors: Option<String>,
141 }
142 let summary = kept(db, repo_id).await?;
143 let full: KeptContributors = db
144 .prepare("SELECT contributors FROM repo_stats WHERE repo_id = ?")
145 .bind(&[repo_id.into()])?
146 .first::<Row>(None)
147 .await?
148 .and_then(|row| row.contributors)
149 .and_then(|text| serde_json::from_str(&text).ok())
150 .unwrap_or_default();
151 Ok(Contributors {
152 freshness: summary.freshness(head),
153 total: summary.contributors_total as u32,
154 commits: full.commits,
155 contributors: full.contributors,
156 weeks: full.weeks,
157 })
158}
159
160/// The languages answer, as kept.
161pub fn languages_of(kept: &Kept, head: Option<String>) -> Languages {
162 Languages { freshness: kept.freshness(head), languages: kept.languages() }
163}
164
165/// Takes the lease for a run, unless one is under way. Whether it was taken.
166pub async fn claim(db: &D1Database, repo_id: &str, now: u64) -> Result<bool> {
167 let taken = db
168 .prepare(
169 "INSERT INTO repo_stats (repo_id, started_ms) VALUES (?1, ?2)
170 ON CONFLICT (repo_id) DO UPDATE SET started_ms = excluded.started_ms
171 WHERE repo_stats.started_ms IS NULL OR repo_stats.started_ms < ?3
172 RETURNING repo_id",
173 )
174 .bind(&[repo_id.into(), JsValue::from_f64(now as f64), JsValue::from_f64(now.saturating_sub(LEASE_MS) as f64)])?
175 .first::<serde_json::Value>(None)
176 .await?;
177 Ok(taken.is_some())
178}
179
180/// Everything one run worked out.
181#[derive(Debug, Default)]
182pub struct Worked {
183 pub commit: String,
184 pub partial: bool,
185 pub license: Option<License>,
186 pub security_policy: Option<String>,
187 pub languages: Vec<LanguageShare>,
188 pub contributors: Vec<Contributor>,
189 pub contributors_total: u32,
190 pub commits: u32,
191 pub weeks: Vec<WeekCommits>,
192}
193
194/// Keeps a run's answer and lets the lease go. A partial answer keeps its
195/// lease's start, so the next is not tried until the lease is over.
196pub async fn keep(db: &D1Database, repo_id: &str, worked: &Worked) -> Result<()> {
197 let top: Vec<Contributor> = worked
198 .contributors
199 .iter()
200 .take(ABOUT_CONTRIBUTORS)
201 .map(|contributor| Contributor { weeks: Vec::new(), ..contributor.clone() })
202 .collect();
203 let full = KeptContributors { commits: worked.commits, contributors: worked.contributors.clone(), weeks: worked.weeks.clone() };
204 db.prepare(
205 "UPDATE repo_stats SET commit_hash = ?2, computed_at = ?3, started_ms = CASE WHEN ?4 = 1 THEN started_ms ELSE NULL END,
206 partial = ?4, license = ?5, security_policy = ?6, languages = ?7, contributors_total = ?8, contributors_top = ?9,
207 contributors = ?10, version = ?11
208 WHERE repo_id = ?1",
209 )
210 .bind(&[
211 repo_id.into(),
212 worked.commit.as_str().into(),
213 rfc3339(now_ms()).into(),
214 JsValue::from_f64(if worked.partial { 1.0 } else { 0.0 }),
215 match &worked.license {
216 Some(license) => json(license).into(),
217 None => JsValue::NULL,
218 },
219 match &worked.security_policy {
220 Some(path) => path.as_str().into(),
221 None => JsValue::NULL,
222 },
223 json(&worked.languages).into(),
224 JsValue::from_f64(f64::from(worked.contributors_total)),
225 json(&top).into(),
226 json(&full).into(),
227 JsValue::from_f64(STATS_VERSION),
228 ])?
229 .run()
230 .await?;
231 Ok(())
232}
233
234/// Lets the lease go after a run that failed, so the next view tries again
235/// once the lease would have ended anyway.
236pub async fn release(db: &D1Database, repo_id: &str) {
237 let statement = db
238 .prepare("UPDATE repo_stats SET started_ms = ?2 WHERE repo_id = ?1")
239 .bind(&[repo_id.into(), JsValue::from_f64(now_ms().saturating_sub(LEASE_MS / 2) as f64)]);
240 if let Ok(statement) = statement
241 && let Err(error) = statement.run().await
242 {
243 worker::console_error!("the About's lease of {repo_id} was not let go: {error}");
244 }
245}
246
247fn json<T: Serialize>(value: &T) -> String {
248 serde_json::to_string(value).unwrap_or_else(|_| "null".to_owned())
249}
250
251/// The security policy among a repository's files, at the root or in
252/// `.g1t`, `.github` or `docs`.
253pub fn security_policy(files: &[FileEntry]) -> Option<String> {
254 POLICY_DIRS.iter().find_map(|dir| {
255 files
256 .iter()
257 .find(|file| {
258 file.path
259 .strip_prefix(dir)
260 .is_some_and(|name| !name.contains('/') && (name.eq_ignore_ascii_case("security.md") || name.eq_ignore_ascii_case("security.txt") && dir.is_empty()))
261 })
262 .map(|file| file.path.clone())
263 })
264}
265
266/// Works out the About of `repo` at `head` (a commit). `identity` matches
267/// authors' addresses to accounts. Reads stop past `budget_ms` (at most
268/// [`BUDGET_MS`]), leaving the answer partial.
269pub async fn work_out<R: GitRepo>(git: &R, head: &str, tree: &str, identity: Option<&Fetcher>, budget_ms: u64) -> Result<Worked> {
270 let started = now_ms();
271 let budget_ms = budget_ms.min(BUDGET_MS);
272 let out_of_time = move || now_ms().saturating_sub(started) > budget_ms;
273 let never: Vec<String> = NEVER_READ.iter().map(|dir| (*dir).to_owned()).collect();
274 let files = async {
275 let (files, truncated) = crate::listing::changed(git, None, tree, &never, MAX_LISTED_FILES).await?;
276 let root: Vec<&FileEntry> = files.iter().filter(|file| !file.path.contains('/')).collect();
277 // The license: the best-named license file that reads as text.
278 let mut license = None;
279 for name in license::candidates(root.iter().map(|file| file.path.as_str())) {
280 let Some(hash) = root.iter().find(|file| file.path == name).and_then(|file| file.hash.clone()) else { continue };
281 if let Some(text) = git.read_blob(&hash).await?.and_then(|bytes| String::from_utf8(bytes).ok()) {
282 license = Some(license::detect(name, &text));
283 break;
284 }
285 }
286 let attributes = match root.iter().find(|file| file.path == ".gitattributes").and_then(|file| file.hash.clone()) {
287 Some(hash) => git
288 .read_blob(&hash)
289 .await?
290 .map(|bytes| Attributes::parse(&String::from_utf8_lossy(&bytes)))
291 .unwrap_or_default(),
292 None => Attributes::default(),
293 };
294 let policy = security_policy(&files);
295 // Each counted file's language, its blob sized once.
296 let mut counted: Vec<(&'static languages::Language, String)> = Vec::new();
297 for file in &files {
298 let Some(hash) = &file.hash else { continue };
299 if let Some(language) = languages::counted(&file.path, &attributes) {
300 counted.push((language, hash.clone()));
301 }
302 }
303 let mut partial = truncated || counted.len() > MAX_SIZED;
304 counted.truncate(MAX_SIZED);
305 let distinct: Vec<String> = counted.iter().map(|(_, hash)| hash.clone()).collect::<HashSet<_>>().into_iter().collect();
306 let mut sizes: HashMap<String, u64> = HashMap::with_capacity(distinct.len());
307 for batch in distinct.chunks(SIZE_BATCH) {
308 if out_of_time() {
309 partial = true;
310 break;
311 }
312 let found = futures_util::future::join_all(batch.iter().map(|hash| git.blob_size(hash))).await;
313 for (hash, size) in batch.iter().zip(found) {
314 if let Ok(Some(size)) = size {
315 sizes.insert(hash.clone(), size);
316 }
317 }
318 }
319 let shares = languages::shares(counted.iter().filter_map(|(language, hash)| Some((*language, *sizes.get(hash)?))));
320 Ok::<_, worker::Error>((license, policy, shares, partial))
321 };
322 let history = async {
323 let mut commits: Vec<Authored> = Vec::new();
324 let mut next = Some(head.to_owned());
325 let mut partial = false;
326 while let Some(from) = next.take() {
327 let page = git.log(&from, LOG_PAGE).await?;
328 let more = page.len() == LOG_PAGE as usize;
329 let after = page.last().and_then(|commit| commit.parents.first().cloned());
330 for commit in page {
331 commits.push(Authored { name: commit.author.name, email: commit.author.email, at: commit.authored_at });
332 }
333 if more && after.is_some() {
334 if commits.len() >= MAX_HISTORY || out_of_time() {
335 partial = true;
336 } else {
337 next = after;
338 }
339 }
340 }
341 commits.truncate(MAX_HISTORY);
342 Ok::<_, worker::Error>((commits, partial))
343 };
344 let (files, history) = futures_util::future::join(files, history).await;
345 let (license, security_policy, languages, files_partial) = files?;
346 let (commits, history_partial) = history?;
347 let owners = match identity {
348 Some(identity) => owners(identity, &commits).await,
349 None => HashMap::new(),
350 };
351 let (contributors, contributors_total, weeks) = tally(&commits, &owners);
352 Ok(Worked {
353 commit: head.to_owned(),
354 partial: files_partial || history_partial,
355 license,
356 security_policy,
357 languages,
358 contributors,
359 contributors_total,
360 commits: commits.len() as u32,
361 weeks,
362 })
363}
364
365/// The accounts behind the commits' addresses, the most frequent first. A
366/// failure matches none: the authors show by name.
367async fn owners(identity: &Fetcher, commits: &[Authored]) -> HashMap<String, EmailOwner> {
368 let mut counts: HashMap<String, usize> = HashMap::new();
369 for commit in commits.iter().filter(|commit| !crate::contributors::is_g1t(commit)) {
370 let email = commit.email.trim().to_ascii_lowercase();
371 if !email.is_empty() {
372 *counts.entry(email).or_default() += 1;
373 }
374 }
375 let mut emails: Vec<(String, usize)> = counts.into_iter().collect();
376 emails.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
377 emails.truncate(MAX_MATCHED);
378 let mut owners = HashMap::new();
379 for chunk in emails.chunks(200) {
380 let args = EmailOwnersArgs { emails: chunk.iter().map(|(email, _)| email.clone()).collect() };
381 match g1t_kit::call::<_, HashMap<String, EmailOwner>>(identity, "email_owners", &args).await {
382 Ok(found) => owners.extend(found),
383 Err(error) => worker::console_error!("contributors' accounts not matched: {error}"),
384 }
385 }
386 owners
387}
388
389/// What a kept contributors answer says when it names an account that
390/// `kind` (a `user.deleting` or `user.restored` event with `data`) changes:
391/// a deleted account's username, which is ghost from now on, or ghost's,
392/// which a restored account may be among. `None` for any other event.
393pub fn shown_differently(kind: &str, data: &serde_json::Value) -> Option<String> {
394 let username = match kind {
395 "user.deleting" => data["username"].as_str().filter(|name| !name.is_empty())?.to_owned(),
396 "user.restored" => GHOST_USERNAME.to_owned(),
397 _ => return None,
398 };
399 Some(format!("\"username\":{}", serde_json::Value::String(username)))
400}
401
402/// Marks every repository whose kept contributors contain `needle` (see
403/// [`shown_differently`]) to be worked out again on its next view, as an
404/// older version's would be.
405pub async fn rework_naming(db: &D1Database, needle: &str) -> Result<()> {
406 db.prepare("UPDATE repo_stats SET version = 0 WHERE instr(contributors, ?1) > 0")
407 .bind(&[needle.into()])?
408 .run()
409 .await?;
410 Ok(())
411}
412
413/// Whether the repository is one whose About is worked out: not a pull
414/// request's working copy.
415pub fn has_about(repo: &Repo) -> bool {
416 repo.fork_of.is_none()
417}
418
419#[cfg(test)]
420mod tests {
421 use super::*;
422
423 fn file(path: &str) -> FileEntry {
424 FileEntry { path: path.into(), hash: Some("h".into()), kind: None }
425 }
426
427 #[test]
428 fn deleting_or_restoring_an_account_reworks_the_answers_that_name_it() {
429 let kept = serde_json::to_string(&Contributor {
430 kind: g1t_contracts::about::ContributorKind::User,
431 name: "ana".into(),
432 username: Some("ana".into()),
433 avatar: None,
434 agent: None,
435 commits: 3,
436 first_at: "2026-10-01T00:00:00Z".into(),
437 last_at: "2026-10-02T00:00:00Z".into(),
438 weeks: Vec::new(),
439 })
440 .unwrap();
441 let deleting = shown_differently("user.deleting", &serde_json::json!({ "userId": "usr_ana", "username": "ana" })).unwrap();
442 assert!(kept.contains(&deleting));
443 // Not a longer name that starts the same.
444 assert!(!kept.replace("\"ana\"", "\"anabel\"").contains(&deleting));
445 // A restored account may be any ghost: those answers are worked out again.
446 let restored = shown_differently("user.restored", &serde_json::json!({ "userId": "usr_ana", "username": "ana" })).unwrap();
447 assert!(kept.replace("\"ana\"", "\"ghost\"").contains(&restored));
448 assert_eq!(shown_differently("user.updated", &serde_json::json!({ "username": "ana" })), None);
449 assert_eq!(shown_differently("user.deleting", &serde_json::json!({})), None);
450 }
451
452 #[test]
453 fn a_security_policy_is_found_where_it_may_be() {
454 assert_eq!(security_policy(&[file("src/SECURITY.md"), file(".github/SECURITY.md")]).as_deref(), Some(".github/SECURITY.md"));
455 assert_eq!(security_policy(&[file("docs/security.md"), file("SECURITY.md")]).as_deref(), Some("SECURITY.md"));
456 assert_eq!(security_policy(&[file(".g1t/SECURITY.md")]).as_deref(), Some(".g1t/SECURITY.md"));
457 assert_eq!(security_policy(&[file("SECURITY.txt")]).as_deref(), Some("SECURITY.txt"));
458 assert_eq!(security_policy(&[file("docs/guides/SECURITY.md"), file("README.md")]), None);
459 }
460
461 #[test]
462 fn a_run_starts_when_the_head_moved_and_none_is_under_way() {
463 let now = 1_000_000;
464 let none = Kept::default();
465 assert!(none.wants_run(Some("c2"), now));
466 assert!(!none.wants_run(None, now), "an empty repository has nothing to work out");
467 let current = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION, ..Kept::default() };
468 assert!(!current.wants_run(Some("c2"), now));
469 assert!(current.wants_run(Some("c3"), now));
470 let older = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION - 1.0, ..Kept::default() };
471 assert!(older.wants_run(Some("c2"), now), "an answer worked out the old way is worked out again");
472 let running = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - 1_000) as f64), ..Kept::default() };
473 assert!(!running.wants_run(Some("c3"), now));
474 let died = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - LEASE_MS - 1) as f64), ..Kept::default() };
475 assert!(died.wants_run(Some("c3"), now));
476 let partial = Kept { commit_hash: Some("c2".into()), partial: 1.0, started_ms: Some((now - 5_000) as f64), ..Kept::default() };
477 assert!(!partial.wants_run(Some("c2"), now), "a partial answer rests a lease before the next try");
478 }
479
480 #[test]
481 fn nothing_kept_is_pending() {
482 let freshness = Kept::default().freshness(Some("c1".into()));
483 assert!(freshness.pending);
484 assert!(!Kept::default().freshness(None).pending, "an empty repository is not waiting on anything");
485 }
486}