Skip to content
Sign up

flagon-io/g1t

public
0
477 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'main' into worktree-agent-a69aeabc4b0deeb971//! What a repository's About reads from its files and history: its
2//! license, its security policy, its languages and its contributors.
3//!
4//! Worked out for the default branch's head in the background, never on
5//! the way to a page: a request that finds the kept answer behind the head
6//! (or none at all) answers with what is kept and starts the work in its
7//! `wait_until` (`lib.rs`, `refresh`). One run at a time per repository,
8//! by a lease in `repo_stats.started_ms`. What it reads is kept by hash in
9//! the store's cache (trees, blob sizes, history), so working out the next
10//! commit reads only what changed. A run that runs out of time keeps what
11//! it read, marked `partial`, and the next view starts another.
12
13use std::collections::{HashMap, HashSet};
14
15use g1t_contracts::about::{ABOUT_CONTRIBUTORS, Contributor, Contributors, Freshness, LanguageShare, Languages, License, WeekCommits};
Merge leftovers: plan activity filtered in the query, pushes counted by account, ghost during the deletion window, profile time zones (identity 0039)16use g1t_contracts::account_deletion::GHOST_USERNAME;
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9717use g1t_contracts::accounts::{EmailOwner, EmailOwnersArgs};
18use g1t_contracts::repos::{FileEntry, MAX_LISTED_FILES, Repo};
19use g1t_contracts::time::rfc3339;
20use g1t_kit::now_ms;
21use serde::{Deserialize, Serialize};
22use worker::wasm_bindgen::JsValue;
23use worker::{D1Database, Fetcher, Result};
24
25use crate::contributors::{Authored, tally};
26use crate::languages::{self, Attributes};
27use crate::license;
28use crate::store::GitRepo;
29
30/// A run older than this is taken to have died.
31pub const LEASE_MS: u64 = 120_000;
32/// How long a run reads before keeping what it has.
33const BUDGET_MS: u64 = 22_000;
34/// The most commits of history read for contributors.
35pub const MAX_HISTORY: usize = 3_000;
36/// Commits read at a time.
37const LOG_PAGE: u32 = 200;
38/// The most blobs sized for languages.
39const MAX_SIZED: usize = 6_000;
40/// Blobs sized at once: each is a round trip to the store.
41const SIZE_BATCH: usize = 32;
42/// The most distinct addresses matched to accounts.
43const MAX_MATCHED: usize = 1_000;
44/// Directories never read: nothing in them ever counts.
45const NEVER_READ: [&str; 3] = ["node_modules", "bower_components", "jspm_packages"];
46/// Where a security policy may be.
47const POLICY_DIRS: [&str; 4] = ["", ".g1t/", ".github/", "docs/"];
48
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)49/// How the About is worked out; a row kept by an older version is worked
50/// out again. 1: g1t's older commit addresses count as g1t.
51pub const STATS_VERSION: f64 = 1.0;
52
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9753/// What `repo_stats` keeps for the About, without the full contributors.
54#[derive(Debug, Default, Deserialize)]
55pub struct Kept {
56 pub commit_hash: Option<String>,
57 pub computed_at: Option<String>,
58 pub started_ms: Option<f64>,
59 pub partial: f64,
60 pub license: Option<String>,
61 pub security_policy: Option<String>,
62 pub languages: Option<String>,
63 pub contributors_total: f64,
64 pub contributors_top: Option<String>,
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)65 /// [`STATS_VERSION`] when it was worked out.
66 #[serde(default)]
67 pub version: f64,
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9768}
69
70impl Kept {
71 /// Whether a run is under way now.
72 pub fn running(&self, now: u64) -> bool {
73 self.started_ms.is_some_and(|started| now.saturating_sub(started as u64) < LEASE_MS)
74 }
75
76 /// Whether a new run should start: the head moved past what is kept,
77 /// or what is kept is partial, and none is under way.
78 pub fn wants_run(&self, head: Option<&str>, now: u64) -> bool {
79 let Some(head) = head else { return false };
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)80 let behind = self.commit_hash.as_deref() != Some(head) || self.partial > 0.0 || self.version < STATS_VERSION;
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9781 // A partial answer is tried again at most every lease.
82 let rested = self.partial == 0.0 || self.started_ms.is_none_or(|started| now.saturating_sub(started as u64) >= LEASE_MS);
83 behind && rested && !self.running(now)
84 }
85
86 pub fn freshness(&self, head: Option<String>) -> Freshness {
87 Freshness {
88 pending: head.is_some() && self.commit_hash.is_none(),
89 head,
90 commit: self.commit_hash.clone(),
91 computed_at: self.computed_at.clone(),
92 partial: self.partial > 0.0,
93 }
94 }
95
96 pub fn license(&self) -> Option<License> {
97 self.license.as_deref().and_then(|text| serde_json::from_str(text).ok())
98 }
99
100 pub fn languages(&self) -> Vec<LanguageShare> {
101 self.languages.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
102 }
103
104 pub fn top_contributors(&self) -> Vec<Contributor> {
105 self.contributors_top.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
106 }
107}
108
109/// What `repo_stats` keeps for a repository; the default when nothing is.
110pub async fn kept(db: &D1Database, repo_id: &str) -> Result<Kept> {
111 Ok(db
112 .prepare(
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)113 "SELECT commit_hash, computed_at, started_ms, partial, license, security_policy, languages, contributors_total, contributors_top, version
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97114 FROM repo_stats WHERE repo_id = ?",
115 )
116 .bind(&[repo_id.into()])?
117 .first::<Kept>(None)
118 .await?
119 .unwrap_or_default())
120}
121
122/// The whole contributors answer kept for a repository.
123#[derive(Debug, Default, Serialize, Deserialize)]
124struct KeptContributors {
125 commits: u32,
126 contributors: Vec<Contributor>,
127 weeks: Vec<WeekCommits>,
128}
129
130/// The Contributors page's answer, as kept.
131pub async fn contributors(db: &D1Database, repo_id: &str, head: Option<String>) -> Result<Contributors> {
132 #[derive(Deserialize)]
133 struct Row {
134 contributors: Option<String>,
135 }
136 let summary = kept(db, repo_id).await?;
137 let full: KeptContributors = db
138 .prepare("SELECT contributors FROM repo_stats WHERE repo_id = ?")
139 .bind(&[repo_id.into()])?
140 .first::<Row>(None)
141 .await?
142 .and_then(|row| row.contributors)
143 .and_then(|text| serde_json::from_str(&text).ok())
144 .unwrap_or_default();
145 Ok(Contributors {
146 freshness: summary.freshness(head),
147 total: summary.contributors_total as u32,
148 commits: full.commits,
149 contributors: full.contributors,
150 weeks: full.weeks,
151 })
152}
153
154/// The languages answer, as kept.
155pub fn languages_of(kept: &Kept, head: Option<String>) -> Languages {
156 Languages { freshness: kept.freshness(head), languages: kept.languages() }
157}
158
159/// Takes the lease for a run, unless one is under way. Whether it was taken.
160pub async fn claim(db: &D1Database, repo_id: &str, now: u64) -> Result<bool> {
161 let taken = db
162 .prepare(
163 "INSERT INTO repo_stats (repo_id, started_ms) VALUES (?1, ?2)
164 ON CONFLICT (repo_id) DO UPDATE SET started_ms = excluded.started_ms
165 WHERE repo_stats.started_ms IS NULL OR repo_stats.started_ms < ?3
166 RETURNING repo_id",
167 )
168 .bind(&[repo_id.into(), JsValue::from_f64(now as f64), JsValue::from_f64(now.saturating_sub(LEASE_MS) as f64)])?
169 .first::<serde_json::Value>(None)
170 .await?;
171 Ok(taken.is_some())
172}
173
174/// Everything one run worked out.
175#[derive(Debug, Default)]
176pub struct Worked {
177 pub commit: String,
178 pub partial: bool,
179 pub license: Option<License>,
180 pub security_policy: Option<String>,
181 pub languages: Vec<LanguageShare>,
182 pub contributors: Vec<Contributor>,
183 pub contributors_total: u32,
184 pub commits: u32,
185 pub weeks: Vec<WeekCommits>,
186}
187
188/// Keeps a run's answer and lets the lease go. A partial answer keeps its
189/// lease's start, so the next is not tried until the lease is over.
190pub async fn keep(db: &D1Database, repo_id: &str, worked: &Worked) -> Result<()> {
191 let top: Vec<Contributor> = worked
192 .contributors
193 .iter()
194 .take(ABOUT_CONTRIBUTORS)
195 .map(|contributor| Contributor { weeks: Vec::new(), ..contributor.clone() })
196 .collect();
197 let full = KeptContributors { commits: worked.commits, contributors: worked.contributors.clone(), weeks: worked.weeks.clone() };
198 db.prepare(
199 "UPDATE repo_stats SET commit_hash = ?2, computed_at = ?3, started_ms = CASE WHEN ?4 = 1 THEN started_ms ELSE NULL END,
200 partial = ?4, license = ?5, security_policy = ?6, languages = ?7, contributors_total = ?8, contributors_top = ?9,
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)201 contributors = ?10, version = ?11
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97202 WHERE repo_id = ?1",
203 )
204 .bind(&[
205 repo_id.into(),
206 worked.commit.as_str().into(),
207 rfc3339(now_ms()).into(),
208 JsValue::from_f64(if worked.partial { 1.0 } else { 0.0 }),
209 match &worked.license {
210 Some(license) => json(license).into(),
211 None => JsValue::NULL,
212 },
213 match &worked.security_policy {
214 Some(path) => path.as_str().into(),
215 None => JsValue::NULL,
216 },
217 json(&worked.languages).into(),
218 JsValue::from_f64(f64::from(worked.contributors_total)),
219 json(&top).into(),
220 json(&full).into(),
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)221 JsValue::from_f64(STATS_VERSION),
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97222 ])?
223 .run()
224 .await?;
225 Ok(())
226}
227
228/// Lets the lease go after a run that failed, so the next view tries again
229/// once the lease would have ended anyway.
230pub async fn release(db: &D1Database, repo_id: &str) {
231 let statement = db
232 .prepare("UPDATE repo_stats SET started_ms = ?2 WHERE repo_id = ?1")
233 .bind(&[repo_id.into(), JsValue::from_f64(now_ms().saturating_sub(LEASE_MS / 2) as f64)]);
234 if let Ok(statement) = statement
235 && let Err(error) = statement.run().await
236 {
237 worker::console_error!("the About's lease of {repo_id} was not let go: {error}");
238 }
239}
240
241fn json<T: Serialize>(value: &T) -> String {
242 serde_json::to_string(value).unwrap_or_else(|_| "null".to_owned())
243}
244
245/// The security policy among a repository's files, at the root or in
246/// `.g1t`, `.github` or `docs`.
247pub fn security_policy(files: &[FileEntry]) -> Option<String> {
248 POLICY_DIRS.iter().find_map(|dir| {
249 files
250 .iter()
251 .find(|file| {
252 file.path
253 .strip_prefix(dir)
254 .is_some_and(|name| !name.contains('/') && (name.eq_ignore_ascii_case("security.md") || name.eq_ignore_ascii_case("security.txt") && dir.is_empty()))
255 })
256 .map(|file| file.path.clone())
257 })
258}
259
260/// Works out the About of `repo` at `head` (a commit). `identity` matches
261/// authors' addresses to accounts.
262pub async fn work_out<R: GitRepo>(git: &R, head: &str, tree: &str, identity: Option<&Fetcher>) -> Result<Worked> {
263 let started = now_ms();
264 let out_of_time = move || now_ms().saturating_sub(started) > BUDGET_MS;
265 let never: Vec<String> = NEVER_READ.iter().map(|dir| (*dir).to_owned()).collect();
266 let files = async {
267 let (files, truncated) = crate::listing::changed(git, None, tree, &never, MAX_LISTED_FILES).await?;
268 let root: Vec<&FileEntry> = files.iter().filter(|file| !file.path.contains('/')).collect();
269 // The license: the best-named license file that reads as text.
270 let mut license = None;
271 for name in license::candidates(root.iter().map(|file| file.path.as_str())) {
272 let Some(hash) = root.iter().find(|file| file.path == name).and_then(|file| file.hash.clone()) else { continue };
273 if let Some(text) = git.read_blob(&hash).await?.and_then(|bytes| String::from_utf8(bytes).ok()) {
274 license = Some(license::detect(name, &text));
275 break;
276 }
277 }
278 let attributes = match root.iter().find(|file| file.path == ".gitattributes").and_then(|file| file.hash.clone()) {
279 Some(hash) => git
280 .read_blob(&hash)
281 .await?
282 .map(|bytes| Attributes::parse(&String::from_utf8_lossy(&bytes)))
283 .unwrap_or_default(),
284 None => Attributes::default(),
285 };
286 let policy = security_policy(&files);
287 // Each counted file's language, its blob sized once.
288 let mut counted: Vec<(&'static languages::Language, String)> = Vec::new();
289 for file in &files {
290 let Some(hash) = &file.hash else { continue };
291 if let Some(language) = languages::counted(&file.path, &attributes) {
292 counted.push((language, hash.clone()));
293 }
294 }
295 let mut partial = truncated || counted.len() > MAX_SIZED;
296 counted.truncate(MAX_SIZED);
297 let distinct: Vec<String> = counted.iter().map(|(_, hash)| hash.clone()).collect::<HashSet<_>>().into_iter().collect();
298 let mut sizes: HashMap<String, u64> = HashMap::with_capacity(distinct.len());
299 for batch in distinct.chunks(SIZE_BATCH) {
300 if out_of_time() {
301 partial = true;
302 break;
303 }
304 let found = futures_util::future::join_all(batch.iter().map(|hash| git.blob_size(hash))).await;
305 for (hash, size) in batch.iter().zip(found) {
306 if let Ok(Some(size)) = size {
307 sizes.insert(hash.clone(), size);
308 }
309 }
310 }
311 let shares = languages::shares(counted.iter().filter_map(|(language, hash)| Some((*language, *sizes.get(hash)?))));
312 Ok::<_, worker::Error>((license, policy, shares, partial))
313 };
314 let history = async {
315 let mut commits: Vec<Authored> = Vec::new();
316 let mut next = Some(head.to_owned());
317 let mut partial = false;
318 while let Some(from) = next.take() {
319 let page = git.log(&from, LOG_PAGE).await?;
320 let more = page.len() == LOG_PAGE as usize;
321 let after = page.last().and_then(|commit| commit.parents.first().cloned());
322 for commit in page {
323 commits.push(Authored { name: commit.author.name, email: commit.author.email, at: commit.authored_at });
324 }
325 if more && after.is_some() {
326 if commits.len() >= MAX_HISTORY || out_of_time() {
327 partial = true;
328 } else {
329 next = after;
330 }
331 }
332 }
333 commits.truncate(MAX_HISTORY);
334 Ok::<_, worker::Error>((commits, partial))
335 };
336 let (files, history) = futures_util::future::join(files, history).await;
337 let (license, security_policy, languages, files_partial) = files?;
338 let (commits, history_partial) = history?;
339 let owners = match identity {
340 Some(identity) => owners(identity, &commits).await,
341 None => HashMap::new(),
342 };
343 let (contributors, contributors_total, weeks) = tally(&commits, &owners);
344 Ok(Worked {
345 commit: head.to_owned(),
346 partial: files_partial || history_partial,
347 license,
348 security_policy,
349 languages,
350 contributors,
351 contributors_total,
352 commits: commits.len() as u32,
353 weeks,
354 })
355}
356
357/// The accounts behind the commits' addresses, the most frequent first. A
358/// failure matches none: the authors show by name.
359async fn owners(identity: &Fetcher, commits: &[Authored]) -> HashMap<String, EmailOwner> {
360 let mut counts: HashMap<String, usize> = HashMap::new();
361 for commit in commits.iter().filter(|commit| !crate::contributors::is_g1t(commit)) {
362 let email = commit.email.trim().to_ascii_lowercase();
363 if !email.is_empty() {
364 *counts.entry(email).or_default() += 1;
365 }
366 }
367 let mut emails: Vec<(String, usize)> = counts.into_iter().collect();
368 emails.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
369 emails.truncate(MAX_MATCHED);
370 let mut owners = HashMap::new();
371 for chunk in emails.chunks(200) {
372 let args = EmailOwnersArgs { emails: chunk.iter().map(|(email, _)| email.clone()).collect() };
373 match g1t_kit::call::<_, HashMap<String, EmailOwner>>(identity, "email_owners", &args).await {
374 Ok(found) => owners.extend(found),
375 Err(error) => worker::console_error!("contributors' accounts not matched: {error}"),
376 }
377 }
378 owners
379}
380
Merge leftovers: plan activity filtered in the query, pushes counted by account, ghost during the deletion window, profile time zones (identity 0039)381/// What a kept contributors answer says when it names an account that
382/// `kind` (a `user.deleting` or `user.restored` event with `data`) changes:
383/// a deleted account's username, which is ghost from now on, or ghost's,
384/// which a restored account may be among. `None` for any other event.
385pub fn shown_differently(kind: &str, data: &serde_json::Value) -> Option<String> {
386 let username = match kind {
387 "user.deleting" => data["username"].as_str().filter(|name| !name.is_empty())?.to_owned(),
388 "user.restored" => GHOST_USERNAME.to_owned(),
389 _ => return None,
390 };
391 Some(format!("\"username\":{}", serde_json::Value::String(username)))
392}
393
394/// Marks every repository whose kept contributors contain `needle` (see
395/// [`shown_differently`]) to be worked out again on its next view, as an
396/// older version's would be.
397pub async fn rework_naming(db: &D1Database, needle: &str) -> Result<()> {
398 db.prepare("UPDATE repo_stats SET version = 0 WHERE instr(contributors, ?1) > 0")
399 .bind(&[needle.into()])?
400 .run()
401 .await?;
402 Ok(())
403}
404
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97405/// Whether the repository is one whose About is worked out: not a pull
406/// request's working copy.
407pub fn has_about(repo: &Repo) -> bool {
408 repo.fork_of.is_none()
409}
410
411#[cfg(test)]
412mod tests {
413 use super::*;
414
415 fn file(path: &str) -> FileEntry {
416 FileEntry { path: path.into(), hash: Some("h".into()) }
417 }
418
419 #[test]
Merge leftovers: plan activity filtered in the query, pushes counted by account, ghost during the deletion window, profile time zones (identity 0039)420 fn deleting_or_restoring_an_account_reworks_the_answers_that_name_it() {
421 let kept = serde_json::to_string(&Contributor {
422 kind: g1t_contracts::about::ContributorKind::User,
423 name: "ana".into(),
424 username: Some("ana".into()),
425 avatar: None,
426 commits: 3,
427 first_at: "2026-10-01T00:00:00Z".into(),
428 last_at: "2026-10-02T00:00:00Z".into(),
429 weeks: Vec::new(),
430 })
431 .unwrap();
432 let deleting = shown_differently("user.deleting", &serde_json::json!({ "userId": "usr_ana", "username": "ana" })).unwrap();
433 assert!(kept.contains(&deleting));
434 // Not a longer name that starts the same.
435 assert!(!kept.replace("\"ana\"", "\"anabel\"").contains(&deleting));
436 // A restored account may be any ghost: those answers are worked out again.
437 let restored = shown_differently("user.restored", &serde_json::json!({ "userId": "usr_ana", "username": "ana" })).unwrap();
438 assert!(kept.replace("\"ana\"", "\"ghost\"").contains(&restored));
439 assert_eq!(shown_differently("user.updated", &serde_json::json!({ "username": "ana" })), None);
440 assert_eq!(shown_differently("user.deleting", &serde_json::json!({})), None);
441 }
442
443 #[test]
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97444 fn a_security_policy_is_found_where_it_may_be() {
445 assert_eq!(security_policy(&[file("src/SECURITY.md"), file(".github/SECURITY.md")]).as_deref(), Some(".github/SECURITY.md"));
446 assert_eq!(security_policy(&[file("docs/security.md"), file("SECURITY.md")]).as_deref(), Some("SECURITY.md"));
447 assert_eq!(security_policy(&[file(".g1t/SECURITY.md")]).as_deref(), Some(".g1t/SECURITY.md"));
448 assert_eq!(security_policy(&[file("SECURITY.txt")]).as_deref(), Some("SECURITY.txt"));
449 assert_eq!(security_policy(&[file("docs/guides/SECURITY.md"), file("README.md")]), None);
450 }
451
452 #[test]
453 fn a_run_starts_when_the_head_moved_and_none_is_under_way() {
454 let now = 1_000_000;
455 let none = Kept::default();
456 assert!(none.wants_run(Some("c2"), now));
457 assert!(!none.wants_run(None, now), "an empty repository has nothing to work out");
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)458 let current = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION, ..Kept::default() };
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97459 assert!(!current.wants_run(Some("c2"), now));
460 assert!(current.wants_run(Some("c3"), now));
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)461 let older = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION - 1.0, ..Kept::default() };
462 assert!(older.wants_run(Some("c2"), now), "an answer worked out the old way is worked out again");
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97463 let running = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - 1_000) as f64), ..Kept::default() };
464 assert!(!running.wants_run(Some("c3"), now));
465 let died = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - LEASE_MS - 1) as f64), ..Kept::default() };
466 assert!(died.wants_run(Some("c3"), now));
467 let partial = Kept { commit_hash: Some("c2".into()), partial: 1.0, started_ms: Some((now - 5_000) as f64), ..Kept::default() };
468 assert!(!partial.wants_run(Some("c2"), now), "a partial answer rests a lease before the next try");
469 }
470
471 #[test]
472 fn nothing_kept_is_pending() {
473 let freshness = Kept::default().freshness(Some("c1".into()));
474 assert!(freshness.pending);
475 assert!(!Kept::default().freshness(None).pending, "an empty repository is not waiting on anything");
476 }
477}

This file's history is long; its oldest lines are credited to the oldest commit read.