Skip to content
Sign in

flagon-io/g1t

public
0
428 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'main' into worktree-agent-a69aeabc4b0deeb971//! What a repository's About reads from its files and history: its
2//! license, its security policy, its languages and its contributors.
3//!
4//! Worked out for the default branch's head in the background, never on
5//! the way to a page: a request that finds the kept answer behind the head
6//! (or none at all) answers with what is kept and starts the work in its
7//! `wait_until` (`lib.rs`, `refresh`). One run at a time per repository,
8//! by a lease in `repo_stats.started_ms`. What it reads is kept by hash in
9//! the store's cache (trees, blob sizes, history), so working out the next
10//! commit reads only what changed. A run that runs out of time keeps what
11//! it read, marked `partial`, and the next view starts another.
12
13use std::collections::{HashMap, HashSet};
14
15use g1t_contracts::about::{ABOUT_CONTRIBUTORS, Contributor, Contributors, Freshness, LanguageShare, Languages, License, WeekCommits};
16use g1t_contracts::accounts::{EmailOwner, EmailOwnersArgs};
17use g1t_contracts::repos::{FileEntry, MAX_LISTED_FILES, Repo};
18use g1t_contracts::time::rfc3339;
19use g1t_kit::now_ms;
20use serde::{Deserialize, Serialize};
21use worker::wasm_bindgen::JsValue;
22use worker::{D1Database, Fetcher, Result};
23
24use crate::contributors::{Authored, tally};
25use crate::languages::{self, Attributes};
26use crate::license;
27use crate::store::GitRepo;
28
29/// A run older than this is taken to have died.
30pub const LEASE_MS: u64 = 120_000;
31/// How long a run reads before keeping what it has.
32const BUDGET_MS: u64 = 22_000;
33/// The most commits of history read for contributors.
34pub const MAX_HISTORY: usize = 3_000;
35/// Commits read at a time.
36const LOG_PAGE: u32 = 200;
37/// The most blobs sized for languages.
38const MAX_SIZED: usize = 6_000;
39/// Blobs sized at once: each is a round trip to the store.
40const SIZE_BATCH: usize = 32;
41/// The most distinct addresses matched to accounts.
42const MAX_MATCHED: usize = 1_000;
43/// Directories never read: nothing in them ever counts.
44const NEVER_READ: [&str; 3] = ["node_modules", "bower_components", "jspm_packages"];
45/// Where a security policy may be.
46const POLICY_DIRS: [&str; 4] = ["", ".g1t/", ".github/", "docs/"];
47
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)48/// How the About is worked out; a row kept by an older version is worked
49/// out again. 1: g1t's older commit addresses count as g1t.
50pub const STATS_VERSION: f64 = 1.0;
51
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9752/// What `repo_stats` keeps for the About, without the full contributors.
53#[derive(Debug, Default, Deserialize)]
54pub struct Kept {
55 pub commit_hash: Option<String>,
56 pub computed_at: Option<String>,
57 pub started_ms: Option<f64>,
58 pub partial: f64,
59 pub license: Option<String>,
60 pub security_policy: Option<String>,
61 pub languages: Option<String>,
62 pub contributors_total: f64,
63 pub contributors_top: Option<String>,
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)64 /// [`STATS_VERSION`] when it was worked out.
65 #[serde(default)]
66 pub version: f64,
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9767}
68
69impl Kept {
70 /// Whether a run is under way now.
71 pub fn running(&self, now: u64) -> bool {
72 self.started_ms.is_some_and(|started| now.saturating_sub(started as u64) < LEASE_MS)
73 }
74
75 /// Whether a new run should start: the head moved past what is kept,
76 /// or what is kept is partial, and none is under way.
77 pub fn wants_run(&self, head: Option<&str>, now: u64) -> bool {
78 let Some(head) = head else { return false };
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)79 let behind = self.commit_hash.as_deref() != Some(head) || self.partial > 0.0 || self.version < STATS_VERSION;
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb9780 // A partial answer is tried again at most every lease.
81 let rested = self.partial == 0.0 || self.started_ms.is_none_or(|started| now.saturating_sub(started as u64) >= LEASE_MS);
82 behind && rested && !self.running(now)
83 }
84
85 pub fn freshness(&self, head: Option<String>) -> Freshness {
86 Freshness {
87 pending: head.is_some() && self.commit_hash.is_none(),
88 head,
89 commit: self.commit_hash.clone(),
90 computed_at: self.computed_at.clone(),
91 partial: self.partial > 0.0,
92 }
93 }
94
95 pub fn license(&self) -> Option<License> {
96 self.license.as_deref().and_then(|text| serde_json::from_str(text).ok())
97 }
98
99 pub fn languages(&self) -> Vec<LanguageShare> {
100 self.languages.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
101 }
102
103 pub fn top_contributors(&self) -> Vec<Contributor> {
104 self.contributors_top.as_deref().and_then(|text| serde_json::from_str(text).ok()).unwrap_or_default()
105 }
106}
107
108/// What `repo_stats` keeps for a repository; the default when nothing is.
109pub async fn kept(db: &D1Database, repo_id: &str) -> Result<Kept> {
110 Ok(db
111 .prepare(
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)112 "SELECT commit_hash, computed_at, started_ms, partial, license, security_policy, languages, contributors_total, contributors_top, version
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97113 FROM repo_stats WHERE repo_id = ?",
114 )
115 .bind(&[repo_id.into()])?
116 .first::<Kept>(None)
117 .await?
118 .unwrap_or_default())
119}
120
121/// The whole contributors answer kept for a repository.
122#[derive(Debug, Default, Serialize, Deserialize)]
123struct KeptContributors {
124 commits: u32,
125 contributors: Vec<Contributor>,
126 weeks: Vec<WeekCommits>,
127}
128
129/// The Contributors page's answer, as kept.
130pub async fn contributors(db: &D1Database, repo_id: &str, head: Option<String>) -> Result<Contributors> {
131 #[derive(Deserialize)]
132 struct Row {
133 contributors: Option<String>,
134 }
135 let summary = kept(db, repo_id).await?;
136 let full: KeptContributors = db
137 .prepare("SELECT contributors FROM repo_stats WHERE repo_id = ?")
138 .bind(&[repo_id.into()])?
139 .first::<Row>(None)
140 .await?
141 .and_then(|row| row.contributors)
142 .and_then(|text| serde_json::from_str(&text).ok())
143 .unwrap_or_default();
144 Ok(Contributors {
145 freshness: summary.freshness(head),
146 total: summary.contributors_total as u32,
147 commits: full.commits,
148 contributors: full.contributors,
149 weeks: full.weeks,
150 })
151}
152
153/// The languages answer, as kept.
154pub fn languages_of(kept: &Kept, head: Option<String>) -> Languages {
155 Languages { freshness: kept.freshness(head), languages: kept.languages() }
156}
157
158/// Takes the lease for a run, unless one is under way. Whether it was taken.
159pub async fn claim(db: &D1Database, repo_id: &str, now: u64) -> Result<bool> {
160 let taken = db
161 .prepare(
162 "INSERT INTO repo_stats (repo_id, started_ms) VALUES (?1, ?2)
163 ON CONFLICT (repo_id) DO UPDATE SET started_ms = excluded.started_ms
164 WHERE repo_stats.started_ms IS NULL OR repo_stats.started_ms < ?3
165 RETURNING repo_id",
166 )
167 .bind(&[repo_id.into(), JsValue::from_f64(now as f64), JsValue::from_f64(now.saturating_sub(LEASE_MS) as f64)])?
168 .first::<serde_json::Value>(None)
169 .await?;
170 Ok(taken.is_some())
171}
172
173/// Everything one run worked out.
174#[derive(Debug, Default)]
175pub struct Worked {
176 pub commit: String,
177 pub partial: bool,
178 pub license: Option<License>,
179 pub security_policy: Option<String>,
180 pub languages: Vec<LanguageShare>,
181 pub contributors: Vec<Contributor>,
182 pub contributors_total: u32,
183 pub commits: u32,
184 pub weeks: Vec<WeekCommits>,
185}
186
187/// Keeps a run's answer and lets the lease go. A partial answer keeps its
188/// lease's start, so the next is not tried until the lease is over.
189pub async fn keep(db: &D1Database, repo_id: &str, worked: &Worked) -> Result<()> {
190 let top: Vec<Contributor> = worked
191 .contributors
192 .iter()
193 .take(ABOUT_CONTRIBUTORS)
194 .map(|contributor| Contributor { weeks: Vec::new(), ..contributor.clone() })
195 .collect();
196 let full = KeptContributors { commits: worked.commits, contributors: worked.contributors.clone(), weeks: worked.weeks.clone() };
197 db.prepare(
198 "UPDATE repo_stats SET commit_hash = ?2, computed_at = ?3, started_ms = CASE WHEN ?4 = 1 THEN started_ms ELSE NULL END,
199 partial = ?4, license = ?5, security_policy = ?6, languages = ?7, contributors_total = ?8, contributors_top = ?9,
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)200 contributors = ?10, version = ?11
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97201 WHERE repo_id = ?1",
202 )
203 .bind(&[
204 repo_id.into(),
205 worked.commit.as_str().into(),
206 rfc3339(now_ms()).into(),
207 JsValue::from_f64(if worked.partial { 1.0 } else { 0.0 }),
208 match &worked.license {
209 Some(license) => json(license).into(),
210 None => JsValue::NULL,
211 },
212 match &worked.security_policy {
213 Some(path) => path.as_str().into(),
214 None => JsValue::NULL,
215 },
216 json(&worked.languages).into(),
217 JsValue::from_f64(f64::from(worked.contributors_total)),
218 json(&top).into(),
219 json(&full).into(),
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)220 JsValue::from_f64(STATS_VERSION),
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97221 ])?
222 .run()
223 .await?;
224 Ok(())
225}
226
227/// Lets the lease go after a run that failed, so the next view tries again
228/// once the lease would have ended anyway.
229pub async fn release(db: &D1Database, repo_id: &str) {
230 let statement = db
231 .prepare("UPDATE repo_stats SET started_ms = ?2 WHERE repo_id = ?1")
232 .bind(&[repo_id.into(), JsValue::from_f64(now_ms().saturating_sub(LEASE_MS / 2) as f64)]);
233 if let Ok(statement) = statement
234 && let Err(error) = statement.run().await
235 {
236 worker::console_error!("the About's lease of {repo_id} was not let go: {error}");
237 }
238}
239
240fn json<T: Serialize>(value: &T) -> String {
241 serde_json::to_string(value).unwrap_or_else(|_| "null".to_owned())
242}
243
244/// The security policy among a repository's files, at the root or in
245/// `.g1t`, `.github` or `docs`.
246pub fn security_policy(files: &[FileEntry]) -> Option<String> {
247 POLICY_DIRS.iter().find_map(|dir| {
248 files
249 .iter()
250 .find(|file| {
251 file.path
252 .strip_prefix(dir)
253 .is_some_and(|name| !name.contains('/') && (name.eq_ignore_ascii_case("security.md") || name.eq_ignore_ascii_case("security.txt") && dir.is_empty()))
254 })
255 .map(|file| file.path.clone())
256 })
257}
258
259/// Works out the About of `repo` at `head` (a commit). `identity` matches
260/// authors' addresses to accounts.
261pub async fn work_out<R: GitRepo>(git: &R, head: &str, tree: &str, identity: Option<&Fetcher>) -> Result<Worked> {
262 let started = now_ms();
263 let out_of_time = move || now_ms().saturating_sub(started) > BUDGET_MS;
264 let never: Vec<String> = NEVER_READ.iter().map(|dir| (*dir).to_owned()).collect();
265 let files = async {
266 let (files, truncated) = crate::listing::changed(git, None, tree, &never, MAX_LISTED_FILES).await?;
267 let root: Vec<&FileEntry> = files.iter().filter(|file| !file.path.contains('/')).collect();
268 // The license: the best-named license file that reads as text.
269 let mut license = None;
270 for name in license::candidates(root.iter().map(|file| file.path.as_str())) {
271 let Some(hash) = root.iter().find(|file| file.path == name).and_then(|file| file.hash.clone()) else { continue };
272 if let Some(text) = git.read_blob(&hash).await?.and_then(|bytes| String::from_utf8(bytes).ok()) {
273 license = Some(license::detect(name, &text));
274 break;
275 }
276 }
277 let attributes = match root.iter().find(|file| file.path == ".gitattributes").and_then(|file| file.hash.clone()) {
278 Some(hash) => git
279 .read_blob(&hash)
280 .await?
281 .map(|bytes| Attributes::parse(&String::from_utf8_lossy(&bytes)))
282 .unwrap_or_default(),
283 None => Attributes::default(),
284 };
285 let policy = security_policy(&files);
286 // Each counted file's language, its blob sized once.
287 let mut counted: Vec<(&'static languages::Language, String)> = Vec::new();
288 for file in &files {
289 let Some(hash) = &file.hash else { continue };
290 if let Some(language) = languages::counted(&file.path, &attributes) {
291 counted.push((language, hash.clone()));
292 }
293 }
294 let mut partial = truncated || counted.len() > MAX_SIZED;
295 counted.truncate(MAX_SIZED);
296 let distinct: Vec<String> = counted.iter().map(|(_, hash)| hash.clone()).collect::<HashSet<_>>().into_iter().collect();
297 let mut sizes: HashMap<String, u64> = HashMap::with_capacity(distinct.len());
298 for batch in distinct.chunks(SIZE_BATCH) {
299 if out_of_time() {
300 partial = true;
301 break;
302 }
303 let found = futures_util::future::join_all(batch.iter().map(|hash| git.blob_size(hash))).await;
304 for (hash, size) in batch.iter().zip(found) {
305 if let Ok(Some(size)) = size {
306 sizes.insert(hash.clone(), size);
307 }
308 }
309 }
310 let shares = languages::shares(counted.iter().filter_map(|(language, hash)| Some((*language, *sizes.get(hash)?))));
311 Ok::<_, worker::Error>((license, policy, shares, partial))
312 };
313 let history = async {
314 let mut commits: Vec<Authored> = Vec::new();
315 let mut next = Some(head.to_owned());
316 let mut partial = false;
317 while let Some(from) = next.take() {
318 let page = git.log(&from, LOG_PAGE).await?;
319 let more = page.len() == LOG_PAGE as usize;
320 let after = page.last().and_then(|commit| commit.parents.first().cloned());
321 for commit in page {
322 commits.push(Authored { name: commit.author.name, email: commit.author.email, at: commit.authored_at });
323 }
324 if more && after.is_some() {
325 if commits.len() >= MAX_HISTORY || out_of_time() {
326 partial = true;
327 } else {
328 next = after;
329 }
330 }
331 }
332 commits.truncate(MAX_HISTORY);
333 Ok::<_, worker::Error>((commits, partial))
334 };
335 let (files, history) = futures_util::future::join(files, history).await;
336 let (license, security_policy, languages, files_partial) = files?;
337 let (commits, history_partial) = history?;
338 let owners = match identity {
339 Some(identity) => owners(identity, &commits).await,
340 None => HashMap::new(),
341 };
342 let (contributors, contributors_total, weeks) = tally(&commits, &owners);
343 Ok(Worked {
344 commit: head.to_owned(),
345 partial: files_partial || history_partial,
346 license,
347 security_policy,
348 languages,
349 contributors,
350 contributors_total,
351 commits: commits.len() as u32,
352 weeks,
353 })
354}
355
356/// The accounts behind the commits' addresses, the most frequent first. A
357/// failure matches none: the authors show by name.
358async fn owners(identity: &Fetcher, commits: &[Authored]) -> HashMap<String, EmailOwner> {
359 let mut counts: HashMap<String, usize> = HashMap::new();
360 for commit in commits.iter().filter(|commit| !crate::contributors::is_g1t(commit)) {
361 let email = commit.email.trim().to_ascii_lowercase();
362 if !email.is_empty() {
363 *counts.entry(email).or_default() += 1;
364 }
365 }
366 let mut emails: Vec<(String, usize)> = counts.into_iter().collect();
367 emails.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
368 emails.truncate(MAX_MATCHED);
369 let mut owners = HashMap::new();
370 for chunk in emails.chunks(200) {
371 let args = EmailOwnersArgs { emails: chunk.iter().map(|(email, _)| email.clone()).collect() };
372 match g1t_kit::call::<_, HashMap<String, EmailOwner>>(identity, "email_owners", &args).await {
373 Ok(found) => owners.extend(found),
374 Err(error) => worker::console_error!("contributors' accounts not matched: {error}"),
375 }
376 }
377 owners
378}
379
380/// Whether the repository is one whose About is worked out: not a pull
381/// request's working copy.
382pub fn has_about(repo: &Repo) -> bool {
383 repo.fork_of.is_none()
384}
385
386#[cfg(test)]
387mod tests {
388 use super::*;
389
390 fn file(path: &str) -> FileEntry {
391 FileEntry { path: path.into(), hash: Some("h".into()) }
392 }
393
394 #[test]
395 fn a_security_policy_is_found_where_it_may_be() {
396 assert_eq!(security_policy(&[file("src/SECURITY.md"), file(".github/SECURITY.md")]).as_deref(), Some(".github/SECURITY.md"));
397 assert_eq!(security_policy(&[file("docs/security.md"), file("SECURITY.md")]).as_deref(), Some("SECURITY.md"));
398 assert_eq!(security_policy(&[file(".g1t/SECURITY.md")]).as_deref(), Some(".g1t/SECURITY.md"));
399 assert_eq!(security_policy(&[file("SECURITY.txt")]).as_deref(), Some("SECURITY.txt"));
400 assert_eq!(security_policy(&[file("docs/guides/SECURITY.md"), file("README.md")]), None);
401 }
402
403 #[test]
404 fn a_run_starts_when_the_head_moved_and_none_is_under_way() {
405 let now = 1_000_000;
406 let none = Kept::default();
407 assert!(none.wants_run(Some("c2"), now));
408 assert!(!none.wants_run(None, now), "an empty repository has nothing to work out");
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)409 let current = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION, ..Kept::default() };
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97410 assert!(!current.wants_run(Some("c2"), now));
411 assert!(current.wants_run(Some("c3"), now));
Merge commit authors and hovercards: commits show their accounts, every name opens a card (repos 0016)412 let older = Kept { commit_hash: Some("c2".into()), version: STATS_VERSION - 1.0, ..Kept::default() };
413 assert!(older.wants_run(Some("c2"), now), "an answer worked out the old way is worked out again");
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97414 let running = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - 1_000) as f64), ..Kept::default() };
415 assert!(!running.wants_run(Some("c3"), now));
416 let died = Kept { commit_hash: Some("c2".into()), started_ms: Some((now - LEASE_MS - 1) as f64), ..Kept::default() };
417 assert!(died.wants_run(Some("c3"), now));
418 let partial = Kept { commit_hash: Some("c2".into()), partial: 1.0, started_ms: Some((now - 5_000) as f64), ..Kept::default() };
419 assert!(!partial.wants_run(Some("c2"), now), "a partial answer rests a lease before the next try");
420 }
421
422 #[test]
423 fn nothing_kept_is_pending() {
424 let freshness = Kept::default().freshness(Some("c1".into()));
425 assert!(freshness.pending);
426 assert!(!Kept::default().freshness(None).pending, "an empty repository is not waiting on anything");
427 }
428}

This file's history is long; its oldest lines are credited to the oldest commit read.