Skip to content
Sign in

flagon-io/g1t

public
0

Commit

Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing

Every read of a path that is not a file at a ref went to the store, which counts it as a rejected read (985 on 2026-10-07, nearly all crawlers on public blob pages). read_file now notes such a miss beside the file's cache key (absent/<key>, 10 minutes; a push moves the refs version, so a branch's note goes with it) and looks for the note alongside the key's own lookup, so a cold read waits no longer. Metered as cache.absent_hit. Never for the fallback store, which can be behind.

syntaqxcommitted Parent61ffc80Browse files
2 files+51−50/2 viewed
+4−1
444444 2026-10-07: four anonymous views of a missing file on a public repository made four, a real
445445 file none. Nearly all came from crawlers (ClaudeBot, GPTBot) on public `blob/<sha>/…` pages
446446 and pull requests' working copies, hour after hour with no git at all. The site answers 404
447− correctly; each is one store read, and a miss is not cached. They are not operations.
447+ correctly. Each was one store read; since 2026-10-07 a path found not to be a file is
448+ remembered for 10 minutes beside its cache key (`store.rs` `known_absent`, metered as
449+ `cache.absent_hit`; never for the fallback store), so a crawler repeating it reads nothing.
450+ They are not operations.
448451 `--hours DAY` lists them by message and repository.
449452
450453 **2026-10-07: where the gap came from.** Cloudflare counted 581 operations (pull 535, push 39,
+47−4
723723 /// longer than this, which bounds how stale one can be should a change
724724 /// ever fail to move it.
725725 const VERSIONED_MAX_AGE: &str = "public, max-age=300";
726+/// How long a path found not to be a file at a ref is remembered. Short:
727+/// a miss by commit hash is true for good, but one for a commit not yet in
728+/// the store would not be.
729+const ABSENT_MAX_AGE: &str = "public, max-age=600";
726730
727731 /// Whether a ref is a full commit hash (SHA-1 or SHA-256), whose history
728732 /// can be kept for good.
755759 version.map(|version| CacheKey::Versioned(format!("vfile/{version}/{}/{path}", g1t_secrets::sha256_hex(git_ref))))
756760 }
757761
762+/// Where `read_file` notes that its key is not a file (see `known_absent`).
763+fn absent_path(key: &CacheKey) -> String {
764+ match key {
765+ CacheKey::Forever(path) | CacheKey::Versioned(path) => format!("absent/{path}"),
766+ }
767+}
768+
758769 /// The cache key for the branch list.
759770 pub fn branches_key(version: Option<u64>) -> Option<CacheKey> {
760771 version.map(|version| CacheKey::Versioned(format!("branches/{version}")))
842853 let _ = worker::Cache::default().put(self.cache_url(path), response).await;
843854 }
844855
856+ /// Whether `read_file`'s key was found not to be a file a little while
857+ /// ago. Metered only when it was (`cache.absent_hit`): the lookup runs
858+ /// beside the key's own, which already counts the miss.
859+ async fn known_absent(&self, key: &CacheKey) -> bool {
860+ let url = self.cache_url(&absent_path(key));
861+ let found = matches!(worker::Cache::default().get(url, false).await, Ok(Some(_)));
862+ if found {
863+ meters::record("cache.absent_hit", &self.key, 0, 0);
864+ }
865+ found
866+ }
867+
845868 /// Keeps an object for next time. A failure only costs a later read.
846869 async fn keep(&self, kind: &str, hash: &str, bytes: Vec<u8>) {
847870 self.keep_at(&format!("{kind}/{hash}"), bytes, OBJECT_MAX_AGE).await;
10931116
10941117 async fn read_file(&self, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>> {
10951118 let key = file_key(git_ref, path, self.refs_version);
1096− if let Some(key) = &key
1097− && let Some(bytes) = self.get_key(key).await
1098− {
1099− return Ok(Some(bytes));
1119+ // A path that is not a file is remembered too, briefly: the store
1120+ // answers each such read as a rejected read (crawlers asking for
1121+ // old or missing paths make most of them). Never for the fallback
1122+ // store, which can be behind.
1123+ let remember_absent = !self.binding.is_fallback();
1124+ if let Some(key) = &key {
1125+ let (found, absent) = futures_util::future::join(self.get_key(key), async {
1126+ remember_absent && self.known_absent(key).await
1127+ })
1128+ .await;
1129+ if let Some(bytes) = found {
1130+ return Ok(Some(bytes));
1131+ }
1132+ if absent {
1133+ return Ok(None);
1134+ }
11001135 }
11011136 let args = js::to_js(&serde_json::json!({ "ref": git_ref, "path": path }))?;
11021137 let bytes = blob_bytes(self.call("readFile", &[args], true).await?).await?;
11031138 if let Some(bytes) = &bytes {
11041139 meters::record_bytes("binding.read_file", &self.key, 0, bytes.len() as u64);
1140+ } else if remember_absent && let Some(key) = &key {
1141+ self.keep_at(&absent_path(key), vec![1], ABSENT_MAX_AGE).await;
11051142 }
11061143 if let (Some(key), Some(bytes)) = (&key, bytes.as_ref().filter(|bytes| bytes.len() <= MAX_CACHED_BLOB)) {
11071144 self.put_key(key, bytes.clone()).await;
13131350 assert_eq!(file_key("main", "src/main.rs", None), None);
13141351 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "b", Some(1)));
13151352 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "a", Some(2)));
1353+ // A path noted as not a file sits beside the file's own key, and a
1354+ // push (a new refs version) leaves the old note behind.
1355+ let at = |version| absent_path(&file_key("main", "a", Some(version)).unwrap());
1356+ assert!(at(1).starts_with("absent/vfile/1/"));
1357+ assert_ne!(at(1), at(2));
1358+ assert_eq!(absent_path(&file_key(&hash, "a", None).unwrap()), format!("absent/file/{hash}/{}", g1t_secrets::sha256_hex("a")));
13161359 }
13171360
13181361 #[test]