Skip to content
Sign up

flagon-io/g1t

public
0
1,626 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01//! The storage that actually holds git repositories.
2//!
3//! The service depends on the [`GitStore`] and [`GitRepo`] ports;
4//! [`ArtifactsStore`] is the adapter for Cloudflare Artifacts.
5//!
6//! Every call on the binding goes through [`invoke`]: it is counted
7//! (meters.rs), timed for the store's health, refused at once while its
8//! namespace's breaker is open, and tried again after a failure that may
9//! pass when it only reads (resilience.rs). Repositories may live in
10//! several namespaces (shards.rs).
11
12use g1t_contracts::repos::{Branch, Commit, EntryKind, GitAccess, Signature, TreeEntry};
13use g1t_contracts::time::rfc3339;
14use g1t_kit::js::{self, Thrown};
15use serde::{Deserialize, Serialize};
16use std::cell::RefCell;
17use std::collections::HashMap;
18use std::rc::Rc;
19use worker::js_sys::{Reflect, Uint8Array};
20use worker::wasm_bindgen::{JsCast, JsValue};
21use worker::{Env, Result};
22
Merge branch 'worktree-agent-a2013627e5ea4ab13'23use crate::fallback;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.024use crate::meters::{self, Outcome};
25use crate::resilience::{self, Admit, Busy, Failure};
26use crate::shards;
27
28/// Credentials that never leave this service: g1t has already decided who
29/// may do what before one is used. They live an hour and are used for 50
30/// minutes, so each one used has at least ten minutes left.
31const INTERNAL_TTL_SECONDS: u32 = 3_600;
32const INTERNAL_REUSE_MS: u64 = 50 * 60 * 1000;
Merge branch 'worktree-agent-a57ff9fecefa1eaf7'33/// Credentials handed out: to a nightly backup's sandbox (backups.rs), and
34/// by `git_access`, which nothing deployed asks yet (git over SSH will).
35/// Other sandboxes never get one: they use g1t's git endpoints. Five
36/// minutes, used for three, so whoever gets one has at least two.
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.037const HANDOUT_TTL_SECONDS: u32 = 300;
38const HANDOUT_REUSE_MS: u64 = 180_000;
39/// How long a handed-out credential stays valid, in milliseconds.
40pub const CREDENTIAL_LIFE_MS: u64 = HANDOUT_TTL_SECONDS as u64 * 1000;
41
42#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
43pub enum Scope {
44 Read,
45 Write,
46}
47
48impl Scope {
49 fn as_str(self) -> &'static str {
50 match self {
51 Scope::Read => "read",
52 Scope::Write => "write",
53 }
54 }
55}
56
57/// Who a credential is for.
58#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
59pub enum Use {
60 /// This service, talking to the store itself.
61 Internal,
62 /// Someone outside it, through `git_access`.
63 Handout,
64}
65
66impl Use {
67 pub fn ttl_seconds(self) -> u32 {
68 match self {
69 Use::Internal => INTERNAL_TTL_SECONDS,
70 Use::Handout => HANDOUT_TTL_SECONDS,
71 }
72 }
73
74 pub fn reuse_ms(self) -> u64 {
75 match self {
76 Use::Internal => INTERNAL_REUSE_MS,
77 Use::Handout => HANDOUT_REUSE_MS,
78 }
79 }
80
81 fn as_str(self) -> &'static str {
82 match self {
83 Use::Internal => "internal",
84 Use::Handout => "handout",
85 }
86 }
87}
88
89/// Where a credential handed out came from, for `Server-Timing`.
90#[derive(Clone, Copy, PartialEq, Eq, Debug)]
91pub enum Kept {
92 /// This isolate made it, or had it from another, a moment ago.
93 Isolate,
94 /// Another isolate made it and shared it.
95 Shared,
96}
97
98impl Kept {
99 pub fn as_str(self) -> &'static str {
100 match self {
101 Kept::Isolate => "isolate",
102 Kept::Shared => "shared",
103 }
104 }
105}
106
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer107/// Work a request starts that need not hold up its answer, such as keeping
108/// an object in the Cache API: begun at once, and handed to the request's
109/// `waitUntil` once it has answered ([`Deferred::hand_over`]), so it is
110/// neither awaited on the way nor cut short after. Each request has its own.
111#[derive(Default)]
112pub struct Deferred {
113 started: RefCell<Vec<worker::js_sys::Promise>>,
114}
115
116impl Deferred {
117 /// Starts `work` now.
118 pub fn spawn(&self, work: impl std::future::Future<Output = ()> + 'static) {
119 let promise = worker::wasm_bindgen_futures::future_to_promise(async move {
120 work.await;
121 Ok(JsValue::UNDEFINED)
122 });
123 self.started.borrow_mut().push(promise);
124 }
125
126 /// Hands what was started to `ctx`, to finish after the answer.
127 pub fn hand_over(&self, ctx: &worker::Context) {
128 let started = std::mem::take(&mut *self.started.borrow_mut());
129 if started.is_empty() {
130 return;
131 }
132 ctx.wait_until(async move {
133 futures_util::future::join_all(started.into_iter().map(worker::wasm_bindgen_futures::JsFuture::from)).await;
134 });
135 }
136
137 /// Waits for what was started: for work already running after an
138 /// answer, in a `waitUntil` of its own.
139 pub async fn settle(&self) {
140 let started = std::mem::take(&mut *self.started.borrow_mut());
141 futures_util::future::join_all(started.into_iter().map(worker::wasm_bindgen_futures::JsFuture::from)).await;
142 }
143}
144
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0145/// A place repositories live. `key` is the store's own name for a repo.
146#[allow(async_fn_in_trait)]
147pub trait GitStore {
148 type Repo: GitRepo;
149
150 /// Creates an empty repository. Succeeds if it already exists.
151 async fn create(&self, key: &str, description: Option<&str>, default_branch: &str) -> Result<()>;
152 async fn open(&self, key: &str) -> Result<Self::Repo>;
153 /// A credential for `key` made a moment ago, if the store keeps one.
154 async fn kept_access(&self, _key: &str, _scope: Scope) -> Option<(GitAccess, Kept)> {
155 None
156 }
157 /// A new credential for `key`, which the store may keep for next time.
158 async fn mint_access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
159 self.open(key).await?.access(scope).await
160 }
161 /// A remote URL and credential for this service's own use. A store may
162 /// hand out one it made a moment ago.
163 async fn access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
164 match self.kept_access(key, scope).await {
165 Some((access, _)) => Ok(access),
166 None => self.mint_access(key, scope).await,
167 }
168 }
169 /// A short-lived credential for someone outside this service.
170 async fn handout(&self, key: &str, scope: Scope) -> Result<GitAccess> {
171 self.access(key, scope).await
172 }
173 /// Stops handing out the credentials it keeps for `key`: the store
174 /// turned one down, or the repository is gone.
175 async fn forget_access(&self, _key: &str) {}
176 /// Removes a repository and everything in it, for good. Succeeds if it
177 /// is already gone.
178 async fn delete(&self, key: &str) -> Result<()>;
179 /// The namespaces new repositories may be placed in (shards.rs).
180 fn namespaces(&self) -> Vec<String> {
181 vec![shards::DEFAULT_NAMESPACE.to_owned()]
182 }
183 /// The namespace bound as the default.
184 fn default_namespace(&self) -> String {
185 shards::DEFAULT_NAMESPACE.to_owned()
186 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'187 /// Whether the repository at `key` is served from the fallback store
188 /// now (fallback.rs): answers kept from the usual store may name refs
189 /// it does not have, so none are used.
190 fn on_fallback(&self, _key: &str) -> bool {
191 false
192 }
193 /// Whether `namespace` takes writes now: not while it is served from a
194 /// read-only fallback.
195 fn writable(&self, _namespace: &str) -> bool {
196 true
197 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0198}
199
200/// One open repository.
201#[allow(async_fn_in_trait)]
202pub trait GitRepo {
203 /// A remote URL and credential for git itself, for this service.
204 async fn access(&self, scope: Scope) -> Result<GitAccess>;
205 /// Every branch and the commit it points to.
206 async fn branches(&self) -> Result<Vec<Branch>>;
207 /// Newest first along the first-parent chain; empty for an unknown ref.
208 async fn log(&self, git_ref: &str, limit: u32) -> Result<Vec<Commit>>;
209 /// The parents of a commit, or `None` if the commit does not exist.
210 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>>;
211 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>>;
212 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>>;
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb97213 /// A blob's size in bytes, `None` when it is missing. By default its
214 /// bytes are read; a store that can say less does.
215 async fn blob_size(&self, blob_hash: &str) -> Result<Option<u64>> {
216 Ok(self.read_blob(blob_hash).await?.map(|bytes| bytes.len() as u64))
217 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0218 /// `None` when the ref or path does not resolve to a file.
219 async fn read_file(&self, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>>;
220 /// Makes a copy of this repository under `target_key`, in the same
221 /// namespace.
222 async fn fork(&self, target_key: &str) -> Result<()>;
223 /// The version of the repository's refs (registry.rs `RefsState`),
224 /// when answers about branches may be kept under it (R9). Reads by
225 /// branch name are then kept until the version moves.
226 fn at_refs_version(&mut self, _version: Option<u64>) {}
227}
228
229thread_local! {
The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook.230 /// The namespace bound to `GITSTORE`, for keys that name none.
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0231 static DEFAULT_NS: RefCell<String> = RefCell::new(shards::DEFAULT_NAMESPACE.to_owned());
232 /// Where each namespace's remotes start: `https://<account>.artifacts.cloudflare.net/git/<namespace>/`.
233 static REMOTE_PREFIX: RefCell<HashMap<String, String>> = RefCell::new(HashMap::new());
234}
235
236/// The namespace and name a store key stands for.
237pub fn locate(key: &str) -> (String, String) {
238 let (namespace, name) = shards::split(key);
239 let namespace = namespace.map_or_else(|| DEFAULT_NS.with(|ns| ns.borrow().clone()), str::to_owned);
240 (namespace, name.to_owned())
241}
242
Merge branch 'worktree-agent-a2013627e5ea4ab13'243thread_local! {
244 /// Where the fallback store's remotes start, when one is configured.
245 static FALLBACK_BASE: RefCell<Option<String>> = const { RefCell::new(None) };
246}
247
248/// Whose breaker and health git requests to `remote` count toward: its
249/// namespace's, or `<namespace>@fallback` for the fallback store's.
250pub fn health_namespace(remote: &str) -> String {
251 let (namespace, _) = locate(&key_from_remote(remote).unwrap_or_default());
252 let on_fallback = FALLBACK_BASE.with(|base| base.borrow().as_deref().is_some_and(|base| remote.starts_with(base)));
253 if on_fallback { format!("{namespace}@fallback") } else { namespace }
254}
255
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0256/// The store key a git remote is for, from its last two path segments.
257pub fn key_from_remote(remote: &str) -> Option<String> {
258 let path = remote.trim_end_matches('/');
259 let path = path.strip_suffix(".git").unwrap_or(path);
260 let (rest, name) = path.rsplit_once('/')?;
261 let namespace = rest.rsplit('/').next()?;
262 let default = DEFAULT_NS.with(|ns| ns.borrow().clone());
263 Some(shards::compose(Some(namespace), name, &default))
264}
265
266/// Where a namespace's remotes start, learned from one remote the store
267/// gave for `name`.
268fn learn_prefix(remote: &str, name: &str) -> Option<String> {
269 remote.strip_suffix(&format!("{name}.git")).filter(|prefix| prefix.ends_with('/')).map(str::to_owned)
270}
271
272/// A repository's remote, from where its namespace's remotes start.
273fn remote_from(prefix: &str, name: &str) -> String {
274 format!("{prefix}{name}.git")
275}
276
277struct Namespace {
278 name: String,
Merge branch 'worktree-agent-a2013627e5ea4ab13'279 /// Its binding, for every call: Artifacts, or the fallback store.
280 target: Target,
281}
282
283/// What a call on the store goes to: an Artifacts binding or one of its
284/// repository handles, or the fallback store (fallback.rs) for a
285/// namespace, or for one repository in it.
286#[derive(Clone)]
287enum Target {
288 Js(JsValue),
289 Fallback {
290 settings: Rc<fallback::Settings>,
291 namespace: String,
292 repo: Option<String>,
293 },
294}
295
296impl Target {
297 fn is_fallback(&self) -> bool {
298 matches!(self, Target::Fallback { .. })
299 }
300
301 /// Whose breaker and health a call counts toward: the fallback store's
302 /// own, so an Artifacts outage never holds it back.
303 fn health_name(&self, namespace: &str) -> String {
304 if self.is_fallback() { format!("{namespace}@fallback") } else { namespace.to_owned() }
305 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0306}
307
308pub struct ArtifactsStore {
309 namespaces: Rc<Vec<Namespace>>,
310 /// Where isolates share the credentials they make; see shared.rs.
311 shared: Option<Rc<crate::shared::Shared>>,
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer312 /// The request's work that need not hold up its answer: objects kept
313 /// for next time.
314 deferred: Rc<Deferred>,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0315}
316
317impl ArtifactsStore {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer318 pub fn new(env: &Env, shared: Option<Rc<crate::shared::Shared>>, deferred: Rc<Deferred>) -> Result<Self> {
The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook.319 let config = env.var("GITSTORE_NAMESPACES").ok().map(|value| value.to_string());
Merge branch 'worktree-agent-a2013627e5ea4ab13'320 let text = |name: &str| env.var(name).ok().map(|value| value.to_string());
321 let secret = env.secret("GIT_FALLBACK_SECRET").ok().map(|value| value.to_string());
322 let fallback = fallback::Settings::from_vars(
323 text("GIT_FALLBACK_URL").as_deref(),
324 secret.as_deref(),
325 text("GIT_FALLBACK_NAMESPACES").as_deref(),
326 text("GIT_FALLBACK_WRITES").as_deref(),
327 )
328 .map(Rc::new);
329 FALLBACK_BASE.with(|base| *base.borrow_mut() = fallback.as_ref().map(|settings| format!("{}/git/", settings.url)));
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0330 let mut namespaces = Vec::new();
331 for (binding, name) in shards::bindings(config.as_deref()) {
Merge branch 'worktree-agent-a2013627e5ea4ab13'332 // Served from the fallback store, by configuration.
333 if let Some(settings) = fallback.as_ref().filter(|settings| settings.serves(&name)) {
334 worker::console_log!("git store {name}: served from the fallback store");
335 let target = Target::Fallback { settings: settings.clone(), namespace: name.clone(), repo: None };
336 namespaces.push(Namespace { name, target });
337 continue;
338 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0339 match js::binding(env, &binding) {
Merge branch 'worktree-agent-a2013627e5ea4ab13'340 Ok(value) => namespaces.push(Namespace { name, target: Target::Js(value) }),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0341 // The default binding is required; the others are optional.
342 Err(error) if binding == shards::DEFAULT_BINDING => return Err(error),
The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook.343 Err(_) => worker::console_error!("GITSTORE_NAMESPACES names {binding}, which is not bound"),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0344 }
345 }
346 if let Some(default) = namespaces.first() {
347 DEFAULT_NS.with(|ns| ns.borrow_mut().clone_from(&default.name));
348 }
349 // Optional: where remotes start, `https://<account>.artifacts.cloudflare.net/git`,
350 // so the first credential an isolate makes needs no `info()` either.
The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook.351 if let Ok(base) = env.var("GITSTORE_REMOTE_BASE") {
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0352 let base = base.to_string().trim_end_matches('/').to_owned();
353 if base.starts_with("https://") {
354 REMOTE_PREFIX.with(|prefixes| {
355 let mut prefixes = prefixes.borrow_mut();
356 for namespace in &namespaces {
357 prefixes.entry(namespace.name.clone()).or_insert_with(|| format!("{base}/{}/", namespace.name));
358 }
359 });
360 }
361 }
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer362 Ok(Self { namespaces: Rc::new(namespaces), shared, deferred })
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0363 }
364
Merge branch 'worktree-agent-a2013627e5ea4ab13'365 fn binding(&self, namespace: &str) -> Result<&Target> {
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0366 self.namespaces
367 .iter()
368 .find(|candidate| candidate.name == namespace)
Merge branch 'worktree-agent-a2013627e5ea4ab13'369 .map(|found| &found.target)
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0370 .ok_or_else(|| worker::Error::RustError(format!("git store namespace {namespace} is not bound")))
371 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'372
373 /// The fallback store's settings, when `namespace` is served from it.
374 fn fallback_of(&self, namespace: &str) -> Option<&fallback::Settings> {
375 match self.binding(namespace).ok()? {
376 Target::Fallback { settings, .. } => Some(settings),
377 Target::Js(_) => None,
378 }
379 }
380
381 /// The name credentials for `key` are kept under: those of the
382 /// fallback store never stand in for Artifacts' own, nor the reverse.
383 fn cred_key(&self, key: &str) -> String {
384 let (namespace, _) = locate(key);
385 cred_key(key, self.fallback_of(&namespace).is_some())
386 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0387}
388
Merge branch 'worktree-agent-a2013627e5ea4ab13'389/// See [`ArtifactsStore::cred_key`].
390fn cred_key(key: &str, on_fallback: bool) -> String {
391 if on_fallback { format!("fallback:{key}") } else { key.to_owned() }
392}
393
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0394/// A failed call on the binding.
395pub struct StoreError {
396 pub thrown: Thrown,
397 pub busy: Option<Busy>,
398}
399
400impl StoreError {
401 pub fn is(&self, code: &str) -> bool {
402 self.thrown.is(code)
403 }
404}
405
406impl From<StoreError> for worker::Error {
407 fn from(error: StoreError) -> Self {
408 match error.busy {
409 Some(busy) => busy.error(&error.thrown.to_string()),
410 None => error.thrown.into(),
411 }
412 }
413}
414
415/// The meter for a binding method: `binding.create_token`.
416fn meter_of(method: &str) -> String {
417 let mut out = String::from("binding.");
418 for c in method.chars() {
419 if c.is_ascii_uppercase() {
420 out.push('_');
421 out.push(c.to_ascii_lowercase());
422 } else {
423 out.push(c);
424 }
425 }
426 out
427}
428
429/// Seconds a caller is told to wait when the store is busy.
430const BUSY_RETRY_AFTER: u64 = 5;
431
432/// Calls `target[method](...args)` on namespace `namespace`'s binding, for
433/// the repository at `key`. `retry`: whether a failure that may pass is
434/// tried again (reads and credentials only).
435async fn invoke(
436 namespace: &str,
437 key: &str,
Merge branch 'worktree-agent-a2013627e5ea4ab13'438 target: &Target,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0439 method: &str,
440 args: &[JsValue],
441 retry: bool,
442) -> std::result::Result<JsValue, StoreError> {
443 let meter = meter_of(method);
Merge branch 'worktree-agent-a2013627e5ea4ab13'444 let health = target.health_name(namespace);
445 let namespace = health.as_str();
446 // A read-only fallback refuses writes before asking (fallback.rs).
447 if let Target::Fallback { settings, repo, .. } = target
448 && !settings.writes
449 {
450 let values: Vec<serde_json::Value> = args.iter().map(|arg| js::from_js(arg).unwrap_or(serde_json::Value::Null)).collect();
451 if fallback::writes(repo.as_deref(), method, &values) {
452 return Err(StoreError {
453 thrown: Thrown { code: Some("READ_ONLY".to_owned()), message: format!("{method} refused: the git store is read-only") },
454 busy: Some(Busy::read_only()),
455 });
456 }
457 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0458 let mut attempt = 0;
459 loop {
460 let now = g1t_kit::now_ms();
461 let admit = resilience::with_breaker(namespace, |breaker| breaker.admit(now));
462 if let Admit::Wait(ms) = admit {
463 meters::record_health(namespace, Outcome::Rejected, 0);
464 return Err(StoreError {
465 thrown: Thrown { code: None, message: format!("{method} not asked: the git store has been failing") },
Merge branch 'worktree-agent-a2013627e5ea4ab13'466 busy: Some(Busy { rate_limited: false, retry_after: resilience::seconds(ms).max(BUSY_RETRY_AFTER), read_only: false }),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0467 });
468 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'469 // Calls on Artifacts are metered; the fallback store costs nothing
470 // per call.
471 if !target.is_fallback() {
472 meters::record(&meter, key, 0, 0);
473 }
474 let called = dispatch(target, method, args).await;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0475 let ms = g1t_kit::now_ms().saturating_sub(now);
476 match called {
477 Ok(value) => {
478 resilience::with_breaker(namespace, |breaker| breaker.succeeded());
479 meters::record_health(namespace, Outcome::Ok, ms);
480 return Ok(value);
481 }
482 Err(thrown) => {
483 let failure = resilience::classify(thrown.code.as_deref(), &thrown.message);
484 if failure == Failure::Permanent {
485 // An answer (NOT_FOUND, ALREADY_EXISTS): the store is up.
486 resilience::with_breaker(namespace, |breaker| breaker.succeeded());
487 meters::record_health(namespace, Outcome::Ok, ms);
488 return Err(StoreError { thrown, busy: None });
489 }
490 if retry && resilience::retry(failure, attempt) {
491 let wait = resilience::backoff_ms(failure, attempt, worker::js_sys::Math::random());
492 worker::Delay::from(std::time::Duration::from_millis(wait)).await;
493 attempt += 1;
494 continue;
495 }
496 resilience::with_breaker(namespace, |breaker| breaker.failed(failure, g1t_kit::now_ms()));
497 let outcome = if failure == Failure::RateLimited { Outcome::RateLimited } else { Outcome::Failed };
498 meters::record_health(namespace, outcome, ms);
499 worker::console_error!("git store {namespace}: {method} for {key} failed: {thrown}");
500 return Err(StoreError {
501 thrown,
Merge branch 'worktree-agent-a2013627e5ea4ab13'502 busy: Some(Busy { rate_limited: failure == Failure::RateLimited, retry_after: BUSY_RETRY_AFTER, read_only: false }),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0503 });
504 }
505 }
506 }
507}
508
Merge branch 'worktree-agent-a2013627e5ea4ab13'509/// Makes one call: on the binding, or as a request to the fallback store.
510async fn dispatch(target: &Target, method: &str, args: &[JsValue]) -> std::result::Result<JsValue, Thrown> {
511 let (settings, namespace, repo) = match target {
512 Target::Js(value) => return js::call(value, method, args).await,
513 Target::Fallback { settings, namespace, repo } => (settings, namespace, repo),
514 };
515 let values: Vec<serde_json::Value> = args.iter().map(|arg| js::from_js(arg).unwrap_or(serde_json::Value::Null)).collect();
516 let route = fallback::route(namespace, repo.as_deref(), method, &values)
517 .map_err(|refused| Thrown { code: Some(refused.code.to_owned()), message: refused.message })?;
518 let unreachable = |error: worker::Error| Thrown { code: None, message: format!("the fallback store could not be reached: {error}") };
519 let headers = worker::Headers::new();
520 headers.set("x-gitstore-secret", &settings.secret).map_err(unreachable)?;
521 let mut init = worker::RequestInit::new();
522 init.with_method(match route.method {
523 "POST" => worker::Method::Post,
524 "DELETE" => worker::Method::Delete,
525 _ => worker::Method::Get,
526 });
527 if let Some(body) = &route.body {
528 headers.set("content-type", "application/json").map_err(unreachable)?;
529 init.with_body(Some(JsValue::from_str(&body.to_string())));
530 }
531 init.with_headers(headers);
532 let request = worker::Request::new_with_init(&format!("{}{}", settings.url, route.path), &init).map_err(unreachable)?;
533 let mut response = worker::Fetch::Request(request).send().await.map_err(unreachable)?;
534 let status = response.status_code();
535 let body = response.bytes().await.map_err(unreachable)?;
536 match fallback::answer(&route, status, body) {
537 fallback::Answer::Json(value) => js::to_js(&value).map_err(|error| Thrown { code: None, message: error.to_string() }),
538 fallback::Answer::Bytes(bytes) => Ok(Uint8Array::from(bytes.as_slice()).into()),
539 fallback::Answer::Null => Ok(JsValue::NULL),
540 fallback::Answer::Error { code, message } => Err(Thrown { code, message }),
541 }
542}
543
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0544/// A credential as isolates share it, sealed (see shared.rs): with when it
545/// was made, so that one shared is reused no longer than one kept here.
546#[derive(Serialize, Deserialize)]
547struct SharedCredential {
548 remote: String,
549 token: String,
550 made: u64,
551}
552
553/// The shared cache's key for a credential: the store's key for the
554/// repository, the scope, and who it is for. (`cred2`: credentials kept
555/// before their lives differed by use are not read.)
556fn shared_key(key: &str, scope: Scope, using: Use) -> String {
557 format!("cred2:{key}:{}:{}", scope.as_str(), using.as_str())
558}
559
560/// A shared credential, if it was made less than its reuse window before
561/// `now`; with when it was made.
562fn shared_credential(bytes: &[u8], now: u64, using: Use) -> Option<(GitAccess, u64)> {
563 let kept: SharedCredential = serde_json::from_slice(bytes).ok()?;
564 (now.saturating_sub(kept.made) < using.reuse_ms()).then_some((
565 GitAccess {
566 remote: kept.remote,
567 token: kept.token,
568 },
569 kept.made,
570 ))
571}
572
573/// Credentials made a while ago, by repository, scope and use. Making one
574/// is a round trip to the store; reusing it saves that, and the store's
575/// lookup of the repository with it.
576#[derive(Default)]
577pub struct Credentials {
578 kept: HashMap<(String, Scope, Use), (GitAccess, u64)>,
579}
580
581impl Credentials {
582 /// One made for `key`, `scope` and `using` within its reuse window of `now`.
583 pub fn get(&self, key: &str, scope: Scope, using: Use, now: u64) -> Option<GitAccess> {
584 self.kept
585 .get(&(key.to_owned(), scope, using))
586 .filter(|(_, made)| now.saturating_sub(*made) < using.reuse_ms())
587 .map(|(access, _)| access.clone())
588 }
589
590 pub fn keep(&mut self, key: &str, scope: Scope, using: Use, access: GitAccess, made: u64, now: u64) {
591 // Expired ones go first, so the map stays as small as the isolate's
592 // recent repositories.
593 self.kept
594 .retain(|(_, _, kept_use), (_, at)| now.saturating_sub(*at) < kept_use.reuse_ms());
595 self.kept.insert((key.to_owned(), scope, using), (access, made));
596 }
597
598 pub fn forget(&mut self, key: &str) {
599 self.kept.retain(|(kept, _, _), _| kept != key);
600 }
601}
602
603thread_local! {
604 static CREDENTIALS: RefCell<Credentials> = RefCell::new(Credentials::default());
605}
606
607/// A credential kept in this isolate, else one another isolate shared.
608async fn kept(shared: Option<&crate::shared::Shared>, key: &str, scope: Scope, using: Use) -> Option<(GitAccess, Kept)> {
609 let now = g1t_kit::now_ms();
610 if let Some(access) = CREDENTIALS.with(|kept| kept.borrow().get(key, scope, using, now)) {
611 return Some((access, Kept::Isolate));
612 }
613 let bytes = shared?.get(&shared_key(key, scope, using)).await?;
614 let (access, made) = shared_credential(&bytes, now, using)?;
615 CREDENTIALS.with(|kept| kept.borrow_mut().keep(key, scope, using, access.clone(), made, now));
616 Some((access, Kept::Shared))
617}
618
619/// Keeps a credential just made here, and shares it.
620async fn keep(shared: Option<&crate::shared::Shared>, key: &str, scope: Scope, using: Use, access: &GitAccess) {
621 let now = g1t_kit::now_ms();
622 CREDENTIALS.with(|kept| kept.borrow_mut().keep(key, scope, using, access.clone(), now, now));
623 if let Some(shared) = shared {
624 let value = SharedCredential {
625 remote: access.remote.clone(),
626 token: access.token.clone(),
627 made: now,
628 };
629 if let Ok(bytes) = serde_json::to_vec(&value) {
630 shared.put(&shared_key(key, scope, using), &bytes, using.reuse_ms() / 1000).await;
631 }
632 }
633}
634
635impl GitStore for ArtifactsStore {
636 type Repo = ArtifactsRepo;
637
638 async fn kept_access(&self, key: &str, scope: Scope) -> Option<(GitAccess, Kept)> {
Merge branch 'worktree-agent-a2013627e5ea4ab13'639 kept(self.shared.as_deref(), &self.cred_key(key), scope, Use::Internal).await
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0640 }
641
642 async fn mint_access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
643 let repo = self.open(key).await?;
644 let access = repo.mint(scope, Use::Internal).await?;
Merge branch 'worktree-agent-a2013627e5ea4ab13'645 keep(self.shared.as_deref(), &self.cred_key(key), scope, Use::Internal, &access).await;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0646 Ok(access)
647 }
648
649 async fn handout(&self, key: &str, scope: Scope) -> Result<GitAccess> {
Merge branch 'worktree-agent-a2013627e5ea4ab13'650 let cred_key = self.cred_key(key);
651 if let Some((access, _)) = kept(self.shared.as_deref(), &cred_key, scope, Use::Handout).await {
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0652 return Ok(access);
653 }
654 let access = self.open(key).await?.mint(scope, Use::Handout).await?;
Merge branch 'worktree-agent-a2013627e5ea4ab13'655 keep(self.shared.as_deref(), &cred_key, scope, Use::Handout, &access).await;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0656 Ok(access)
657 }
658
659 async fn forget_access(&self, key: &str) {
Merge branch 'worktree-agent-a2013627e5ea4ab13'660 // Both stores' credentials: whichever serves the key now.
661 let names = [cred_key(key, false), cred_key(key, true)];
662 CREDENTIALS.with(|kept| {
663 let mut kept = kept.borrow_mut();
664 for name in &names {
665 kept.forget(name);
666 }
667 });
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0668 if let Some(shared) = &self.shared {
Merge branch 'worktree-agent-a2013627e5ea4ab13'669 let keys: Vec<String> = names
670 .iter()
671 .flat_map(|name| {
672 [Scope::Read, Scope::Write]
673 .into_iter()
674 .flat_map(move |scope| [Use::Internal, Use::Handout].map(|using| shared_key(name, scope, using)))
675 })
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0676 .collect();
677 futures_util::future::join_all(keys.iter().map(|shared_key| shared.delete(shared_key))).await;
678 }
679 }
680
681 async fn create(&self, key: &str, description: Option<&str>, default_branch: &str) -> Result<()> {
682 let (namespace, name) = locate(key);
683 let options = js::to_js(&serde_json::json!({
684 "description": description,
685 "setDefaultBranch": default_branch,
686 }))?;
687 match invoke(&namespace, key, self.binding(&namespace)?, "create", &[name.as_str().into(), options], true).await {
688 // Left behind by an earlier failed attempt; adopt it.
689 Err(failed) if !failed.is("ALREADY_EXISTS") => Err(failed.into()),
690 _ => Ok(()),
691 }
692 }
693
694 async fn delete(&self, key: &str) -> Result<()> {
695 let (namespace, name) = locate(key);
Merge branch 'worktree-agent-a2013627e5ea4ab13'696 // Nothing is forgotten while the store refuses to delete.
697 if self.writable(&namespace) {
698 self.forget_access(key).await;
699 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0700 match invoke(&namespace, key, self.binding(&namespace)?, "delete", &[name.as_str().into()], true).await {
701 // Gone already: an earlier purge got this far.
702 Err(failed) if !failed.is("NOT_FOUND") => Err(failed.into()),
703 _ => Ok(()),
704 }
705 }
706
707 async fn open(&self, key: &str) -> Result<ArtifactsRepo> {
708 let (namespace, name) = locate(key);
Merge branch 'worktree-agent-a2013627e5ea4ab13'709 let binding = self.binding(&namespace)?.clone();
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0710 Ok(ArtifactsRepo {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for711 handle: RefCell::new(None),
Merge branch 'worktree-agent-a2013627e5ea4ab13'712 cred_key: cred_key(key, binding.is_fallback()),
713 binding,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0714 key: key.to_owned(),
715 name,
716 namespace,
717 shared: self.shared.clone(),
718 refs_version: None,
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer719 deferred: self.deferred.clone(),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0720 })
721 }
722
723 fn namespaces(&self) -> Vec<String> {
724 self.namespaces.iter().map(|namespace| namespace.name.clone()).collect()
725 }
726
727 fn default_namespace(&self) -> String {
728 DEFAULT_NS.with(|ns| ns.borrow().clone())
729 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'730
731 fn on_fallback(&self, key: &str) -> bool {
732 let (namespace, _) = locate(key);
733 self.fallback_of(&namespace).is_some()
734 }
735
736 fn writable(&self, namespace: &str) -> bool {
737 self.fallback_of(namespace).is_none_or(|settings| settings.writes)
738 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0739}
740
Merge branch 'worktree-agent-a2013627e5ea4ab13'741/// A handle to one repository in the store. On Artifacts it is an RPC
742/// stub, so it is released when dropped.
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0743pub struct ArtifactsRepo {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for744 /// The store's handle, asked for (`get`) on the first call that needs
745 /// the store: an answer from a cache, or a fetch over git with a kept
746 /// credential, never costs a `get`.
Merge branch 'worktree-agent-a2013627e5ea4ab13'747 handle: RefCell<Option<Target>>,
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for748 /// The namespace's binding, for that `get`.
Merge branch 'worktree-agent-a2013627e5ea4ab13'749 binding: Target,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0750 /// The repository's store key, which scopes its cached objects.
751 key: String,
Merge branch 'worktree-agent-a2013627e5ea4ab13'752 /// What its credentials are kept under (`ArtifactsStore::cred_key`).
753 cred_key: String,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0754 /// Its name in its namespace.
755 name: String,
756 namespace: String,
757 shared: Option<Rc<crate::shared::Shared>>,
758 /// See [`GitRepo::at_refs_version`].
759 refs_version: Option<u64>,
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer760 deferred: Rc<Deferred>,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0761}
762
763/// Where cached git objects live. Trees and blobs are named by their
764/// content, so a cached one is never stale; each is kept under its own
765/// repository's key, so a repository only ever finds its own objects.
766const OBJECT_CACHE: &str = "https://objects.g1t.internal/";
767/// Blobs and files larger than this are not cached.
768const MAX_CACHED_BLOB: usize = 1024 * 1024;
769const OBJECT_MAX_AGE: &str = "public, max-age=31536000, immutable";
770/// Answers kept under a refs version: until the version moves, and no
771/// longer than this, which bounds how stale one can be should a change
772/// ever fail to move it.
773const VERSIONED_MAX_AGE: &str = "public, max-age=300";
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing774/// How long a path found not to be a file at a ref is remembered. Short:
775/// a miss by commit hash is true for good, but one for a commit not yet in
776/// the store would not be.
777const ABSENT_MAX_AGE: &str = "public, max-age=600";
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0778
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 25779/// Histories by hash at least this long are put together from a short
780/// read and one kept before, where they can be (`spliced_log`).
781const SPLICE_FROM: u32 = 100;
782/// How many commits that short read takes.
783const SPLICE_PROBE: u32 = 16;
784
785/// `short[..at]` followed by `kept` (the history from `short[at]`), cut
786/// to `limit` commits.
787pub fn splice(short: &[Commit], at: usize, kept: Vec<Commit>, limit: u32) -> Vec<Commit> {
788 let mut out: Vec<Commit> = short[..at.min(short.len())].to_vec();
789 out.extend(kept);
790 out.truncate(limit as usize);
791 out
792}
793
Merge branch drift: count across merges the way git does; v2 cache key794/// The history from `short[0]`, from the newest of `kept` (each the history
795/// kept from the commit of `short` at the same index, if any) that really
796/// is the history from that commit: it starts there and goes on to the
797/// next commit `short` lists, so the join repeats and skips nothing.
798pub fn splice_first(short: &[Commit], kept: Vec<Option<Vec<Commit>>>, limit: u32) -> Option<Vec<Commit>> {
799 kept.into_iter().enumerate().skip(1).find_map(|(at, kept)| {
800 let kept = kept?;
801 let starts = kept.first().is_some_and(|first| short.get(at).is_some_and(|commit| commit.hash == first.hash));
802 let goes_on = match (short.get(at + 1), kept.get(1)) {
803 (Some(next), Some(kept_next)) => next.hash == kept_next.hash,
804 // `short` ends at `at`: it reached the first commit, or its limit.
805 (None, _) => true,
806 // The kept history ends where `short` goes on.
807 (Some(_), None) => false,
808 };
809 (starts && goes_on).then(|| splice(short, at, kept, limit))
810 })
811}
812
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0813/// Whether a ref is a full commit hash (SHA-1 or SHA-256), whose history
814/// can be kept for good.
815pub fn is_commit_hash(git_ref: &str) -> bool {
816 (git_ref.len() == 40 || git_ref.len() == 64) && git_ref.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
817}
818
819/// Where a read is kept: under the object it names for good, under the
820/// refs version for a name that can move, or nowhere.
821#[derive(Debug, PartialEq, Eq)]
822pub enum CacheKey {
823 Forever(String),
824 Versioned(String),
825}
826
827/// The cache key for `log(git_ref, limit)`.
828pub fn log_key(git_ref: &str, limit: u32, version: Option<u64>) -> Option<CacheKey> {
829 if is_commit_hash(git_ref) {
830 return Some(CacheKey::Forever(format!("log/{git_ref}-{limit}")));
831 }
832 version.map(|version| CacheKey::Versioned(format!("vlog/{version}/{}-{limit}", g1t_secrets::sha256_hex(git_ref))))
833}
834
835/// The cache key for `read_file(git_ref, path)`.
836pub fn file_key(git_ref: &str, path: &str, version: Option<u64>) -> Option<CacheKey> {
837 let path = g1t_secrets::sha256_hex(path);
838 if is_commit_hash(git_ref) {
839 return Some(CacheKey::Forever(format!("file/{git_ref}/{path}")));
840 }
841 version.map(|version| CacheKey::Versioned(format!("vfile/{version}/{}/{path}", g1t_secrets::sha256_hex(git_ref))))
842}
843
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing844/// Where `read_file` notes that its key is not a file (see `known_absent`).
845fn absent_path(key: &CacheKey) -> String {
846 match key {
847 CacheKey::Forever(path) | CacheKey::Versioned(path) => format!("absent/{path}"),
848 }
849}
850
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0851/// The cache key for the branch list.
852pub fn branches_key(version: Option<u64>) -> Option<CacheKey> {
853 version.map(|version| CacheKey::Versioned(format!("branches/{version}")))
854}
855
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for856/// Objects named by their content, kept in the isolate ahead of the Cache
857/// API, oldest out first past `MEMORY_CACHE_BYTES`. They never go stale,
858/// and each is under its repository's key.
859const MEMORY_CACHE_BYTES: usize = 16 * 1024 * 1024;
860
861#[derive(Default)]
862struct MemoryCache {
863 entries: HashMap<String, Rc<Vec<u8>>>,
864 order: std::collections::VecDeque<String>,
865 bytes: usize,
866}
867
868impl MemoryCache {
869 fn get(&self, url: &str) -> Option<Vec<u8>> {
870 self.entries.get(url).map(|bytes| bytes.as_ref().clone())
871 }
872
873 fn put(&mut self, url: String, bytes: &[u8]) {
874 if bytes.len() > MEMORY_CACHE_BYTES / 16 || self.entries.contains_key(&url) {
875 return;
876 }
877 self.bytes += bytes.len();
878 self.entries.insert(url.clone(), Rc::new(bytes.to_vec()));
879 self.order.push_back(url);
880 while self.bytes > MEMORY_CACHE_BYTES {
881 let Some(oldest) = self.order.pop_front() else { break };
882 if let Some(gone) = self.entries.remove(&oldest) {
883 self.bytes -= gone.len();
884 }
885 }
886 }
887}
888
889thread_local! {
890 static MEMORY: RefCell<MemoryCache> = RefCell::new(MemoryCache::default());
891}
892
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0893impl ArtifactsRepo {
894 fn cache_url(&self, path: &str) -> String {
895 format!("{OBJECT_CACHE}{}/{path}", self.key)
896 }
897
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for898 /// A kept answer: from the isolate for one kept for good, else the
899 /// Cache API. Each look is metered (`cache.memory_hit`, `cache.edge_hit`,
900 /// `cache.miss`), so the usage check shows which cache answers.
901 async fn cached_at(&self, path: &str, forever: bool) -> Option<Vec<u8>> {
902 let url = self.cache_url(path);
903 if forever && let Some(bytes) = MEMORY.with(|memory| memory.borrow().get(&url)) {
Cache hits are counted, not only their bytes904 meters::record("cache.memory_hit", &self.key, 0, bytes.len() as u64);
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for905 return Some(bytes);
906 }
907 let found = match worker::Cache::default().get(url.clone(), false).await {
908 Ok(Some(mut response)) => response.bytes().await.ok(),
909 _ => None,
910 };
911 match &found {
912 Some(bytes) => {
Cache hits are counted, not only their bytes913 meters::record("cache.edge_hit", &self.key, 0, bytes.len() as u64);
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for914 if forever {
915 MEMORY.with(|memory| memory.borrow_mut().put(url, bytes));
916 }
917 }
918 None => meters::record("cache.miss", &self.key, 0, 0),
919 }
920 found
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0921 }
922
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 25923 /// A history from the store itself.
924 async fn read_log(&self, git_ref: &str, limit: u32) -> Result<Vec<Commit>> {
925 let options = js::to_js(&serde_json::json!({ "ref": git_ref, "limit": limit }))?;
926 let raw: Vec<RawCommit> = js::from_js(&self.call("log", &[options], true).await?)?;
927 Ok(raw
928 .into_iter()
929 .map(|commit| Commit {
930 hash: commit.hash,
931 tree_hash: commit.tree_hash,
932 message: commit.message,
933 author: commit.author,
934 parents: commit.parents,
935 authored_at: rfc3339(commit.authored_at * 1000),
936 })
937 .collect())
938 }
939
940 /// A long history by commit hash, from a short read and one kept
941 /// before: when a branch moves a few commits, the history from its new
942 /// head is those commits, then the history kept from its old one (the
943 /// first-parent chain from a commit never changes). Only when none of
944 /// the short read's commits has the same history kept is the whole of
945 /// it read. A default branch that moved by a merge then costs a read
946 /// of [`SPLICE_PROBE`] commits instead of a thousand.
947 async fn spliced_log(&self, hash: &str, limit: u32) -> Result<Vec<Commit>> {
948 let short = self.read_log(hash, SPLICE_PROBE).await?;
949 if (short.len() as u32) < SPLICE_PROBE {
950 // The whole history fits in the short read.
951 return Ok(short);
952 }
Merge branch drift: count across merges the way git does; v2 cache key953 let kept = futures_util::future::join_all(short.iter().enumerate().map(|(at, commit)| async move {
954 if at == 0 {
955 return None;
956 }
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 25957 let Some(CacheKey::Forever(path)) = log_key(&commit.hash, limit, None) else {
958 return None;
959 };
960 let bytes = self.peek(&path).await?;
Merge branch drift: count across merges the way git does; v2 cache key961 serde_json::from_slice::<Vec<Commit>>(&bytes).ok()
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 25962 }))
963 .await;
Merge branch drift: count across merges the way git does; v2 cache key964 match splice_first(&short, kept, limit) {
965 Some(history) => Ok(history),
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 25966 None => self.read_log(hash, limit).await,
967 }
968 }
969
970 /// A kept answer, if there is one, without counting a miss: the
971 /// splice looks for many and expects most to be absent.
972 async fn peek(&self, path: &str) -> Option<Vec<u8>> {
973 let url = self.cache_url(path);
974 if let Some(bytes) = MEMORY.with(|memory| memory.borrow().get(&url)) {
975 meters::record("cache.memory_hit", &self.key, 0, bytes.len() as u64);
976 return Some(bytes);
977 }
978 let mut response = worker::Cache::default().get(url.clone(), false).await.ok()??;
979 let bytes = response.bytes().await.ok()?;
980 meters::record("cache.edge_hit", &self.key, 0, bytes.len() as u64);
981 MEMORY.with(|memory| memory.borrow_mut().put(url, &bytes));
982 Some(bytes)
983 }
984
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0985 async fn cached(&self, kind: &str, hash: &str) -> Option<Vec<u8>> {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for986 self.cached_at(&format!("{kind}/{hash}"), true).await
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.0987 }
988
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer989 /// Keeps an answer at `path`: in the isolate at once, and in the Cache
990 /// API by a write that starts now and finishes in the request's
991 /// `waitUntil`. A read never waits on the write, which only saves a
992 /// later read.
993 fn keep_at(&self, path: &str, bytes: Vec<u8>, max_age: &'static str) {
994 let url = self.cache_url(path);
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for995 if max_age == OBJECT_MAX_AGE {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer996 MEMORY.with(|memory| memory.borrow_mut().put(url.clone(), &bytes));
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for997 }
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer998 self.deferred.spawn(async move {
999 let Ok(mut response) = worker::Response::from_bytes(bytes) else {
1000 return;
1001 };
1002 let _ = response.headers_mut().set("cache-control", max_age);
1003 let _ = worker::Cache::default().put(url, response).await;
1004 });
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01005 }
1006
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing1007 /// Whether `read_file`'s key was found not to be a file a little while
1008 /// ago. Metered only when it was (`cache.absent_hit`): the lookup runs
1009 /// beside the key's own, which already counts the miss.
1010 async fn known_absent(&self, key: &CacheKey) -> bool {
1011 let url = self.cache_url(&absent_path(key));
1012 let found = matches!(worker::Cache::default().get(url, false).await, Ok(Some(_)));
1013 if found {
1014 meters::record("cache.absent_hit", &self.key, 0, 0);
1015 }
1016 found
1017 }
1018
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01019 /// Keeps an object for next time. A failure only costs a later read.
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1020 fn keep(&self, kind: &str, hash: &str, bytes: Vec<u8>) {
1021 self.keep_at(&format!("{kind}/{hash}"), bytes, OBJECT_MAX_AGE);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01022 }
1023
1024 async fn get_key(&self, key: &CacheKey) -> Option<Vec<u8>> {
1025 match key {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1026 CacheKey::Forever(path) => self.cached_at(path, true).await,
1027 CacheKey::Versioned(path) => self.cached_at(path, false).await,
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01028 }
1029 }
1030
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1031 fn put_key(&self, key: &CacheKey, bytes: Vec<u8>) {
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01032 match key {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1033 CacheKey::Forever(path) => self.keep_at(path, bytes, OBJECT_MAX_AGE),
1034 CacheKey::Versioned(path) => self.keep_at(path, bytes, VERSIONED_MAX_AGE),
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01035 }
1036 }
1037
1038 async fn call(&self, method: &str, args: &[JsValue], retry: bool) -> std::result::Result<JsValue, StoreError> {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1039 let handle = self.handle().await?;
1040 invoke(&self.namespace, &self.key, &handle, method, args, retry).await
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01041 }
1042
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1043 /// The store's handle, asked for the first time it is needed.
Merge branch 'worktree-agent-a2013627e5ea4ab13'1044 async fn handle(&self) -> std::result::Result<Target, StoreError> {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1045 if let Some(handle) = self.handle.borrow().as_ref() {
1046 return Ok(handle.clone());
1047 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'1048 let found = invoke(&self.namespace, &self.key, &self.binding, "get", &[self.name.as_str().into()], true).await?;
1049 let handle = match &self.binding {
1050 Target::Js(_) => Target::Js(found),
1051 // The fallback store said the repository is there.
1052 Target::Fallback { settings, namespace, .. } => Target::Fallback {
1053 settings: settings.clone(),
1054 namespace: namespace.clone(),
1055 repo: Some(self.name.clone()),
1056 },
1057 };
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1058 *self.handle.borrow_mut() = Some(handle.clone());
1059 Ok(handle)
1060 }
1061
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01062 /// A new credential from the store. Its remote is worked out from the
1063 /// key once this isolate knows where the namespace's remotes start;
Merge branch 'worktree-agent-a2013627e5ea4ab13'1064 /// until then the store is asked (`info()`) alongside the token. The
1065 /// fallback store's remotes are known from its address.
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01066 pub async fn mint(&self, scope: Scope, using: Use) -> Result<GitAccess> {
1067 let args = [scope.as_str().into(), using.ttl_seconds().into()];
Merge branch 'worktree-agent-a2013627e5ea4ab13'1068 if let Target::Fallback { settings, .. } = &self.binding {
1069 let token: RawToken = js::from_js(&self.call("createToken", &args, true).await?)?;
1070 return Ok(GitAccess { remote: settings.remote(&self.namespace, &self.name), token: token.plaintext });
1071 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01072 let prefix = REMOTE_PREFIX.with(|prefixes| prefixes.borrow().get(&self.namespace).cloned());
1073 if let Some(prefix) = prefix {
1074 let token: RawToken = js::from_js(&self.call("createToken", &args, true).await?)?;
1075 return Ok(GitAccess { remote: remote_from(&prefix, &self.name), token: token.plaintext });
1076 }
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1077 self.handle().await?;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01078 let (info, token) = futures_util::future::join(self.call("info", &[], true), self.call("createToken", &args, true)).await;
1079 let info: RawInfo = js::from_js(&info?)?;
1080 let token: RawToken = js::from_js(&token?)?;
1081 match learn_prefix(&info.remote, &self.name) {
1082 Some(prefix) => REMOTE_PREFIX.with(|prefixes| {
1083 prefixes.borrow_mut().insert(self.namespace.clone(), prefix);
1084 }),
1085 None => worker::console_error!("the git store's remote {} does not end in {}.git", info.remote, self.name),
1086 }
1087 Ok(GitAccess { remote: info.remote, token: token.plaintext })
1088 }
1089}
1090
1091impl Drop for ArtifactsRepo {
1092 fn drop(&mut self) {
1093 let symbol = js::get(&worker::js_sys::global(), "Symbol");
1094 let dispose = js::get(&symbol, "dispose");
Merge branch 'worktree-agent-a2013627e5ea4ab13'1095 let Some(Target::Js(handle)) = self.handle.borrow_mut().take() else { return };
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1096 if let Ok(function) = Reflect::get(&handle, &dispose).and_then(|value| value.dyn_into::<worker::js_sys::Function>()) {
1097 let _ = function.call0(&handle);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01098 }
1099 }
1100}
1101
1102#[derive(Deserialize)]
1103#[serde(rename_all = "camelCase")]
1104struct RawCommit {
1105 hash: String,
1106 tree_hash: String,
1107 message: String,
1108 author: Signature,
1109 parents: Vec<String>,
1110 /// Seconds since the epoch.
1111 authored_at: u64,
1112}
1113
1114#[derive(Deserialize)]
1115struct RawEntry {
1116 name: String,
1117 hash: String,
1118 #[serde(rename = "type")]
1119 kind: EntryKind,
1120}
1121
1122#[derive(Deserialize)]
1123struct RawInfo {
1124 remote: String,
1125}
1126
1127#[derive(Deserialize)]
1128struct RawToken {
1129 plaintext: String,
1130}
1131
Merge branch 'worktree-agent-a2013627e5ea4ab13'1132/// The bytes of a `Blob` (or of the bytes the fallback store sent), or
1133/// `None` for null.
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01134async fn blob_bytes(blob: JsValue) -> Result<Option<Vec<u8>>> {
1135 if blob.is_null() || blob.is_undefined() {
1136 return Ok(None);
1137 }
Merge branch 'worktree-agent-a2013627e5ea4ab13'1138 if let Some(bytes) = blob.dyn_ref::<Uint8Array>() {
1139 return Ok(Some(bytes.to_vec()));
1140 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01141 let buffer = js::call(&blob, "arrayBuffer", &[]).await?;
1142 Ok(Some(Uint8Array::new(&buffer).to_vec()))
1143}
1144
1145impl GitRepo for ArtifactsRepo {
1146 /// One made a while ago, here or in another isolate, else a new one.
1147 async fn access(&self, scope: Scope) -> Result<GitAccess> {
Merge branch 'worktree-agent-a2013627e5ea4ab13'1148 if let Some((access, _)) = kept(self.shared.as_deref(), &self.cred_key, scope, Use::Internal).await {
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01149 return Ok(access);
1150 }
1151 let access = self.mint(scope, Use::Internal).await?;
Merge branch 'worktree-agent-a2013627e5ea4ab13'1152 keep(self.shared.as_deref(), &self.cred_key, scope, Use::Internal, &access).await;
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01153 Ok(access)
1154 }
1155
1156 async fn branches(&self) -> Result<Vec<Branch>> {
1157 let key = branches_key(self.refs_version);
1158 if let Some(key) = &key
1159 && let Some(bytes) = self.get_key(key).await
1160 && let Ok(branches) = serde_json::from_slice::<Vec<Branch>>(&bytes)
1161 {
1162 return Ok(branches);
1163 }
1164 let branches = crate::refs::branches(&self.access(Scope::Read).await?).await?;
1165 if let (Some(key), Ok(bytes)) = (&key, serde_json::to_vec(&branches)) {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1166 self.put_key(key, bytes);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01167 }
1168 Ok(branches)
1169 }
1170
1171 async fn log(&self, git_ref: &str, limit: u32) -> Result<Vec<Commit>> {
1172 // History from a commit never changes, so a log asked for by hash is
1173 // kept like an object: walking it is a read per commit. One asked
1174 // for by a branch's name is kept until the branch can have moved.
1175 let key = log_key(git_ref, limit, self.refs_version);
1176 if let Some(key) = &key
1177 && let Some(bytes) = self.get_key(key).await
1178 && let Ok(commits) = serde_json::from_slice::<Vec<Commit>>(&bytes)
1179 {
1180 return Ok(commits);
1181 }
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 251182 let commits = match &key {
1183 Some(CacheKey::Forever(_)) if limit >= SPLICE_FROM => self.spliced_log(git_ref, limit).await?,
1184 _ => self.read_log(git_ref, limit).await?,
1185 };
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01186 // An unknown ref logs nothing; that is not kept, in case it arrives.
1187 if !commits.is_empty()
1188 && let Ok(bytes) = serde_json::to_vec(&commits)
1189 {
1190 if let Some(key) = &key {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1191 self.put_key(key, bytes.clone());
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01192 }
1193 // The same history, by the commit the name led to.
1194 if !is_commit_hash(git_ref)
1195 && let Some(CacheKey::Forever(path)) = log_key(&commits[0].hash, limit, None)
1196 {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1197 self.keep_at(&path, bytes, OBJECT_MAX_AGE);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01198 }
1199 }
1200 Ok(commits)
1201 }
1202
1203 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
1204 // A commit never changes.
1205 if let Some(bytes) = self.cached("commit", commit_hash).await
1206 && let Ok(parents) = serde_json::from_slice::<Vec<String>>(&bytes)
1207 {
1208 return Ok(Some(parents));
1209 }
1210 let commit: Option<RawCommit> = js::from_js(&self.call("readCommit", &[commit_hash.into()], true).await?)?;
1211 let parents = commit.map(|commit| commit.parents);
1212 if let Some(parents) = &parents
1213 && let Ok(bytes) = serde_json::to_vec(parents)
1214 {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1215 self.keep("commit", commit_hash, bytes);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01216 }
1217 Ok(parents)
1218 }
1219
1220 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
1221 if let Some(bytes) = self.cached("tree", tree_hash).await
1222 && let Ok(entries) = serde_json::from_slice::<Vec<TreeEntry>>(&bytes)
1223 {
1224 return Ok(Some(entries));
1225 }
1226 let entries: Option<Vec<RawEntry>> = js::from_js(&self.call("readTree", &[tree_hash.into()], true).await?)?;
1227 let entries: Option<Vec<TreeEntry>> = entries.map(|entries| {
1228 entries
1229 .into_iter()
1230 .map(|entry| TreeEntry {
1231 name: entry.name,
1232 hash: entry.hash,
1233 kind: entry.kind,
1234 })
1235 .collect()
1236 });
1237 if let Some(entries) = &entries
1238 && let Ok(bytes) = serde_json::to_vec(entries)
1239 {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1240 self.keep("tree", tree_hash, bytes);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01241 }
1242 Ok(entries)
1243 }
1244
1245 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
1246 if let Some(bytes) = self.cached("blob", blob_hash).await {
1247 return Ok(Some(bytes));
1248 }
1249 let bytes = blob_bytes(self.call("readBlob", &[blob_hash.into()], true).await?).await?;
1250 if let Some(bytes) = &bytes {
1251 meters::record_bytes("binding.read_blob", &self.key, 0, bytes.len() as u64);
1252 }
1253 if let Some(bytes) = bytes.as_ref().filter(|bytes| bytes.len() <= MAX_CACHED_BLOB) {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1254 self.keep("blob", blob_hash, bytes.clone());
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01255 }
1256 Ok(bytes)
1257 }
1258
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb971259 /// Kept for good by hash, as a blob is: its bytes never cross into
1260 /// this isolate's memory, only the size of the store's answer.
1261 async fn blob_size(&self, blob_hash: &str) -> Result<Option<u64>> {
1262 let path = format!("size/{blob_hash}");
1263 if let Some(bytes) = self.cached_at(&path, true).await
1264 && let Some(size) = std::str::from_utf8(&bytes).ok().and_then(|text| text.parse().ok())
1265 {
1266 return Ok(Some(size));
1267 }
1268 let blob = self.call("readBlob", &[blob_hash.into()], true).await?;
1269 let size = if blob.is_null() || blob.is_undefined() {
1270 None
1271 } else if let Some(bytes) = blob.dyn_ref::<Uint8Array>() {
1272 Some(u64::from(bytes.length()))
1273 } else {
1274 js::get(&blob, "size").as_f64().map(|size| size as u64)
1275 };
1276 if let Some(size) = size {
1277 meters::record_bytes("binding.read_blob", &self.key, 0, size);
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1278 self.keep_at(&path, size.to_string().into_bytes(), OBJECT_MAX_AGE);
Merge branch 'main' into worktree-agent-a69aeabc4b0deeb971279 }
1280 Ok(size)
1281 }
1282
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01283 async fn read_file(&self, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>> {
1284 let key = file_key(git_ref, path, self.refs_version);
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing1285 // A path that is not a file is remembered too, briefly: the store
1286 // answers each such read as a rejected read (crawlers asking for
1287 // old or missing paths make most of them). Never for the fallback
1288 // store, which can be behind.
1289 let remember_absent = !self.binding.is_fallback();
1290 if let Some(key) = &key {
1291 let (found, absent) = futures_util::future::join(self.get_key(key), async {
1292 remember_absent && self.known_absent(key).await
1293 })
1294 .await;
1295 if let Some(bytes) = found {
1296 return Ok(Some(bytes));
1297 }
1298 if absent {
1299 return Ok(None);
1300 }
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01301 }
1302 let args = js::to_js(&serde_json::json!({ "ref": git_ref, "path": path }))?;
Teams and CODEOWNERS, labels and milestones, dependency updates, the security suite, and a clearer top bar1303 // The store answers a path that is not a file at the ref with
1304 // NOT_FOUND (its "read rejected"), not with nothing.
1305 let bytes = match self.call("readFile", &[args], true).await {
1306 Ok(blob) => blob_bytes(blob).await?,
1307 Err(failed) if failed.is("NOT_FOUND") => None,
1308 Err(failed) => return Err(failed.into()),
1309 };
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01310 if let Some(bytes) = &bytes {
1311 meters::record_bytes("binding.read_file", &self.key, 0, bytes.len() as u64);
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing1312 } else if remember_absent && let Some(key) = &key {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1313 self.keep_at(&absent_path(key), vec![1], ABSENT_MAX_AGE);
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01314 }
1315 if let (Some(key), Some(bytes)) = (&key, bytes.as_ref().filter(|bytes| bytes.len() <= MAX_CACHED_BLOB)) {
A push is checked once and side by side: its pack is read and its bases fetched once for the rules, the workflow gate and the secret scan, which run together, other services are asked while the pack is read, and cache writes and rule records finish after git has its answer1316 self.put_key(key, bytes.clone());
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01317 }
1318 Ok(bytes)
1319 }
1320
1321 async fn fork(&self, target_key: &str) -> Result<()> {
1322 let (namespace, name) = locate(target_key);
1323 if namespace != self.namespace {
1324 return Err(worker::Error::RustError(format!(
1325 "a fork stays in its repository's namespace: {target_key} is not in {}",
1326 self.namespace
1327 )));
1328 }
1329 let options = js::to_js(&serde_json::json!({ "defaultBranchOnly": true }))?;
1330 match self.call("fork", &[name.as_str().into(), options], false).await {
1331 Err(failed) if !failed.is("ALREADY_EXISTS") => Err(failed.into()),
1332 _ => Ok(()),
1333 }
1334 }
1335
1336 fn at_refs_version(&mut self, version: Option<u64>) {
Merge branch 'worktree-agent-a2013627e5ea4ab13'1337 // The fallback store holds what the last backup held, which may be
1338 // behind what was kept under the version: nothing is kept for it.
1339 self.refs_version = if self.binding.is_fallback() { None } else { version };
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01340 }
1341}
1342
1343#[cfg(test)]
1344mod tests {
1345 use super::*;
1346
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 251347 fn chain(names: &[&str]) -> Vec<Commit> {
1348 names
1349 .iter()
1350 .enumerate()
1351 .map(|(at, name)| Commit {
1352 hash: (*name).to_owned(),
1353 tree_hash: String::new(),
1354 message: String::new(),
1355 author: g1t_contracts::repos::Signature { name: "a".into(), email: "a@example.com".into() },
1356 parents: names.get(at + 1).map(|parent| vec![(*parent).to_owned()]).unwrap_or_default(),
1357 authored_at: String::new(),
1358 })
1359 .collect()
1360 }
1361
1362 fn hashes(commits: &[Commit]) -> Vec<&str> {
1363 commits.iter().map(|commit| commit.hash.as_str()).collect()
1364 }
1365
1366 #[test]
1367 fn a_history_is_the_new_commits_then_the_one_kept_from_an_old_head() {
1368 // The branch moved from c3 to c5; c3's history (limit 4) was kept.
1369 let short = chain(&["c5", "c4", "c3", "c2"]);
1370 let kept = chain(&["c3", "c2", "c1", "c0"]);
1371 assert_eq!(hashes(&splice(&short, 2, kept.clone(), 4)), ["c5", "c4", "c3", "c2"]);
1372 assert_eq!(hashes(&splice(&short, 2, kept.clone(), 6)), ["c5", "c4", "c3", "c2", "c1", "c0"]);
1373 // A kept history that reached the first commit ends there.
1374 assert_eq!(hashes(&splice(&short, 2, chain(&["c3", "c2"]), 10)), ["c5", "c4", "c3", "c2"]);
1375 }
1376
1377 #[test]
Merge branch drift: count across merges the way git does; v2 cache key1378 fn a_splice_joins_at_the_newest_kept_history_without_repeats_or_gaps() {
1379 // 16 read from c20; histories of 8 kept from c17 and c12.
1380 let all: Vec<String> = (0..=20).rev().map(|i| format!("c{i}")).collect();
1381 let names: Vec<&str> = all.iter().map(String::as_str).collect();
1382 let short = chain(&names[..16]);
1383 let kept_from = |name: &str| {
1384 let at = names.iter().position(|n| *n == name).unwrap();
1385 chain(&names[at..(at + 8).min(names.len())])
1386 };
1387 let mut kept: Vec<Option<Vec<Commit>>> = vec![None; short.len()];
1388 kept[3] = Some(kept_from("c17"));
1389 kept[8] = Some(kept_from("c12"));
1390 let joined = splice_first(&short, kept.clone(), 8).unwrap();
1391 assert_eq!(hashes(&joined), names[..8]);
1392 // Newest first, every commit once, each the first parent of the one before.
1393 let joined = splice_first(&short, kept, 11).unwrap();
1394 assert_eq!(hashes(&joined), names[..11]);
1395 for pair in joined.windows(2) {
1396 assert_eq!(pair[0].parents.first(), Some(&pair[1].hash));
1397 }
1398 let unique: std::collections::HashSet<_> = joined.iter().map(|commit| &commit.hash).collect();
1399 assert_eq!(unique.len(), joined.len());
1400 }
1401
1402 #[test]
1403 fn a_kept_history_that_does_not_fit_is_not_spliced() {
1404 let short = chain(&["c5", "c4", "c3", "c2"]);
1405 // Kept under c4's key, but from somewhere else.
1406 let wrong = vec![None, Some(chain(&["x4", "x3"])), None, None];
1407 assert!(splice_first(&short, wrong, 10).is_none());
1408 // Starts at c4 but goes on to another commit.
1409 let forked = vec![None, Some(chain(&["c4", "y3"])), None, None];
1410 assert!(splice_first(&short, forked, 10).is_none());
1411 // Ends at c4 where `short` goes on: not the history from c4.
1412 let cut = vec![None, Some(chain(&["c4"])), None, None];
1413 assert!(splice_first(&short, cut, 10).is_none());
1414 // The commit read itself is never spliced onto.
1415 let own = vec![Some(chain(&["c5", "c4"])), None, None, None];
1416 assert!(splice_first(&short, own, 10).is_none());
1417 // Nothing kept.
1418 assert!(splice_first(&short, vec![None; 4], 10).is_none());
1419 // The last commit read: anything kept from it fits.
1420 let last = vec![None, None, None, Some(chain(&["c2", "c1", "c0"]))];
1421 assert_eq!(hashes(&splice_first(&short, last, 10).unwrap()), ["c5", "c4", "c3", "c2", "c1", "c0"]);
1422 }
1423
1424 #[test]
Merge project overview: one branch_drift call, spliced histories, cached tags, 6 repos calls instead of 251425 fn only_long_histories_by_hash_are_spliced() {
1426 assert!(SPLICE_PROBE < SPLICE_FROM);
1427 let hash = "a".repeat(40);
1428 assert!(matches!(log_key(&hash, SPLICE_FROM, None), Some(CacheKey::Forever(_))));
1429 assert!(matches!(log_key("main", SPLICE_FROM, Some(1)), Some(CacheKey::Versioned(_))));
1430 }
1431
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01432 fn access(token: &str) -> GitAccess {
1433 GitAccess {
1434 remote: "https://store.example/acme--rocket.git".to_owned(),
1435 token: token.to_owned(),
1436 }
1437 }
1438
1439 #[test]
1440 fn internal_credentials_live_an_hour_and_are_reused_for_fifty_minutes() {
1441 assert_eq!(Use::Internal.ttl_seconds(), 3_600);
1442 assert_eq!(Use::Internal.reuse_ms(), 50 * 60 * 1000);
1443 // Handed out: five minutes, reused three, so at least two are left.
1444 assert_eq!(Use::Handout.ttl_seconds(), 300);
1445 assert_eq!(Use::Handout.reuse_ms(), 180_000);
1446 assert_eq!(CREDENTIAL_LIFE_MS, 300_000);
1447 for using in [Use::Internal, Use::Handout] {
1448 assert!(u64::from(using.ttl_seconds()) * 1000 - using.reuse_ms() >= 120_000);
1449 }
1450 }
1451
1452 #[test]
1453 fn a_credential_is_reused_only_while_it_has_time_left() {
1454 let mut kept = Credentials::default();
1455 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
1456 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000).unwrap().token, "r1");
1457 let last = 1_000 + INTERNAL_REUSE_MS - 1;
1458 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Internal, last).unwrap().token, "r1");
1459 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, last + 1).is_none());
1460 }
1461
1462 #[test]
1463 fn a_credential_is_kept_for_its_own_repository_scope_and_use() {
1464 let mut kept = Credentials::default();
1465 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
1466 // A read credential never stands in for a write one.
1467 assert!(kept.get("acme--rocket", Scope::Write, Use::Internal, 1_000).is_none());
1468 assert!(kept.get("acme--booster", Scope::Read, Use::Internal, 1_000).is_none());
1469 // An hour-long credential is never handed out.
1470 assert!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000).is_none());
1471 kept.keep("acme--rocket", Scope::Read, Use::Handout, access("h1"), 1_000, 1_000);
1472 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000).unwrap().token, "h1");
1473 assert!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000 + HANDOUT_REUSE_MS).is_none());
1474 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000 + HANDOUT_REUSE_MS).is_some());
1475 }
1476
1477 #[test]
1478 fn a_shared_credential_is_reused_only_in_its_own_window() {
1479 let value = serde_json::to_vec(&SharedCredential {
1480 remote: "https://store.example/acme--rocket.git".to_owned(),
1481 token: "r1".to_owned(),
1482 made: 10_000,
1483 })
1484 .unwrap();
1485 let (access, made) = shared_credential(&value, 10_000 + INTERNAL_REUSE_MS - 1, Use::Internal).unwrap();
1486 assert_eq!(access.token, "r1");
1487 // Kept here only for what is left of its window, not a new one.
1488 assert_eq!(made, 10_000);
1489 assert!(shared_credential(&value, 10_000 + INTERNAL_REUSE_MS, Use::Internal).is_none());
1490 assert!(shared_credential(&value, 10_000 + HANDOUT_REUSE_MS, Use::Handout).is_none());
1491 assert!(shared_credential(b"not json", 10_000, Use::Internal).is_none());
1492 // Each repository, scope and use has its own key, and none is read
1493 // from before uses differed.
1494 assert_eq!(shared_key("acme--rocket", Scope::Read, Use::Internal), "cred2:acme--rocket:read:internal");
1495 assert_ne!(shared_key("acme--rocket", Scope::Read, Use::Internal), shared_key("acme--rocket", Scope::Write, Use::Internal));
1496 assert_ne!(shared_key("acme--rocket", Scope::Read, Use::Internal), shared_key("acme--rocket", Scope::Read, Use::Handout));
1497 }
1498
1499 #[test]
1500 fn a_shared_credential_kept_here_expires_with_the_original() {
1501 let mut kept = Credentials::default();
1502 // Made at 1_000 elsewhere, found here at 100_000.
1503 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 100_000);
1504 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 100_000).is_some());
1505 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000 + INTERNAL_REUSE_MS).is_none());
1506 }
1507
1508 #[test]
1509 fn a_turned_down_credential_is_forgotten_and_old_ones_are_dropped() {
1510 let mut kept = Credentials::default();
1511 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
1512 kept.keep("acme--rocket", Scope::Write, Use::Handout, access("w1"), 1_000, 1_000);
1513 kept.keep("acme--booster", Scope::Read, Use::Internal, access("b1"), 1_000, 1_000);
1514 kept.forget("acme--rocket");
1515 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000).is_none());
1516 assert!(kept.get("acme--rocket", Scope::Write, Use::Handout, 1_000).is_none());
1517 assert!(kept.get("acme--booster", Scope::Read, Use::Internal, 1_000).is_some());
1518 // Keeping another later drops the expired one from the map.
1519 let later = 1_000 + INTERNAL_REUSE_MS;
1520 kept.keep("acme--other", Scope::Read, Use::Internal, access("o1"), later, later);
1521 assert_eq!(kept.kept.len(), 1);
1522 }
1523
1524 #[test]
1525 fn a_remote_is_worked_out_from_where_its_namespace_starts() {
1526 // As https://developers.cloudflare.com/artifacts/api/git-protocol/ documents.
1527 let remote = "https://1e6f2cffa3f445920836e8ebe446bb58.artifacts.cloudflare.net/git/g1t/acme--rocket.git";
1528 let prefix = learn_prefix(remote, "acme--rocket").unwrap();
1529 assert_eq!(prefix, "https://1e6f2cffa3f445920836e8ebe446bb58.artifacts.cloudflare.net/git/g1t/");
1530 assert_eq!(remote_from(&prefix, "pulls--pul_1"), prefix.clone() + "pulls--pul_1.git");
1531 assert_eq!(remote_from(&prefix, "acme--rocket"), remote);
1532 // A remote that does not end in the name teaches nothing.
1533 assert_eq!(learn_prefix(remote, "rocket"), None);
1534 assert_eq!(learn_prefix("https://x/acme--rocket", "acme--rocket"), None);
1535 // And back: a remote names its key.
1536 assert_eq!(key_from_remote(remote).as_deref(), Some("acme--rocket"));
1537 assert_eq!(
1538 key_from_remote("https://a.artifacts.cloudflare.net/git/g1t-us-1/acme--rocket.git").as_deref(),
1539 Some("g1t-us-1/acme--rocket")
1540 );
1541 assert_eq!(locate("g1t-us-1/acme--rocket"), ("g1t-us-1".to_owned(), "acme--rocket".to_owned()));
1542 assert_eq!(locate("acme--rocket"), ("g1t".to_owned(), "acme--rocket".to_owned()));
1543 }
1544
1545 #[test]
Merge branch 'worktree-agent-a2013627e5ea4ab13'1546 fn the_fallback_stores_credentials_and_remotes_are_its_own() {
1547 assert_eq!(cred_key("acme--rocket", false), "acme--rocket");
1548 assert_eq!(cred_key("g1t-us-1/acme--rocket", true), "fallback:g1t-us-1/acme--rocket");
1549 // A credential Artifacts made is never handed out for the fallback
1550 // store, nor the reverse.
1551 assert_ne!(
1552 shared_key(&cred_key("acme--rocket", true), Scope::Write, Use::Internal),
1553 shared_key(&cred_key("acme--rocket", false), Scope::Write, Use::Internal)
1554 );
1555 // Its remotes name their keys as Artifacts' do.
1556 let settings = fallback::Settings::from_vars(Some("https://gitstore.example"), Some("0123456789abcdef"), Some("*"), None).unwrap();
1557 assert_eq!(key_from_remote(&settings.remote("g1t", "acme--rocket")).as_deref(), Some("acme--rocket"));
1558 assert_eq!(key_from_remote(&settings.remote("g1t-us-1", "pulls--pul_1")).as_deref(), Some("g1t-us-1/pulls--pul_1"));
1559 // Git requests to it count toward its own health, not Artifacts'.
1560 FALLBACK_BASE.with(|base| *base.borrow_mut() = Some(format!("{}/git/", settings.url)));
1561 assert_eq!(health_namespace(&settings.remote("g1t-us-1", "acme--rocket")), "g1t-us-1@fallback");
1562 assert_eq!(health_namespace("https://a.artifacts.cloudflare.net/git/g1t-us-1/acme--rocket.git"), "g1t-us-1");
1563 FALLBACK_BASE.with(|base| *base.borrow_mut() = None);
1564 }
1565
1566 #[test]
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1567 fn the_memory_cache_drops_its_oldest_past_its_budget() {
1568 let mut cache = MemoryCache::default();
1569 let chunk = vec![7u8; MEMORY_CACHE_BYTES / 16];
1570 for n in 0..17 {
1571 cache.put(format!("k{n}"), &chunk);
1572 }
1573 // Sixteen chunks fit; the seventeenth pushed the first out.
1574 assert!(cache.get("k0").is_none());
1575 assert_eq!(cache.get("k16").map(|b| b.len()), Some(chunk.len()));
1576 assert!(cache.bytes <= MEMORY_CACHE_BYTES);
1577 // Too large to keep at all.
1578 cache.put("big".into(), &vec![0u8; MEMORY_CACHE_BYTES / 16 + 1]);
1579 assert!(cache.get("big").is_none());
1580 }
1581
1582 #[test]
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01583 fn binding_methods_have_snake_case_meters() {
1584 assert_eq!(meter_of("createToken"), "binding.create_token");
1585 assert_eq!(meter_of("readBlob"), "binding.read_blob");
1586 assert_eq!(meter_of("get"), "binding.get");
1587 }
1588
1589 #[test]
1590 fn reads_by_name_are_kept_under_the_refs_version_and_by_hash_for_good() {
1591 let hash = "a".repeat(40);
1592 assert_eq!(log_key(&hash, 1, Some(3)), Some(CacheKey::Forever(format!("log/{hash}-1"))));
1593 assert_eq!(log_key(&hash, 1, None), Some(CacheKey::Forever(format!("log/{hash}-1"))));
1594 // A branch is kept only when the version is known.
1595 assert_eq!(log_key("main", 1, None), None);
1596 let v3 = log_key("main", 1, Some(3)).unwrap();
1597 assert!(matches!(&v3, CacheKey::Versioned(path) if path.starts_with("vlog/3/")));
1598 // A push moves the version and leaves the old answer behind.
1599 assert_ne!(Some(v3), log_key("main", 1, Some(4)));
1600 assert_ne!(log_key("main", 1, Some(3)), log_key("main", 50, Some(3)));
1601 assert_ne!(log_key("main", 1, Some(3)), log_key("dev", 1, Some(3)));
1602 // Odd branch names make a usable address.
1603 assert!(matches!(log_key("fix/#1 %20", 1, Some(3)), Some(CacheKey::Versioned(path)) if !path.contains('#') && !path.contains(' ')));
1604 assert_eq!(branches_key(None), None);
1605 assert_ne!(branches_key(Some(1)), branches_key(Some(2)));
1606 assert!(matches!(file_key(&hash, "src/main.rs", None), Some(CacheKey::Forever(_))));
1607 assert_eq!(file_key("main", "src/main.rs", None), None);
1608 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "b", Some(1)));
1609 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "a", Some(2)));
Repos: a path found not to be a file is remembered for 10 minutes, so repeated misses read nothing1610 // A path noted as not a file sits beside the file's own key, and a
1611 // push (a new refs version) leaves the old note behind.
1612 let at = |version| absent_path(&file_key("main", "a", Some(version)).unwrap());
1613 assert!(at(1).starts_with("absent/vfile/1/"));
1614 assert_ne!(at(1), at(2));
1615 assert_eq!(absent_path(&file_key(&hash, "a", None).unwrap()), format!("absent/file/{hash}/{}", g1t_secrets::sha256_hex("a")));
Cleanup: clippy is quiet outside billing, the README says g1t, and http-cache-semantics is 4.3.01616 }
1617
1618 #[test]
1619 fn only_full_lowercase_hashes_are_kept() {
1620 assert!(is_commit_hash(&"a".repeat(40)));
1621 assert!(is_commit_hash(&"0123456789abcdef".repeat(4)));
1622 assert!(!is_commit_hash("main"));
1623 assert!(!is_commit_hash(&"A".repeat(40)));
1624 assert!(!is_commit_hash(&"a".repeat(39)));
1625 }
1626}

This file's history is long; its oldest lines are credited to the oldest commit read.