Sign in

flagon-io/g1t

public

g1t/services/repos/src/store.rs

1,114 lines46,544 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Rust repos service with shipping; pull requests kept in the model1//! The storage that actually holds git repositories.
2//!
3//! The service depends on the [`GitStore`] and [`GitRepo`] ports;
4//! [`ArtifactsStore`] is the adapter for Cloudflare Artifacts.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily5//!
6//! Every call on the binding goes through [`invoke`]: it is counted
7//! (meters.rs), timed for the store's health, refused at once while its
8//! namespace's breaker is open, and tried again after a failure that may
9//! pass when it only reads (resilience.rs). Repositories may live in
10//! several namespaces (shards.rs).
Rust repos service with shipping; pull requests kept in the model11
Pull requests from branches12use g1t_contracts::repos::{Branch, Commit, EntryKind, GitAccess, Signature, TreeEntry};
RFC 3339 timestamps in identity and repos13use g1t_contracts::time::rfc3339;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily14use g1t_kit::js::{self, Thrown};
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms15use serde::{Deserialize, Serialize};
16use std::cell::RefCell;
17use std::collections::HashMap;
18use std::rc::Rc;
Rust repos service with shipping; pull requests kept in the model19use worker::js_sys::{Reflect, Uint8Array};
20use worker::wasm_bindgen::{JsCast, JsValue};
21use worker::{Env, Result};
22
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily23use crate::meters::{self, Outcome};
24use crate::resilience::{self, Admit, Busy, Failure};
25use crate::shards;
26
27/// Credentials that never leave this service: g1t has already decided who
28/// may do what before one is used. They live an hour and are used for 50
29/// minutes, so each one used has at least ten minutes left.
30const INTERNAL_TTL_SECONDS: u32 = 3_600;
31const INTERNAL_REUSE_MS: u64 = 50 * 60 * 1000;
32/// Credentials handed out (`git_access`, to sandboxes): five minutes, used
33/// for three, so whoever gets one has at least two.
34const HANDOUT_TTL_SECONDS: u32 = 300;
35const HANDOUT_REUSE_MS: u64 = 180_000;
36/// How long a handed-out credential stays valid, in milliseconds.
37pub const CREDENTIAL_LIFE_MS: u64 = HANDOUT_TTL_SECONDS as u64 * 1000;
Rust repos service with shipping; pull requests kept in the model38
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms39#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
Rust repos service with shipping; pull requests kept in the model40pub enum Scope {
41 Read,
42 Write,
43}
44
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms45impl Scope {
46 fn as_str(self) -> &'static str {
47 match self {
48 Scope::Read => "read",
49 Scope::Write => "write",
50 }
51 }
52}
53
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily54/// Who a credential is for.
55#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
56pub enum Use {
57 /// This service, talking to the store itself.
58 Internal,
59 /// Someone outside it, through `git_access`.
60 Handout,
61}
62
63impl Use {
64 pub fn ttl_seconds(self) -> u32 {
65 match self {
66 Use::Internal => INTERNAL_TTL_SECONDS,
67 Use::Handout => HANDOUT_TTL_SECONDS,
68 }
69 }
70
71 pub fn reuse_ms(self) -> u64 {
72 match self {
73 Use::Internal => INTERNAL_REUSE_MS,
74 Use::Handout => HANDOUT_REUSE_MS,
75 }
76 }
77
78 fn as_str(self) -> &'static str {
79 match self {
80 Use::Internal => "internal",
81 Use::Handout => "handout",
82 }
83 }
84}
85
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms86/// Where a credential handed out came from, for `Server-Timing`.
87#[derive(Clone, Copy, PartialEq, Eq, Debug)]
88pub enum Kept {
89 /// This isolate made it, or had it from another, a moment ago.
90 Isolate,
91 /// Another isolate made it and shared it.
92 Shared,
93}
94
95impl Kept {
96 pub fn as_str(self) -> &'static str {
97 match self {
98 Kept::Isolate => "isolate",
99 Kept::Shared => "shared",
100 }
101 }
102}
103
Rust repos service with shipping; pull requests kept in the model104/// A place repositories live. `key` is the store's own name for a repo.
105#[allow(async_fn_in_trait)]
106pub trait GitStore {
107 type Repo: GitRepo;
108
109 /// Creates an empty repository. Succeeds if it already exists.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily110 async fn create(&self, key: &str, description: Option<&str>, default_branch: &str) -> Result<()>;
Rust repos service with shipping; pull requests kept in the model111 async fn open(&self, key: &str) -> Result<Self::Repo>;
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms112 /// A credential for `key` made a moment ago, if the store keeps one.
113 async fn kept_access(&self, _key: &str, _scope: Scope) -> Option<(GitAccess, Kept)> {
114 None
115 }
116 /// A new credential for `key`, which the store may keep for next time.
117 async fn mint_access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
118 self.open(key).await?.access(scope).await
119 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily120 /// A remote URL and credential for this service's own use. A store may
121 /// hand out one it made a moment ago.
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms122 async fn access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
123 match self.kept_access(key, scope).await {
124 Some((access, _)) => Ok(access),
125 None => self.mint_access(key, scope).await,
126 }
127 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily128 /// A short-lived credential for someone outside this service.
129 async fn handout(&self, key: &str, scope: Scope) -> Result<GitAccess> {
130 self.access(key, scope).await
131 }
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms132 /// Stops handing out the credentials it keeps for `key`: the store
133 /// turned one down, or the repository is gone.
134 async fn forget_access(&self, _key: &str) {}
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look135 /// Removes a repository and everything in it, for good. Succeeds if it
136 /// is already gone.
137 async fn delete(&self, key: &str) -> Result<()>;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily138 /// The namespaces new repositories may be placed in (shards.rs).
139 fn namespaces(&self) -> Vec<String> {
140 vec![shards::DEFAULT_NAMESPACE.to_owned()]
141 }
142 /// The namespace bound as the default.
143 fn default_namespace(&self) -> String {
144 shards::DEFAULT_NAMESPACE.to_owned()
145 }
Rust repos service with shipping; pull requests kept in the model146}
147
148/// One open repository.
149#[allow(async_fn_in_trait)]
150pub trait GitRepo {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily151 /// A remote URL and credential for git itself, for this service.
Rust repos service with shipping; pull requests kept in the model152 async fn access(&self, scope: Scope) -> Result<GitAccess>;
Pull requests from branches153 /// Every branch and the commit it points to.
154 async fn branches(&self) -> Result<Vec<Branch>>;
Rust repos service with shipping; pull requests kept in the model155 /// Newest first along the first-parent chain; empty for an unknown ref.
156 async fn log(&self, git_ref: &str, limit: u32) -> Result<Vec<Commit>>;
157 /// The parents of a commit, or `None` if the commit does not exist.
158 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>>;
159 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>>;
160 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>>;
161 /// `None` when the ref or path does not resolve to a file.
162 async fn read_file(&self, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>>;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily163 /// Makes a copy of this repository under `target_key`, in the same
164 /// namespace.
Rust repos service with shipping; pull requests kept in the model165 async fn fork(&self, target_key: &str) -> Result<()>;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily166 /// The version of the repository's refs (registry.rs `RefsState`),
167 /// when answers about branches may be kept under it (R9). Reads by
168 /// branch name are then kept until the version moves.
169 fn at_refs_version(&mut self, _version: Option<u64>) {}
170}
171
172thread_local! {
173 /// The namespace bound to `ARTIFACTS`, for keys that name none.
174 static DEFAULT_NS: RefCell<String> = RefCell::new(shards::DEFAULT_NAMESPACE.to_owned());
175 /// Where each namespace's remotes start: `https://<account>.artifacts.cloudflare.net/git/<namespace>/`.
176 static REMOTE_PREFIX: RefCell<HashMap<String, String>> = RefCell::new(HashMap::new());
177}
178
179/// The namespace and name a store key stands for.
180pub fn locate(key: &str) -> (String, String) {
181 let (namespace, name) = shards::split(key);
182 let namespace = namespace.map_or_else(|| DEFAULT_NS.with(|ns| ns.borrow().clone()), str::to_owned);
183 (namespace, name.to_owned())
184}
185
186/// The store key a git remote is for, from its last two path segments.
187pub fn key_from_remote(remote: &str) -> Option<String> {
188 let path = remote.trim_end_matches('/');
189 let path = path.strip_suffix(".git").unwrap_or(path);
190 let (rest, name) = path.rsplit_once('/')?;
191 let namespace = rest.rsplit('/').next()?;
192 let default = DEFAULT_NS.with(|ns| ns.borrow().clone());
193 Some(shards::compose(Some(namespace), name, &default))
194}
195
196/// Where a namespace's remotes start, learned from one remote the store
197/// gave for `name`.
198fn learn_prefix(remote: &str, name: &str) -> Option<String> {
199 remote.strip_suffix(&format!("{name}.git")).filter(|prefix| prefix.ends_with('/')).map(str::to_owned)
Rust repos service with shipping; pull requests kept in the model200}
201
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily202/// A repository's remote, from where its namespace's remotes start.
203fn remote_from(prefix: &str, name: &str) -> String {
204 format!("{prefix}{name}.git")
205}
206
207struct Namespace {
208 name: String,
Rust repos service with shipping; pull requests kept in the model209 binding: JsValue,
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily210}
211
212pub struct ArtifactsStore {
213 namespaces: Rc<Vec<Namespace>>,
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms214 /// Where isolates share the credentials they make; see shared.rs.
215 shared: Option<Rc<crate::shared::Shared>>,
Rust repos service with shipping; pull requests kept in the model216}
217
218impl ArtifactsStore {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms219 pub fn new(env: &Env, shared: Option<Rc<crate::shared::Shared>>) -> Result<Self> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily220 let config = env.var("ARTIFACTS_NAMESPACES").ok().map(|value| value.to_string());
221 let mut namespaces = Vec::new();
222 for (binding, name) in shards::bindings(config.as_deref()) {
223 match js::binding(env, &binding) {
224 Ok(value) => namespaces.push(Namespace { name, binding: value }),
225 // The default binding is required; the others are optional.
226 Err(error) if binding == shards::DEFAULT_BINDING => return Err(error),
227 Err(_) => worker::console_error!("ARTIFACTS_NAMESPACES names {binding}, which is not bound"),
228 }
229 }
230 if let Some(default) = namespaces.first() {
231 DEFAULT_NS.with(|ns| ns.borrow_mut().clone_from(&default.name));
232 }
233 // Optional: where remotes start, `https://<account>.artifacts.cloudflare.net/git`,
234 // so the first credential an isolate makes needs no `info()` either.
235 if let Ok(base) = env.var("ARTIFACTS_REMOTE_BASE") {
236 let base = base.to_string().trim_end_matches('/').to_owned();
237 if base.starts_with("https://") {
238 REMOTE_PREFIX.with(|prefixes| {
239 let mut prefixes = prefixes.borrow_mut();
240 for namespace in &namespaces {
241 prefixes.entry(namespace.name.clone()).or_insert_with(|| format!("{base}/{}/", namespace.name));
242 }
243 });
244 }
245 }
246 Ok(Self { namespaces: Rc::new(namespaces), shared })
247 }
248
249 fn binding(&self, namespace: &str) -> Result<&JsValue> {
250 self.namespaces
251 .iter()
252 .find(|candidate| candidate.name == namespace)
253 .map(|found| &found.binding)
254 .ok_or_else(|| worker::Error::RustError(format!("git store namespace {namespace} is not bound")))
255 }
256}
257
258/// A failed call on the binding.
259pub struct StoreError {
260 pub thrown: Thrown,
261 pub busy: Option<Busy>,
262}
263
264impl StoreError {
265 pub fn is(&self, code: &str) -> bool {
266 self.thrown.is(code)
Rust repos service with shipping; pull requests kept in the model267 }
268}
269
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily270impl From<StoreError> for worker::Error {
271 fn from(error: StoreError) -> Self {
272 match error.busy {
273 Some(busy) => busy.error(&error.thrown.to_string()),
274 None => error.thrown.into(),
275 }
276 }
277}
278
279/// The meter for a binding method: `binding.create_token`.
280fn meter_of(method: &str) -> String {
281 let mut out = String::from("binding.");
282 for c in method.chars() {
283 if c.is_ascii_uppercase() {
284 out.push('_');
285 out.push(c.to_ascii_lowercase());
286 } else {
287 out.push(c);
288 }
289 }
290 out
291}
292
293/// Seconds a caller is told to wait when the store is busy.
294const BUSY_RETRY_AFTER: u64 = 5;
295
296/// Calls `target[method](...args)` on namespace `namespace`'s binding, for
297/// the repository at `key`. `retry`: whether a failure that may pass is
298/// tried again (reads and credentials only).
299async fn invoke(
300 namespace: &str,
301 key: &str,
302 target: &JsValue,
303 method: &str,
304 args: &[JsValue],
305 retry: bool,
306) -> std::result::Result<JsValue, StoreError> {
307 let meter = meter_of(method);
308 let mut attempt = 0;
309 loop {
310 let now = g1t_kit::now_ms();
311 let admit = resilience::with_breaker(namespace, |breaker| breaker.admit(now));
312 if let Admit::Wait(ms) = admit {
313 meters::record_health(namespace, Outcome::Rejected, 0);
314 return Err(StoreError {
315 thrown: Thrown { code: None, message: format!("{method} not asked: the git store has been failing") },
316 busy: Some(Busy { rate_limited: false, retry_after: resilience::seconds(ms).max(BUSY_RETRY_AFTER) }),
317 });
318 }
319 meters::record(&meter, key, 0, 0);
320 let called = js::call(target, method, args).await;
321 let ms = g1t_kit::now_ms().saturating_sub(now);
322 match called {
323 Ok(value) => {
324 resilience::with_breaker(namespace, |breaker| breaker.succeeded());
325 meters::record_health(namespace, Outcome::Ok, ms);
326 return Ok(value);
327 }
328 Err(thrown) => {
329 let failure = resilience::classify(thrown.code.as_deref(), &thrown.message);
330 if failure == Failure::Permanent {
331 // An answer (NOT_FOUND, ALREADY_EXISTS): the store is up.
332 resilience::with_breaker(namespace, |breaker| breaker.succeeded());
333 meters::record_health(namespace, Outcome::Ok, ms);
334 return Err(StoreError { thrown, busy: None });
335 }
336 if retry && resilience::retry(failure, attempt) {
337 let wait = resilience::backoff_ms(failure, attempt, worker::js_sys::Math::random());
338 worker::Delay::from(std::time::Duration::from_millis(wait)).await;
339 attempt += 1;
340 continue;
341 }
342 resilience::with_breaker(namespace, |breaker| breaker.failed(failure, g1t_kit::now_ms()));
343 let outcome = if failure == Failure::RateLimited { Outcome::RateLimited } else { Outcome::Failed };
344 meters::record_health(namespace, outcome, ms);
345 worker::console_error!("git store {namespace}: {method} for {key} failed: {thrown}");
346 return Err(StoreError {
347 thrown,
348 busy: Some(Busy { rate_limited: failure == Failure::RateLimited, retry_after: BUSY_RETRY_AFTER }),
349 });
350 }
351 }
352 }
353}
354
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms355/// A credential as isolates share it, sealed (see shared.rs): with when it
356/// was made, so that one shared is reused no longer than one kept here.
357#[derive(Serialize, Deserialize)]
358struct SharedCredential {
359 remote: String,
360 token: String,
361 made: u64,
362}
363
364/// The shared cache's key for a credential: the store's key for the
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily365/// repository, the scope, and who it is for. (`cred2`: credentials kept
366/// before their lives differed by use are not read.)
367fn shared_key(key: &str, scope: Scope, using: Use) -> String {
368 format!("cred2:{key}:{}:{}", scope.as_str(), using.as_str())
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms369}
370
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily371/// A shared credential, if it was made less than its reuse window before
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms372/// `now`; with when it was made.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily373fn shared_credential(bytes: &[u8], now: u64, using: Use) -> Option<(GitAccess, u64)> {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms374 let kept: SharedCredential = serde_json::from_slice(bytes).ok()?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily375 (now.saturating_sub(kept.made) < using.reuse_ms()).then_some((
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms376 GitAccess {
377 remote: kept.remote,
378 token: kept.token,
379 },
380 kept.made,
381 ))
382}
383
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily384/// Credentials made a while ago, by repository, scope and use. Making one
385/// is a round trip to the store; reusing it saves that, and the store's
386/// lookup of the repository with it.
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms387#[derive(Default)]
388pub struct Credentials {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily389 kept: HashMap<(String, Scope, Use), (GitAccess, u64)>,
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms390}
391
392impl Credentials {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily393 /// One made for `key`, `scope` and `using` within its reuse window of `now`.
394 pub fn get(&self, key: &str, scope: Scope, using: Use, now: u64) -> Option<GitAccess> {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms395 self.kept
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily396 .get(&(key.to_owned(), scope, using))
397 .filter(|(_, made)| now.saturating_sub(*made) < using.reuse_ms())
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms398 .map(|(access, _)| access.clone())
399 }
400
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily401 pub fn keep(&mut self, key: &str, scope: Scope, using: Use, access: GitAccess, made: u64, now: u64) {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms402 // Expired ones go first, so the map stays as small as the isolate's
403 // recent repositories.
404 self.kept
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily405 .retain(|(_, _, kept_use), (_, at)| now.saturating_sub(*at) < kept_use.reuse_ms());
406 self.kept.insert((key.to_owned(), scope, using), (access, made));
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms407 }
408
409 pub fn forget(&mut self, key: &str) {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily410 self.kept.retain(|(kept, _, _), _| kept != key);
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms411 }
412}
413
414thread_local! {
415 static CREDENTIALS: RefCell<Credentials> = RefCell::new(Credentials::default());
416}
417
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily418/// A credential kept in this isolate, else one another isolate shared.
419async fn kept(shared: Option<&crate::shared::Shared>, key: &str, scope: Scope, using: Use) -> Option<(GitAccess, Kept)> {
420 let now = g1t_kit::now_ms();
421 if let Some(access) = CREDENTIALS.with(|kept| kept.borrow().get(key, scope, using, now)) {
422 return Some((access, Kept::Isolate));
423 }
424 let bytes = shared?.get(&shared_key(key, scope, using)).await?;
425 let (access, made) = shared_credential(&bytes, now, using)?;
426 CREDENTIALS.with(|kept| kept.borrow_mut().keep(key, scope, using, access.clone(), made, now));
427 Some((access, Kept::Shared))
428}
429
430/// Keeps a credential just made here, and shares it.
431async fn keep(shared: Option<&crate::shared::Shared>, key: &str, scope: Scope, using: Use, access: &GitAccess) {
432 let now = g1t_kit::now_ms();
433 CREDENTIALS.with(|kept| kept.borrow_mut().keep(key, scope, using, access.clone(), now, now));
434 if let Some(shared) = shared {
435 let value = SharedCredential {
436 remote: access.remote.clone(),
437 token: access.token.clone(),
438 made: now,
439 };
440 if let Ok(bytes) = serde_json::to_vec(&value) {
441 shared.put(&shared_key(key, scope, using), &bytes, using.reuse_ms() / 1000).await;
442 }
443 }
444}
445
Rust repos service with shipping; pull requests kept in the model446impl GitStore for ArtifactsStore {
447 type Repo = ArtifactsRepo;
448
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms449 async fn kept_access(&self, key: &str, scope: Scope) -> Option<(GitAccess, Kept)> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily450 kept(self.shared.as_deref(), key, scope, Use::Internal).await
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms451 }
452
453 async fn mint_access(&self, key: &str, scope: Scope) -> Result<GitAccess> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily454 let repo = self.open(key).await?;
455 let access = repo.mint(scope, Use::Internal).await?;
456 keep(self.shared.as_deref(), key, scope, Use::Internal, &access).await;
457 Ok(access)
458 }
459
460 async fn handout(&self, key: &str, scope: Scope) -> Result<GitAccess> {
461 if let Some((access, _)) = kept(self.shared.as_deref(), key, scope, Use::Handout).await {
462 return Ok(access);
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms463 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily464 let access = self.open(key).await?.mint(scope, Use::Handout).await?;
465 keep(self.shared.as_deref(), key, scope, Use::Handout, &access).await;
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms466 Ok(access)
467 }
468
469 async fn forget_access(&self, key: &str) {
470 CREDENTIALS.with(|kept| kept.borrow_mut().forget(key));
471 if let Some(shared) = &self.shared {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily472 let keys: Vec<String> = [Scope::Read, Scope::Write]
473 .into_iter()
474 .flat_map(|scope| [Use::Internal, Use::Handout].map(|using| shared_key(key, scope, using)))
475 .collect();
476 futures_util::future::join_all(keys.iter().map(|shared_key| shared.delete(shared_key))).await;
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms477 }
478 }
479
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily480 async fn create(&self, key: &str, description: Option<&str>, default_branch: &str) -> Result<()> {
481 let (namespace, name) = locate(key);
Rust repos service with shipping; pull requests kept in the model482 let options = js::to_js(&serde_json::json!({
483 "description": description,
484 "setDefaultBranch": default_branch,
485 }))?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily486 match invoke(&namespace, key, self.binding(&namespace)?, "create", &[name.as_str().into(), options], true).await {
Rust repos service with shipping; pull requests kept in the model487 // Left behind by an earlier failed attempt; adopt it.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily488 Err(failed) if !failed.is("ALREADY_EXISTS") => Err(failed.into()),
Rust repos service with shipping; pull requests kept in the model489 _ => Ok(()),
490 }
491 }
492
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look493 async fn delete(&self, key: &str) -> Result<()> {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms494 self.forget_access(key).await;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily495 let (namespace, name) = locate(key);
496 match invoke(&namespace, key, self.binding(&namespace)?, "delete", &[name.as_str().into()], true).await {
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look497 // Gone already: an earlier purge got this far.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily498 Err(failed) if !failed.is("NOT_FOUND") => Err(failed.into()),
Invite-only launch: sign in with GitHub, repository access and lifecycle, many emails, a new look499 _ => Ok(()),
500 }
501 }
502
Rust repos service with shipping; pull requests kept in the model503 async fn open(&self, key: &str) -> Result<ArtifactsRepo> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily504 let (namespace, name) = locate(key);
Rust repos service with shipping; pull requests kept in the model505 Ok(ArtifactsRepo {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for506 handle: RefCell::new(None),
507 binding: self.binding(&namespace)?.clone(),
Agents as a team: lifecycle, merge queue, billing and a new shell508 key: key.to_owned(),
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily509 name,
510 namespace,
511 shared: self.shared.clone(),
512 refs_version: None,
Rust repos service with shipping; pull requests kept in the model513 })
514 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily515
516 fn namespaces(&self) -> Vec<String> {
517 self.namespaces.iter().map(|namespace| namespace.name.clone()).collect()
518 }
519
520 fn default_namespace(&self) -> String {
521 DEFAULT_NS.with(|ns| ns.borrow().clone())
522 }
Rust repos service with shipping; pull requests kept in the model523}
524
525/// A handle to one Artifacts repository. It is an RPC stub, so it is
526/// released when dropped.
527pub struct ArtifactsRepo {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for528 /// The store's handle, asked for (`get`) on the first call that needs
529 /// the store: an answer from a cache, or a fetch over git with a kept
530 /// credential, never costs a `get`.
531 handle: RefCell<Option<JsValue>>,
532 /// The namespace's binding, for that `get`.
533 binding: JsValue,
Agents as a team: lifecycle, merge queue, billing and a new shell534 /// The repository's store key, which scopes its cached objects.
535 key: String,
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily536 /// Its name in its namespace.
537 name: String,
538 namespace: String,
539 shared: Option<Rc<crate::shared::Shared>>,
540 /// See [`GitRepo::at_refs_version`].
541 refs_version: Option<u64>,
Agents as a team: lifecycle, merge queue, billing and a new shell542}
543
544/// Where cached git objects live. Trees and blobs are named by their
545/// content, so a cached one is never stale; each is kept under its own
546/// repository's key, so a repository only ever finds its own objects.
547const OBJECT_CACHE: &str = "https://objects.g1t.internal/";
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily548/// Blobs and files larger than this are not cached.
Agents as a team: lifecycle, merge queue, billing and a new shell549const MAX_CACHED_BLOB: usize = 1024 * 1024;
550const OBJECT_MAX_AGE: &str = "public, max-age=31536000, immutable";
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily551/// Answers kept under a refs version: until the version moves, and no
552/// longer than this, which bounds how stale one can be should a change
553/// ever fail to move it.
554const VERSIONED_MAX_AGE: &str = "public, max-age=300";
Agents as a team: lifecycle, merge queue, billing and a new shell555
Fast pages, required checks on the branch, self-hosted runners, honest incidents556/// Whether a ref is a full commit hash (SHA-1 or SHA-256), whose history
557/// can be kept for good.
558pub fn is_commit_hash(git_ref: &str) -> bool {
559 (git_ref.len() == 40 || git_ref.len() == 64) && git_ref.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase())
560}
561
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily562/// Where a read is kept: under the object it names for good, under the
563/// refs version for a name that can move, or nowhere.
564#[derive(Debug, PartialEq, Eq)]
565pub enum CacheKey {
566 Forever(String),
567 Versioned(String),
568}
569
570/// The cache key for `log(git_ref, limit)`.
571pub fn log_key(git_ref: &str, limit: u32, version: Option<u64>) -> Option<CacheKey> {
572 if is_commit_hash(git_ref) {
573 return Some(CacheKey::Forever(format!("log/{git_ref}-{limit}")));
574 }
575 version.map(|version| CacheKey::Versioned(format!("vlog/{version}/{}-{limit}", g1t_secrets::sha256_hex(git_ref))))
576}
577
578/// The cache key for `read_file(git_ref, path)`.
579pub fn file_key(git_ref: &str, path: &str, version: Option<u64>) -> Option<CacheKey> {
580 let path = g1t_secrets::sha256_hex(path);
581 if is_commit_hash(git_ref) {
582 return Some(CacheKey::Forever(format!("file/{git_ref}/{path}")));
583 }
584 version.map(|version| CacheKey::Versioned(format!("vfile/{version}/{}/{path}", g1t_secrets::sha256_hex(git_ref))))
585}
586
587/// The cache key for the branch list.
588pub fn branches_key(version: Option<u64>) -> Option<CacheKey> {
589 version.map(|version| CacheKey::Versioned(format!("branches/{version}")))
590}
591
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for592/// Objects named by their content, kept in the isolate ahead of the Cache
593/// API, oldest out first past `MEMORY_CACHE_BYTES`. They never go stale,
594/// and each is under its repository's key.
595const MEMORY_CACHE_BYTES: usize = 16 * 1024 * 1024;
596
597#[derive(Default)]
598struct MemoryCache {
599 entries: HashMap<String, Rc<Vec<u8>>>,
600 order: std::collections::VecDeque<String>,
601 bytes: usize,
602}
603
604impl MemoryCache {
605 fn get(&self, url: &str) -> Option<Vec<u8>> {
606 self.entries.get(url).map(|bytes| bytes.as_ref().clone())
607 }
608
609 fn put(&mut self, url: String, bytes: &[u8]) {
610 if bytes.len() > MEMORY_CACHE_BYTES / 16 || self.entries.contains_key(&url) {
611 return;
612 }
613 self.bytes += bytes.len();
614 self.entries.insert(url.clone(), Rc::new(bytes.to_vec()));
615 self.order.push_back(url);
616 while self.bytes > MEMORY_CACHE_BYTES {
617 let Some(oldest) = self.order.pop_front() else { break };
618 if let Some(gone) = self.entries.remove(&oldest) {
619 self.bytes -= gone.len();
620 }
621 }
622 }
623}
624
625thread_local! {
626 static MEMORY: RefCell<MemoryCache> = RefCell::new(MemoryCache::default());
627}
628
Agents as a team: lifecycle, merge queue, billing and a new shell629impl ArtifactsRepo {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily630 fn cache_url(&self, path: &str) -> String {
631 format!("{OBJECT_CACHE}{}/{path}", self.key)
632 }
633
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for634 /// A kept answer: from the isolate for one kept for good, else the
635 /// Cache API. Each look is metered (`cache.memory_hit`, `cache.edge_hit`,
636 /// `cache.miss`), so the usage check shows which cache answers.
637 async fn cached_at(&self, path: &str, forever: bool) -> Option<Vec<u8>> {
638 let url = self.cache_url(path);
639 if forever && let Some(bytes) = MEMORY.with(|memory| memory.borrow().get(&url)) {
640 meters::record_bytes("cache.memory_hit", &self.key, 0, bytes.len() as u64);
641 return Some(bytes);
642 }
643 let found = match worker::Cache::default().get(url.clone(), false).await {
644 Ok(Some(mut response)) => response.bytes().await.ok(),
645 _ => None,
646 };
647 match &found {
648 Some(bytes) => {
649 meters::record_bytes("cache.edge_hit", &self.key, 0, bytes.len() as u64);
650 if forever {
651 MEMORY.with(|memory| memory.borrow_mut().put(url, bytes));
652 }
653 }
654 None => meters::record("cache.miss", &self.key, 0, 0),
655 }
656 found
Agents as a team: lifecycle, merge queue, billing and a new shell657 }
658
659 async fn cached(&self, kind: &str, hash: &str) -> Option<Vec<u8>> {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for660 self.cached_at(&format!("{kind}/{hash}"), true).await
Agents as a team: lifecycle, merge queue, billing and a new shell661 }
662
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily663 async fn keep_at(&self, path: &str, bytes: Vec<u8>, max_age: &str) {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for664 if max_age == OBJECT_MAX_AGE {
665 MEMORY.with(|memory| memory.borrow_mut().put(self.cache_url(path), &bytes));
666 }
Agents as a team: lifecycle, merge queue, billing and a new shell667 let Ok(mut response) = worker::Response::from_bytes(bytes) else {
668 return;
669 };
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily670 let _ = response.headers_mut().set("cache-control", max_age);
671 let _ = worker::Cache::default().put(self.cache_url(path), response).await;
Agents as a team: lifecycle, merge queue, billing and a new shell672 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily673
674 /// Keeps an object for next time. A failure only costs a later read.
675 async fn keep(&self, kind: &str, hash: &str, bytes: Vec<u8>) {
676 self.keep_at(&format!("{kind}/{hash}"), bytes, OBJECT_MAX_AGE).await;
677 }
678
679 async fn get_key(&self, key: &CacheKey) -> Option<Vec<u8>> {
680 match key {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for681 CacheKey::Forever(path) => self.cached_at(path, true).await,
682 CacheKey::Versioned(path) => self.cached_at(path, false).await,
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily683 }
684 }
685
686 async fn put_key(&self, key: &CacheKey, bytes: Vec<u8>) {
687 match key {
688 CacheKey::Forever(path) => self.keep_at(path, bytes, OBJECT_MAX_AGE).await,
689 CacheKey::Versioned(path) => self.keep_at(path, bytes, VERSIONED_MAX_AGE).await,
690 }
691 }
692
693 async fn call(&self, method: &str, args: &[JsValue], retry: bool) -> std::result::Result<JsValue, StoreError> {
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for694 let handle = self.handle().await?;
695 invoke(&self.namespace, &self.key, &handle, method, args, retry).await
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily696 }
697
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for698 /// The store's handle, asked for the first time it is needed.
699 async fn handle(&self) -> std::result::Result<JsValue, StoreError> {
700 if let Some(handle) = self.handle.borrow().as_ref() {
701 return Ok(handle.clone());
702 }
703 let handle = invoke(&self.namespace, &self.key, &self.binding, "get", &[self.name.as_str().into()], true).await?;
704 *self.handle.borrow_mut() = Some(handle.clone());
705 Ok(handle)
706 }
707
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily708 /// A new credential from the store. Its remote is worked out from the
709 /// key once this isolate knows where the namespace's remotes start;
710 /// until then the store is asked (`info()`) alongside the token.
711 pub async fn mint(&self, scope: Scope, using: Use) -> Result<GitAccess> {
712 let args = [scope.as_str().into(), using.ttl_seconds().into()];
713 let prefix = REMOTE_PREFIX.with(|prefixes| prefixes.borrow().get(&self.namespace).cloned());
714 if let Some(prefix) = prefix {
715 let token: RawToken = js::from_js(&self.call("createToken", &args, true).await?)?;
716 return Ok(GitAccess { remote: remote_from(&prefix, &self.name), token: token.plaintext });
717 }
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for718 self.handle().await?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily719 let (info, token) = futures_util::future::join(self.call("info", &[], true), self.call("createToken", &args, true)).await;
720 let info: RawInfo = js::from_js(&info?)?;
721 let token: RawToken = js::from_js(&token?)?;
722 match learn_prefix(&info.remote, &self.name) {
723 Some(prefix) => REMOTE_PREFIX.with(|prefixes| {
724 prefixes.borrow_mut().insert(self.namespace.clone(), prefix);
725 }),
726 None => worker::console_error!("the git store's remote {} does not end in {}.git", info.remote, self.name),
727 }
728 Ok(GitAccess { remote: info.remote, token: token.plaintext })
729 }
Rust repos service with shipping; pull requests kept in the model730}
731
732impl Drop for ArtifactsRepo {
733 fn drop(&mut self) {
734 let symbol = js::get(&worker::js_sys::global(), "Symbol");
735 let dispose = js::get(&symbol, "dispose");
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for736 let Some(handle) = self.handle.borrow_mut().take() else { return };
737 if let Ok(function) = Reflect::get(&handle, &dispose).and_then(|value| value.dyn_into::<worker::js_sys::Function>()) {
738 let _ = function.call0(&handle);
Rust repos service with shipping; pull requests kept in the model739 }
740 }
741}
742
743#[derive(Deserialize)]
744#[serde(rename_all = "camelCase")]
745struct RawCommit {
746 hash: String,
747 tree_hash: String,
748 message: String,
749 author: Signature,
750 parents: Vec<String>,
751 /// Seconds since the epoch.
752 authored_at: u64,
753}
754
755#[derive(Deserialize)]
756struct RawEntry {
757 name: String,
758 hash: String,
759 #[serde(rename = "type")]
760 kind: EntryKind,
761}
762
763#[derive(Deserialize)]
764struct RawInfo {
765 remote: String,
766}
767
768#[derive(Deserialize)]
769struct RawToken {
770 plaintext: String,
771}
772
773/// The bytes of a `Blob`, or `None` for null.
774async fn blob_bytes(blob: JsValue) -> Result<Option<Vec<u8>>> {
775 if blob.is_null() || blob.is_undefined() {
776 return Ok(None);
777 }
778 let buffer = js::call(&blob, "arrayBuffer", &[]).await?;
779 Ok(Some(Uint8Array::new(&buffer).to_vec()))
780}
781
782impl GitRepo for ArtifactsRepo {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily783 /// One made a while ago, here or in another isolate, else a new one.
Rust repos service with shipping; pull requests kept in the model784 async fn access(&self, scope: Scope) -> Result<GitAccess> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily785 if let Some((access, _)) = kept(self.shared.as_deref(), &self.key, scope, Use::Internal).await {
786 return Ok(access);
787 }
788 let access = self.mint(scope, Use::Internal).await?;
789 keep(self.shared.as_deref(), &self.key, scope, Use::Internal, &access).await;
790 Ok(access)
Rust repos service with shipping; pull requests kept in the model791 }
792
Pull requests from branches793 async fn branches(&self) -> Result<Vec<Branch>> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily794 let key = branches_key(self.refs_version);
795 if let Some(key) = &key
796 && let Some(bytes) = self.get_key(key).await
797 && let Ok(branches) = serde_json::from_slice::<Vec<Branch>>(&bytes)
798 {
799 return Ok(branches);
800 }
801 let branches = crate::refs::branches(&self.access(Scope::Read).await?).await?;
802 if let (Some(key), Ok(bytes)) = (&key, serde_json::to_vec(&branches)) {
803 self.put_key(key, bytes).await;
804 }
805 Ok(branches)
Pull requests from branches806 }
807
Rust repos service with shipping; pull requests kept in the model808 async fn log(&self, git_ref: &str, limit: u32) -> Result<Vec<Commit>> {
Fast pages, required checks on the branch, self-hosted runners, honest incidents809 // History from a commit never changes, so a log asked for by hash is
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily810 // kept like an object: walking it is a read per commit. One asked
811 // for by a branch's name is kept until the branch can have moved.
812 let key = log_key(git_ref, limit, self.refs_version);
813 if let Some(key) = &key
814 && let Some(bytes) = self.get_key(key).await
Fast pages, required checks on the branch, self-hosted runners, honest incidents815 && let Ok(commits) = serde_json::from_slice::<Vec<Commit>>(&bytes)
816 {
817 return Ok(commits);
818 }
Rust repos service with shipping; pull requests kept in the model819 let options = js::to_js(&serde_json::json!({ "ref": git_ref, "limit": limit }))?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily820 let raw: Vec<RawCommit> = js::from_js(&self.call("log", &[options], true).await?)?;
Fast pages, required checks on the branch, self-hosted runners, honest incidents821 let commits: Vec<Commit> = raw
Rust repos service with shipping; pull requests kept in the model822 .into_iter()
823 .map(|commit| Commit {
824 hash: commit.hash,
825 tree_hash: commit.tree_hash,
826 message: commit.message,
827 author: commit.author,
828 parents: commit.parents,
RFC 3339 timestamps in identity and repos829 authored_at: rfc3339(commit.authored_at * 1000),
Rust repos service with shipping; pull requests kept in the model830 })
Fast pages, required checks on the branch, self-hosted runners, honest incidents831 .collect();
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily832 // An unknown ref logs nothing; that is not kept, in case it arrives.
833 if !commits.is_empty()
Fast pages, required checks on the branch, self-hosted runners, honest incidents834 && let Ok(bytes) = serde_json::to_vec(&commits)
835 {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily836 if let Some(key) = &key {
837 self.put_key(key, bytes.clone()).await;
838 }
839 // The same history, by the commit the name led to.
840 if !is_commit_hash(git_ref)
841 && let Some(CacheKey::Forever(path)) = log_key(&commits[0].hash, limit, None)
842 {
843 self.keep_at(&path, bytes, OBJECT_MAX_AGE).await;
844 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents845 }
846 Ok(commits)
Rust repos service with shipping; pull requests kept in the model847 }
848
849 async fn parents(&self, commit_hash: &str) -> Result<Option<Vec<String>>> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily850 // A commit never changes.
851 if let Some(bytes) = self.cached("commit", commit_hash).await
852 && let Ok(parents) = serde_json::from_slice::<Vec<String>>(&bytes)
853 {
854 return Ok(Some(parents));
855 }
856 let commit: Option<RawCommit> = js::from_js(&self.call("readCommit", &[commit_hash.into()], true).await?)?;
857 let parents = commit.map(|commit| commit.parents);
858 if let Some(parents) = &parents
859 && let Ok(bytes) = serde_json::to_vec(parents)
860 {
861 self.keep("commit", commit_hash, bytes).await;
862 }
863 Ok(parents)
Rust repos service with shipping; pull requests kept in the model864 }
865
866 async fn read_tree(&self, tree_hash: &str) -> Result<Option<Vec<TreeEntry>>> {
Polish: phones, copy boxes, the plan page, the landing page, a real glide867 if let Some(bytes) = self.cached("tree", tree_hash).await
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily868 && let Ok(entries) = serde_json::from_slice::<Vec<TreeEntry>>(&bytes)
869 {
870 return Ok(Some(entries));
871 }
872 let entries: Option<Vec<RawEntry>> = js::from_js(&self.call("readTree", &[tree_hash.into()], true).await?)?;
Agents as a team: lifecycle, merge queue, billing and a new shell873 let entries: Option<Vec<TreeEntry>> = entries.map(|entries| {
Rust repos service with shipping; pull requests kept in the model874 entries
875 .into_iter()
876 .map(|entry| TreeEntry {
877 name: entry.name,
878 hash: entry.hash,
879 kind: entry.kind,
880 })
881 .collect()
Agents as a team: lifecycle, merge queue, billing and a new shell882 });
Polish: phones, copy boxes, the plan page, the landing page, a real glide883 if let Some(entries) = &entries
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily884 && let Ok(bytes) = serde_json::to_vec(entries)
885 {
886 self.keep("tree", tree_hash, bytes).await;
887 }
Agents as a team: lifecycle, merge queue, billing and a new shell888 Ok(entries)
Rust repos service with shipping; pull requests kept in the model889 }
890
891 async fn read_blob(&self, blob_hash: &str) -> Result<Option<Vec<u8>>> {
Agents as a team: lifecycle, merge queue, billing and a new shell892 if let Some(bytes) = self.cached("blob", blob_hash).await {
893 return Ok(Some(bytes));
894 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily895 let bytes = blob_bytes(self.call("readBlob", &[blob_hash.into()], true).await?).await?;
896 if let Some(bytes) = &bytes {
897 meters::record_bytes("binding.read_blob", &self.key, 0, bytes.len() as u64);
898 }
Agents as a team: lifecycle, merge queue, billing and a new shell899 if let Some(bytes) = bytes.as_ref().filter(|bytes| bytes.len() <= MAX_CACHED_BLOB) {
900 self.keep("blob", blob_hash, bytes.clone()).await;
901 }
902 Ok(bytes)
Rust repos service with shipping; pull requests kept in the model903 }
904
905 async fn read_file(&self, git_ref: &str, path: &str) -> Result<Option<Vec<u8>>> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily906 let key = file_key(git_ref, path, self.refs_version);
907 if let Some(key) = &key
908 && let Some(bytes) = self.get_key(key).await
909 {
910 return Ok(Some(bytes));
911 }
Rust repos service with shipping; pull requests kept in the model912 let args = js::to_js(&serde_json::json!({ "ref": git_ref, "path": path }))?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily913 let bytes = blob_bytes(self.call("readFile", &[args], true).await?).await?;
914 if let Some(bytes) = &bytes {
915 meters::record_bytes("binding.read_file", &self.key, 0, bytes.len() as u64);
916 }
917 if let (Some(key), Some(bytes)) = (&key, bytes.as_ref().filter(|bytes| bytes.len() <= MAX_CACHED_BLOB)) {
918 self.put_key(key, bytes.clone()).await;
919 }
920 Ok(bytes)
Rust repos service with shipping; pull requests kept in the model921 }
922
923 async fn fork(&self, target_key: &str) -> Result<()> {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily924 let (namespace, name) = locate(target_key);
925 if namespace != self.namespace {
926 return Err(worker::Error::RustError(format!(
927 "a fork stays in its repository's namespace: {target_key} is not in {}",
928 self.namespace
929 )));
930 }
Rust repos service with shipping; pull requests kept in the model931 let options = js::to_js(&serde_json::json!({ "defaultBranchOnly": true }))?;
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily932 match self.call("fork", &[name.as_str().into(), options], false).await {
933 Err(failed) if !failed.is("ALREADY_EXISTS") => Err(failed.into()),
Rust repos service with shipping; pull requests kept in the model934 _ => Ok(()),
935 }
936 }
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily937
938 fn at_refs_version(&mut self, version: Option<u64>) {
939 self.refs_version = version;
940 }
Rust repos service with shipping; pull requests kept in the model941}
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms942
943#[cfg(test)]
944mod tests {
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily945 use super::*;
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms946
947 fn access(token: &str) -> GitAccess {
948 GitAccess {
949 remote: "https://store.example/acme--rocket.git".to_owned(),
950 token: token.to_owned(),
951 }
952 }
953
954 #[test]
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily955 fn internal_credentials_live_an_hour_and_are_reused_for_fifty_minutes() {
956 assert_eq!(Use::Internal.ttl_seconds(), 3_600);
957 assert_eq!(Use::Internal.reuse_ms(), 50 * 60 * 1000);
958 // Handed out: five minutes, reused three, so at least two are left.
959 assert_eq!(Use::Handout.ttl_seconds(), 300);
960 assert_eq!(Use::Handout.reuse_ms(), 180_000);
961 assert_eq!(CREDENTIAL_LIFE_MS, 300_000);
962 for using in [Use::Internal, Use::Handout] {
963 assert!(u64::from(using.ttl_seconds()) * 1000 - using.reuse_ms() >= 120_000);
964 }
965 }
966
967 #[test]
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms968 fn a_credential_is_reused_only_while_it_has_time_left() {
969 let mut kept = Credentials::default();
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily970 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
971 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000).unwrap().token, "r1");
972 let last = 1_000 + INTERNAL_REUSE_MS - 1;
973 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Internal, last).unwrap().token, "r1");
974 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, last + 1).is_none());
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms975 }
976
977 #[test]
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily978 fn a_credential_is_kept_for_its_own_repository_scope_and_use() {
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms979 let mut kept = Credentials::default();
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily980 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms981 // A read credential never stands in for a write one.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily982 assert!(kept.get("acme--rocket", Scope::Write, Use::Internal, 1_000).is_none());
983 assert!(kept.get("acme--booster", Scope::Read, Use::Internal, 1_000).is_none());
984 // An hour-long credential is never handed out.
985 assert!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000).is_none());
986 kept.keep("acme--rocket", Scope::Read, Use::Handout, access("h1"), 1_000, 1_000);
987 assert_eq!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000).unwrap().token, "h1");
988 assert!(kept.get("acme--rocket", Scope::Read, Use::Handout, 1_000 + HANDOUT_REUSE_MS).is_none());
989 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000 + HANDOUT_REUSE_MS).is_some());
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms990 }
991
992 #[test]
993 fn a_shared_credential_is_reused_only_in_its_own_window() {
994 let value = serde_json::to_vec(&SharedCredential {
995 remote: "https://store.example/acme--rocket.git".to_owned(),
996 token: "r1".to_owned(),
997 made: 10_000,
998 })
999 .unwrap();
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1000 let (access, made) = shared_credential(&value, 10_000 + INTERNAL_REUSE_MS - 1, Use::Internal).unwrap();
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1001 assert_eq!(access.token, "r1");
1002 // Kept here only for what is left of its window, not a new one.
1003 assert_eq!(made, 10_000);
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1004 assert!(shared_credential(&value, 10_000 + INTERNAL_REUSE_MS, Use::Internal).is_none());
1005 assert!(shared_credential(&value, 10_000 + HANDOUT_REUSE_MS, Use::Handout).is_none());
1006 assert!(shared_credential(b"not json", 10_000, Use::Internal).is_none());
1007 // Each repository, scope and use has its own key, and none is read
1008 // from before uses differed.
1009 assert_eq!(shared_key("acme--rocket", Scope::Read, Use::Internal), "cred2:acme--rocket:read:internal");
1010 assert_ne!(shared_key("acme--rocket", Scope::Read, Use::Internal), shared_key("acme--rocket", Scope::Write, Use::Internal));
1011 assert_ne!(shared_key("acme--rocket", Scope::Read, Use::Internal), shared_key("acme--rocket", Scope::Read, Use::Handout));
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1012 }
1013
1014 #[test]
1015 fn a_shared_credential_kept_here_expires_with_the_original() {
1016 let mut kept = Credentials::default();
1017 // Made at 1_000 elsewhere, found here at 100_000.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1018 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 100_000);
1019 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 100_000).is_some());
1020 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000 + INTERNAL_REUSE_MS).is_none());
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1021 }
1022
1023 #[test]
1024 fn a_turned_down_credential_is_forgotten_and_old_ones_are_dropped() {
1025 let mut kept = Credentials::default();
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1026 kept.keep("acme--rocket", Scope::Read, Use::Internal, access("r1"), 1_000, 1_000);
1027 kept.keep("acme--rocket", Scope::Write, Use::Handout, access("w1"), 1_000, 1_000);
1028 kept.keep("acme--booster", Scope::Read, Use::Internal, access("b1"), 1_000, 1_000);
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1029 kept.forget("acme--rocket");
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1030 assert!(kept.get("acme--rocket", Scope::Read, Use::Internal, 1_000).is_none());
1031 assert!(kept.get("acme--rocket", Scope::Write, Use::Handout, 1_000).is_none());
1032 assert!(kept.get("acme--booster", Scope::Read, Use::Internal, 1_000).is_some());
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1033 // Keeping another later drops the expired one from the map.
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1034 let later = 1_000 + INTERNAL_REUSE_MS;
1035 kept.keep("acme--other", Scope::Read, Use::Internal, access("o1"), later, later);
Mission control shows where you are needed and what agents landed without you; git answers in about 200ms1036 assert_eq!(kept.kept.len(), 1);
1037 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents1038
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1039 #[test]
1040 fn a_remote_is_worked_out_from_where_its_namespace_starts() {
1041 // As https://developers.cloudflare.com/artifacts/api/git-protocol/ documents.
1042 let remote = "https://1e6f2cffa3f445920836e8ebe446bb58.artifacts.cloudflare.net/git/g1t/acme--rocket.git";
1043 let prefix = learn_prefix(remote, "acme--rocket").unwrap();
1044 assert_eq!(prefix, "https://1e6f2cffa3f445920836e8ebe446bb58.artifacts.cloudflare.net/git/g1t/");
1045 assert_eq!(remote_from(&prefix, "pulls--pul_1"), prefix.clone() + "pulls--pul_1.git");
1046 assert_eq!(remote_from(&prefix, "acme--rocket"), remote);
1047 // A remote that does not end in the name teaches nothing.
1048 assert_eq!(learn_prefix(remote, "rocket"), None);
1049 assert_eq!(learn_prefix("https://x/acme--rocket", "acme--rocket"), None);
1050 // And back: a remote names its key.
1051 assert_eq!(key_from_remote(remote).as_deref(), Some("acme--rocket"));
1052 assert_eq!(
1053 key_from_remote("https://a.artifacts.cloudflare.net/git/g1t-us-1/acme--rocket.git").as_deref(),
1054 Some("g1t-us-1/acme--rocket")
1055 );
1056 assert_eq!(locate("g1t-us-1/acme--rocket"), ("g1t-us-1".to_owned(), "acme--rocket".to_owned()));
1057 assert_eq!(locate("acme--rocket"), ("g1t".to_owned(), "acme--rocket".to_owned()));
1058 }
1059
1060 #[test]
Fewer Artifacts reads: the store is asked for a handle only when needed, objects are kept in the isolate, and issue events read no workflows nobody listens for1061 fn the_memory_cache_drops_its_oldest_past_its_budget() {
1062 let mut cache = MemoryCache::default();
1063 let chunk = vec![7u8; MEMORY_CACHE_BYTES / 16];
1064 for n in 0..17 {
1065 cache.put(format!("k{n}"), &chunk);
1066 }
1067 // Sixteen chunks fit; the seventeenth pushed the first out.
1068 assert!(cache.get("k0").is_none());
1069 assert_eq!(cache.get("k16").map(|b| b.len()), Some(chunk.len()));
1070 assert!(cache.bytes <= MEMORY_CACHE_BYTES);
1071 // Too large to keep at all.
1072 cache.put("big".into(), &vec![0u8; MEMORY_CACHE_BYTES / 16 + 1]);
1073 assert!(cache.get("big").is_none());
1074 }
1075
1076 #[test]
Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1077 fn binding_methods_have_snake_case_meters() {
1078 assert_eq!(meter_of("createToken"), "binding.create_token");
1079 assert_eq!(meter_of("readBlob"), "binding.read_blob");
1080 assert_eq!(meter_of("get"), "binding.get");
1081 }
1082
1083 #[test]
1084 fn reads_by_name_are_kept_under_the_refs_version_and_by_hash_for_good() {
1085 let hash = "a".repeat(40);
1086 assert_eq!(log_key(&hash, 1, Some(3)), Some(CacheKey::Forever(format!("log/{hash}-1"))));
1087 assert_eq!(log_key(&hash, 1, None), Some(CacheKey::Forever(format!("log/{hash}-1"))));
1088 // A branch is kept only when the version is known.
1089 assert_eq!(log_key("main", 1, None), None);
1090 let v3 = log_key("main", 1, Some(3)).unwrap();
1091 assert!(matches!(&v3, CacheKey::Versioned(path) if path.starts_with("vlog/3/")));
1092 // A push moves the version and leaves the old answer behind.
1093 assert_ne!(Some(v3), log_key("main", 1, Some(4)));
1094 assert_ne!(log_key("main", 1, Some(3)), log_key("main", 50, Some(3)));
1095 assert_ne!(log_key("main", 1, Some(3)), log_key("dev", 1, Some(3)));
1096 // Odd branch names make a usable address.
1097 assert!(matches!(log_key("fix/#1 %20", 1, Some(3)), Some(CacheKey::Versioned(path)) if !path.contains('#') && !path.contains(' ')));
1098 assert_eq!(branches_key(None), None);
1099 assert_ne!(branches_key(Some(1)), branches_key(Some(2)));
1100 assert!(matches!(file_key(&hash, "src/main.rs", None), Some(CacheKey::Forever(_))));
1101 assert_eq!(file_key("main", "src/main.rs", None), None);
1102 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "b", Some(1)));
1103 assert_ne!(file_key("main", "a", Some(1)), file_key("main", "a", Some(2)));
1104 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents1105
1106 #[test]
1107 fn only_full_lowercase_hashes_are_kept() {
1108 assert!(is_commit_hash(&"a".repeat(40)));
1109 assert!(is_commit_hash(&"0123456789abcdef".repeat(4)));
1110 assert!(!is_commit_hash("main"));
1111 assert!(!is_commit_hash(&"A".repeat(40)));
1112 assert!(!is_commit_hash(&"a".repeat(39)));
1113 }
1114}