Sign in

flagon-io/g1t

public

g1t/services/repos/src/fallback.rs

358 lines16,278 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'worktree-agent-a2013627e5ea4ab13'1//! The cold fallback for the git store (docs/ARTIFACTS.md, R12).
2//!
3//! When Artifacts is down for a namespace, the repos service can serve that
4//! namespace from a self-hosted git store instead
5//! (`deploy/self-host/gitstore`), rebuilt from the nightly backups
6//! (`scripts/ops/restore-to-gitstore.mjs`). It is switched by
7//! configuration, not code:
8//!
9//! - `GIT_FALLBACK_URL`: where the git store answers, `https://...`.
10//! - `GIT_FALLBACK_SECRET` (a secret): the store's API secret.
11//! - `GIT_FALLBACK_NAMESPACES`: the namespaces served from it, comma
12//! separated, or `*` for all. Unset or empty: none, and nothing changes.
13//! - `GIT_FALLBACK_WRITES`: `refuse` (the default) or `allow`. While
14//! refused, a switched namespace is read-only: clones, fetches and pages
15//! work; pushes, merges and new repositories wait, told so in words.
16//!
17//! The store keeps each namespace's repositories under its own directory,
18//! so the key `g1t-us-1/acme--rocket` is `g1t-us-1/acme--rocket` there and
19//! `acme--rocket` (the default namespace) is `g1t/acme--rocket`. Its remotes
20//! read `<url>/git/<namespace>/<name>.git`, the shape Artifacts uses.
21//!
22//! This module is the plain part: the settings, and how each call the
23//! service makes on an Artifacts binding becomes a request to the store's
24//! API and back. store.rs makes the requests.
25
26use serde_json::{Value, json};
27
28/// The fallback store's settings, when it is configured at all.
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub struct Settings {
31 /// Where the store answers, without a trailing slash.
32 pub url: String,
33 pub secret: String,
34 pub switched: Switched,
35 /// Whether a switched namespace takes writes.
36 pub writes: bool,
37}
38
39/// Which namespaces are served from the fallback store.
40#[derive(Clone, Debug, PartialEq, Eq)]
41pub enum Switched {
42 All,
43 Some(Vec<String>),
44}
45
46impl Settings {
47 /// The settings from the variables, or `None` when there is no store
48 /// to fall back to (no address, or no secret).
49 pub fn from_vars(url: Option<&str>, secret: Option<&str>, namespaces: Option<&str>, writes: Option<&str>) -> Option<Self> {
50 let url = url.map(str::trim).filter(|url| url.starts_with("https://") || url.starts_with("http://"))?;
51 let secret = secret.map(str::trim).filter(|secret| secret.len() >= 16)?;
52 let namespaces = namespaces.unwrap_or_default().trim();
53 let switched = if namespaces == "*" {
54 Switched::All
55 } else {
56 Switched::Some(
57 namespaces
58 .split(',')
59 .map(str::trim)
60 .filter(|name| crate::shards::valid_namespace(name))
61 .map(str::to_owned)
62 .collect(),
63 )
64 };
65 Some(Settings {
66 url: url.trim_end_matches('/').to_owned(),
67 secret: secret.to_owned(),
68 switched,
69 writes: writes.is_some_and(|value| value.trim().eq_ignore_ascii_case("allow")),
70 })
71 }
72
73 /// Whether `namespace` is served from the fallback store now.
74 pub fn serves(&self, namespace: &str) -> bool {
75 match &self.switched {
76 Switched::All => true,
77 Switched::Some(names) => names.iter().any(|name| name == namespace),
78 }
79 }
80
81 /// The remote git uses for `name` in `namespace`.
82 pub fn remote(&self, namespace: &str, name: &str) -> String {
83 format!("{}/git/{}.git", self.url, store_key(namespace, name))
84 }
85}
86
87/// A repository's key in the fallback store: always with its namespace.
88pub fn store_key(namespace: &str, name: &str) -> String {
89 format!("{namespace}/{name}")
90}
91
92/// Percent-encodes one path segment.
93fn segment(text: &str) -> String {
94 text.bytes()
95 .map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })
96 .collect()
97}
98
99/// Percent-encodes a query value.
100fn query(pairs: &[(&str, String)]) -> String {
101 pairs
102 .iter()
103 .map(|(key, value)| format!("{key}={}", segment(value)))
104 .collect::<Vec<_>>()
105 .join("&")
106}
107
108/// How a call on the binding is asked of the store's API.
109#[derive(Debug, PartialEq)]
110pub struct Route {
111 pub method: &'static str,
112 /// Below the store's address: `/api/repos/...`.
113 pub path: String,
114 pub body: Option<Value>,
115 /// What the answer is: JSON, or bytes (a blob or a file).
116 pub bytes: bool,
117}
118
119/// A call the fallback cannot make, as the binding would have thrown it.
120#[derive(Debug, PartialEq)]
121pub struct Refused {
122 pub code: &'static str,
123 pub message: String,
124}
125
126fn arg<'a>(args: &'a [Value], at: usize) -> &'a Value {
127 args.get(at).unwrap_or(&Value::Null)
128}
129
130fn text(args: &[Value], at: usize) -> Result<String, Refused> {
131 arg(args, at).as_str().map(str::to_owned).ok_or_else(|| Refused {
132 code: "INVALID_ARGUMENT",
133 message: format!("argument {at} should be text"),
134 })
135}
136
137/// The request for `method(args)`: on the namespace when `repo` is `None`
138/// (`create`, `get`, `delete`), else on the repository named `repo` there.
139pub fn route(namespace: &str, repo: Option<&str>, method: &str, args: &[Value]) -> Result<Route, Refused> {
140 let json_route = |method: &'static str, path: String, body: Option<Value>| Route { method, path, body, bytes: false };
141 let Some(repo) = repo else {
142 let name = text(args, 0)?;
143 let key = store_key(namespace, &name);
144 return match method {
145 "create" => {
146 let options = arg(args, 1);
147 Ok(json_route(
148 "POST",
149 "/api/repos".to_owned(),
150 Some(json!({
151 "name": key,
152 "description": options.get("description").cloned().unwrap_or(Value::Null),
153 "defaultBranch": options.get("setDefaultBranch").cloned().unwrap_or(Value::Null),
154 })),
155 ))
156 }
157 "get" => Ok(json_route("GET", format!("/api/repos/{}", segment(&key)), None)),
158 "delete" => Ok(json_route("DELETE", format!("/api/repos/{}", segment(&key)), None)),
159 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
160 };
161 };
162 let base = format!("/api/repos/{}", segment(&store_key(namespace, repo)));
163 match method {
164 "info" => Ok(json_route("GET", base, None)),
165 "createToken" => Ok(json_route(
166 "POST",
167 format!("{base}/tokens"),
168 Some(json!({ "scope": arg(args, 0).as_str().unwrap_or("write"), "ttl": arg(args, 1).as_u64().unwrap_or(3_600) })),
169 )),
170 "log" => {
171 let options = arg(args, 0);
172 let mut pairs = Vec::new();
173 for name in ["ref", "limit", "offset"] {
174 match options.get(name) {
175 Some(Value::String(value)) => pairs.push((name, value.clone())),
176 Some(Value::Number(value)) => pairs.push((name, value.to_string())),
177 _ => {}
178 }
179 }
180 Ok(json_route("GET", format!("{base}/log?{}", query(&pairs)), None))
181 }
182 "readCommit" => Ok(json_route("GET", format!("{base}/commits/{}", segment(&text(args, 0)?)), None)),
183 "readTree" => Ok(json_route("GET", format!("{base}/trees/{}", segment(&text(args, 0)?)), None)),
184 "readBlob" => Ok(Route { method: "GET", path: format!("{base}/blobs/{}", segment(&text(args, 0)?)), body: None, bytes: true }),
185 "readFile" => {
186 let options = arg(args, 0);
187 let field = |name: &str| options.get(name).and_then(Value::as_str).unwrap_or_default().to_owned();
188 Ok(Route {
189 method: "GET",
190 path: format!("{base}/file?{}", query(&[("ref", field("ref")), ("path", field("path"))])),
191 body: None,
192 bytes: true,
193 })
194 }
195 "fork" => {
196 let target = text(args, 0)?;
197 let options = arg(args, 1);
198 Ok(json_route(
199 "POST",
200 format!("{base}/fork"),
201 Some(json!({
202 "name": store_key(namespace, &target),
203 "defaultBranchOnly": options.get("defaultBranchOnly").and_then(Value::as_bool).unwrap_or(true),
204 })),
205 ))
206 }
207 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
208 }
209}
210
211/// What an answer from the store means for the call that asked.
212#[derive(Debug, PartialEq)]
213pub enum Answer {
214 /// JSON, as the binding would have returned it.
215 Json(Value),
216 /// A blob or a file's bytes.
217 Bytes(Vec<u8>),
218 /// Nothing there: a blob or file not found, as the binding's `null`.
219 Null,
220 /// An error, with the binding's code; `None` for one that may pass.
221 Error { code: Option<String>, message: String },
222}
223
224/// Reads an answer: `status` and `body` from the store, for `route`.
225pub fn answer(route: &Route, status: u16, body: Vec<u8>) -> Answer {
226 if (200..300).contains(&status) {
227 if route.bytes {
228 return Answer::Bytes(body);
229 }
230 return match serde_json::from_slice::<Value>(&body) {
231 Ok(Value::Null) => Answer::Null,
232 Ok(value) => Answer::Json(value),
233 Err(_) => Answer::Error { code: None, message: "the fallback store sent something that is not JSON".to_owned() },
234 };
235 }
236 if status == 404 && route.bytes {
237 return Answer::Null;
238 }
239 let said: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
240 let message = said.get("message").and_then(Value::as_str).map_or_else(|| format!("the fallback store answered {status}"), str::to_owned);
241 // 5xx may pass: like the binding's INTERNAL_ERROR.
242 let code = if status >= 500 {
243 Some("INTERNAL_ERROR".to_owned())
244 } else {
245 Some(said.get("code").and_then(Value::as_str).unwrap_or(if status == 404 { "NOT_FOUND" } else { "INVALID_ARGUMENT" }).to_owned())
246 };
247 Answer::Error { code, message }
248}
249
250/// Whether a call writes: refused while a switched namespace is read-only.
251pub fn writes(repo: Option<&str>, method: &str, args: &[Value]) -> bool {
252 match (repo, method) {
253 (None, "create" | "delete") => true,
254 (Some(_), "fork") => true,
255 (Some(_), "createToken") => arg(args, 0).as_str().unwrap_or("write") == "write",
256 _ => false,
257 }
258}
259
260#[cfg(test)]
261mod tests {
262 use super::*;
263
264 fn settings(namespaces: &str) -> Settings {
265 Settings::from_vars(Some("https://gitstore.example/"), Some("0123456789abcdef"), Some(namespaces), None).unwrap()
266 }
267
268 #[test]
269 fn nothing_is_switched_unless_the_store_and_namespaces_are_named() {
270 assert_eq!(Settings::from_vars(None, Some("0123456789abcdef"), Some("*"), None), None);
271 assert_eq!(Settings::from_vars(Some("https://x"), None, Some("*"), None), None);
272 // A secret too short to be one.
273 assert_eq!(Settings::from_vars(Some("https://x"), Some("short"), Some("*"), None), None);
274 assert_eq!(Settings::from_vars(Some("ftp://x"), Some("0123456789abcdef"), Some("*"), None), None);
275 let none = settings("");
276 assert!(!none.serves("g1t"));
277 let some = settings("g1t, g1t-us-1,bad name");
278 assert!(some.serves("g1t") && some.serves("g1t-us-1") && !some.serves("g1t-eu"));
279 assert!(settings("*").serves("anything"));
280 // Read-only unless told otherwise.
281 assert!(!some.writes);
282 let writable = Settings::from_vars(Some("https://x"), Some("0123456789abcdef"), Some("*"), Some("Allow")).unwrap();
283 assert!(writable.writes);
284 assert_eq!(some.url, "https://gitstore.example");
285 assert_eq!(some.remote("g1t", "acme--rocket"), "https://gitstore.example/git/g1t/acme--rocket.git");
286 }
287
288 #[test]
289 fn a_remote_names_its_key_as_an_artifacts_remote_does() {
290 let remote = settings("*").remote("g1t-us-1", "acme--rocket");
291 // store.rs reads keys back from remotes by their last two segments.
292 assert!(remote.ends_with("/g1t-us-1/acme--rocket.git"));
293 }
294
295 #[test]
296 fn calls_on_the_namespace_become_requests_for_its_key() {
297 let create = route("g1t", None, "create", &[json!("acme--rocket"), json!({ "description": "d", "setDefaultBranch": "trunk" })]).unwrap();
298 assert_eq!(create.method, "POST");
299 assert_eq!(create.path, "/api/repos");
300 assert_eq!(create.body.unwrap(), json!({ "name": "g1t/acme--rocket", "description": "d", "defaultBranch": "trunk" }));
301 let get = route("g1t", None, "get", &[json!("acme--rocket")]).unwrap();
302 assert_eq!((get.method, get.path.as_str()), ("GET", "/api/repos/g1t%2Facme--rocket"));
303 assert_eq!(route("g1t", None, "delete", &[json!("x1")]).unwrap().method, "DELETE");
304 assert_eq!(route("g1t", None, "list", &[json!("x1")]).unwrap_err().code, "NOT_SUPPORTED");
305 assert_eq!(route("g1t", None, "get", &[]).unwrap_err().code, "INVALID_ARGUMENT");
306 }
307
308 #[test]
309 fn calls_on_a_repository_become_requests_below_it() {
310 let base = "/api/repos/g1t-us-1%2Fpulls--pul_1";
311 let at = |method: &str, args: &[Value]| route("g1t-us-1", Some("pulls--pul_1"), method, args).unwrap();
312 assert_eq!(at("info", &[]).path, base);
313 let token = at("createToken", &[json!("read"), json!(300)]);
314 assert_eq!(token.body.unwrap(), json!({ "scope": "read", "ttl": 300 }));
315 assert_eq!(at("log", &[json!({ "ref": "fix/#1", "limit": 20 })]).path, format!("{base}/log?ref=fix%2F%231&limit=20"));
316 assert_eq!(at("readCommit", &[json!("abc")]).path, format!("{base}/commits/abc"));
317 assert_eq!(at("readTree", &[json!("abc")]).path, format!("{base}/trees/abc"));
318 let blob = at("readBlob", &[json!("abc")]);
319 assert!(blob.bytes);
320 let file = at("readFile", &[json!({ "ref": "main", "path": "src/a b.rs" })]);
321 assert_eq!(file.path, format!("{base}/file?ref=main&path=src%2Fa%20b.rs"));
322 assert!(file.bytes);
323 let fork = at("fork", &[json!("pulls--pul_2"), json!({ "defaultBranchOnly": true })]);
324 assert_eq!(fork.body.unwrap(), json!({ "name": "g1t-us-1/pulls--pul_2", "defaultBranchOnly": true }));
325 }
326
327 #[test]
328 fn answers_read_as_the_binding_would_have_returned_them() {
329 let json_route = route("g1t", Some("r"), "readTree", &[json!("a")]).unwrap();
330 let blob = route("g1t", Some("r"), "readBlob", &[json!("a")]).unwrap();
331 assert_eq!(answer(&json_route, 200, b"[]".to_vec()), Answer::Json(json!([])));
332 assert_eq!(answer(&json_route, 200, b"null".to_vec()), Answer::Null);
333 assert_eq!(answer(&blob, 200, b"hi".to_vec()), Answer::Bytes(b"hi".to_vec()));
334 assert_eq!(answer(&blob, 404, b"{}".to_vec()), Answer::Null);
335 assert_eq!(
336 answer(&json_route, 404, br#"{"code":"NOT_FOUND","message":"no repository g1t/r"}"#.to_vec()),
337 Answer::Error { code: Some("NOT_FOUND".into()), message: "no repository g1t/r".into() }
338 );
339 assert_eq!(
340 answer(&json_route, 409, br#"{"code":"ALREADY_EXISTS","message":"x"}"#.to_vec()),
341 Answer::Error { code: Some("ALREADY_EXISTS".into()), message: "x".into() }
342 );
343 // A 5xx may pass, as the binding's INTERNAL_ERROR.
344 assert!(matches!(answer(&json_route, 502, Vec::new()), Answer::Error { code: Some(code), .. } if code == "INTERNAL_ERROR"));
345 assert!(matches!(answer(&json_route, 200, b"<html>".to_vec()), Answer::Error { code: None, .. }));
346 }
347
348 #[test]
349 fn writes_are_told_apart_from_reads() {
350 assert!(writes(None, "create", &[json!("x")]));
351 assert!(writes(None, "delete", &[json!("x")]));
352 assert!(!writes(None, "get", &[json!("x")]));
353 assert!(writes(Some("r"), "fork", &[json!("y")]));
354 assert!(writes(Some("r"), "createToken", &[json!("write"), json!(60)]));
355 assert!(!writes(Some("r"), "createToken", &[json!("read"), json!(60)]));
356 assert!(!writes(Some("r"), "readBlob", &[json!("a")]));
357 }
358}

This file's history is long; its oldest lines are credited to the oldest commit read.