Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/repos/src/fallback.rs

358 lines16,278 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Repositories shard across git store namespaces, move between them, and can keep to the EU; a namespace can be served read-only from the self-hosted git store, rebuilt from the nightly backups (#20, #25)1//! The cold fallback for the git store (docs/ARTIFACTS.md, R12).
2//!
3//! When Artifacts is down for a namespace, the repos service can serve that
4//! namespace from a self-hosted git store instead
5//! (`deploy/self-host/gitstore`), rebuilt from the nightly backups
6//! (`scripts/ops/restore-to-gitstore.mjs`). It is switched by
7//! configuration, not code:
8//!
9//! - `GIT_FALLBACK_URL`: where the git store answers, `https://...`.
10//! - `GIT_FALLBACK_SECRET` (a secret): the store's API secret.
11//! - `GIT_FALLBACK_NAMESPACES`: the namespaces served from it, comma
12//! separated, or `*` for all. Unset or empty: none, and nothing changes.
13//! - `GIT_FALLBACK_WRITES`: `refuse` (the default) or `allow`. While
14//! refused, a switched namespace is read-only: clones, fetches and pages
15//! work; pushes, merges and new repositories wait, told so in words.
16//!
17//! The store keeps each namespace's repositories under its own directory,
18//! so the key `g1t-us-1/acme--rocket` is `g1t-us-1/acme--rocket` there and
19//! `acme--rocket` (the default namespace) is `g1t/acme--rocket`. Its remotes
20//! read `<url>/git/<namespace>/<name>.git`, the shape Artifacts uses.
21//!
22//! This module is the plain part: the settings, and how each call the
23//! service makes on an Artifacts binding becomes a request to the store's
24//! API and back. store.rs makes the requests.
25
26use serde_json::{Value, json};
27
28/// The fallback store's settings, when it is configured at all.
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub struct Settings {
31 /// Where the store answers, without a trailing slash.
32 pub url: String,
33 pub secret: String,
34 pub switched: Switched,
35 /// Whether a switched namespace takes writes.
36 pub writes: bool,
37}
38
39/// Which namespaces are served from the fallback store.
40#[derive(Clone, Debug, PartialEq, Eq)]
41pub enum Switched {
42 All,
43 Some(Vec<String>),
44}
45
46impl Settings {
47 /// The settings from the variables, or `None` when there is no store
48 /// to fall back to (no address, or no secret).
49 pub fn from_vars(url: Option<&str>, secret: Option<&str>, namespaces: Option<&str>, writes: Option<&str>) -> Option<Self> {
50 let url = url.map(str::trim).filter(|url| url.starts_with("https://") || url.starts_with("http://"))?;
51 let secret = secret.map(str::trim).filter(|secret| secret.len() >= 16)?;
52 let namespaces = namespaces.unwrap_or_default().trim();
53 let switched = if namespaces == "*" {
54 Switched::All
55 } else {
56 Switched::Some(
57 namespaces
58 .split(',')
59 .map(str::trim)
60 .filter(|name| crate::shards::valid_namespace(name))
61 .map(str::to_owned)
62 .collect(),
63 )
64 };
65 Some(Settings {
66 url: url.trim_end_matches('/').to_owned(),
67 secret: secret.to_owned(),
68 switched,
69 writes: writes.is_some_and(|value| value.trim().eq_ignore_ascii_case("allow")),
70 })
71 }
72
73 /// Whether `namespace` is served from the fallback store now.
74 pub fn serves(&self, namespace: &str) -> bool {
75 match &self.switched {
76 Switched::All => true,
77 Switched::Some(names) => names.iter().any(|name| name == namespace),
78 }
79 }
80
81 /// The remote git uses for `name` in `namespace`.
82 pub fn remote(&self, namespace: &str, name: &str) -> String {
83 format!("{}/git/{}.git", self.url, store_key(namespace, name))
84 }
85}
86
87/// A repository's key in the fallback store: always with its namespace.
88pub fn store_key(namespace: &str, name: &str) -> String {
89 format!("{namespace}/{name}")
90}
91
92/// Percent-encodes one path segment.
93fn segment(text: &str) -> String {
94 text.bytes()
95 .map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })
96 .collect()
97}
98
99/// Percent-encodes a query value.
100fn query(pairs: &[(&str, String)]) -> String {
101 pairs
102 .iter()
103 .map(|(key, value)| format!("{key}={}", segment(value)))
104 .collect::<Vec<_>>()
105 .join("&")
106}
107
108/// How a call on the binding is asked of the store's API.
109#[derive(Debug, PartialEq)]
110pub struct Route {
111 pub method: &'static str,
112 /// Below the store's address: `/api/repos/...`.
113 pub path: String,
114 pub body: Option<Value>,
115 /// What the answer is: JSON, or bytes (a blob or a file).
116 pub bytes: bool,
117}
118
119/// A call the fallback cannot make, as the binding would have thrown it.
120#[derive(Debug, PartialEq)]
121pub struct Refused {
122 pub code: &'static str,
123 pub message: String,
124}
125
126fn arg<'a>(args: &'a [Value], at: usize) -> &'a Value {
127 args.get(at).unwrap_or(&Value::Null)
128}
129
130fn text(args: &[Value], at: usize) -> Result<String, Refused> {
131 arg(args, at).as_str().map(str::to_owned).ok_or_else(|| Refused {
132 code: "INVALID_ARGUMENT",
133 message: format!("argument {at} should be text"),
134 })
135}
136
137/// The request for `method(args)`: on the namespace when `repo` is `None`
138/// (`create`, `get`, `delete`), else on the repository named `repo` there.
139pub fn route(namespace: &str, repo: Option<&str>, method: &str, args: &[Value]) -> Result<Route, Refused> {
140 let json_route = |method: &'static str, path: String, body: Option<Value>| Route { method, path, body, bytes: false };
141 let Some(repo) = repo else {
142 let name = text(args, 0)?;
143 let key = store_key(namespace, &name);
144 return match method {
145 "create" => {
146 let options = arg(args, 1);
147 Ok(json_route(
148 "POST",
149 "/api/repos".to_owned(),
150 Some(json!({
151 "name": key,
152 "description": options.get("description").cloned().unwrap_or(Value::Null),
153 "defaultBranch": options.get("setDefaultBranch").cloned().unwrap_or(Value::Null),
154 })),
155 ))
156 }
157 "get" => Ok(json_route("GET", format!("/api/repos/{}", segment(&key)), None)),
158 "delete" => Ok(json_route("DELETE", format!("/api/repos/{}", segment(&key)), None)),
159 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
160 };
161 };
162 let base = format!("/api/repos/{}", segment(&store_key(namespace, repo)));
163 match method {
164 "info" => Ok(json_route("GET", base, None)),
165 "createToken" => Ok(json_route(
166 "POST",
167 format!("{base}/tokens"),
168 Some(json!({ "scope": arg(args, 0).as_str().unwrap_or("write"), "ttl": arg(args, 1).as_u64().unwrap_or(3_600) })),
169 )),
170 "log" => {
171 let options = arg(args, 0);
172 let mut pairs = Vec::new();
173 for name in ["ref", "limit", "offset"] {
174 match options.get(name) {
175 Some(Value::String(value)) => pairs.push((name, value.clone())),
176 Some(Value::Number(value)) => pairs.push((name, value.to_string())),
177 _ => {}
178 }
179 }
180 Ok(json_route("GET", format!("{base}/log?{}", query(&pairs)), None))
181 }
182 "readCommit" => Ok(json_route("GET", format!("{base}/commits/{}", segment(&text(args, 0)?)), None)),
183 "readTree" => Ok(json_route("GET", format!("{base}/trees/{}", segment(&text(args, 0)?)), None)),
184 "readBlob" => Ok(Route { method: "GET", path: format!("{base}/blobs/{}", segment(&text(args, 0)?)), body: None, bytes: true }),
185 "readFile" => {
186 let options = arg(args, 0);
187 let field = |name: &str| options.get(name).and_then(Value::as_str).unwrap_or_default().to_owned();
188 Ok(Route {
189 method: "GET",
190 path: format!("{base}/file?{}", query(&[("ref", field("ref")), ("path", field("path"))])),
191 body: None,
192 bytes: true,
193 })
194 }
195 "fork" => {
196 let target = text(args, 0)?;
197 let options = arg(args, 1);
198 Ok(json_route(
199 "POST",
200 format!("{base}/fork"),
201 Some(json!({
202 "name": store_key(namespace, &target),
203 "defaultBranchOnly": options.get("defaultBranchOnly").and_then(Value::as_bool).unwrap_or(true),
204 })),
205 ))
206 }
207 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
208 }
209}
210
211/// What an answer from the store means for the call that asked.
212#[derive(Debug, PartialEq)]
213pub enum Answer {
214 /// JSON, as the binding would have returned it.
215 Json(Value),
216 /// A blob or a file's bytes.
217 Bytes(Vec<u8>),
218 /// Nothing there: a blob or file not found, as the binding's `null`.
219 Null,
220 /// An error, with the binding's code; `None` for one that may pass.
221 Error { code: Option<String>, message: String },
222}
223
224/// Reads an answer: `status` and `body` from the store, for `route`.
225pub fn answer(route: &Route, status: u16, body: Vec<u8>) -> Answer {
226 if (200..300).contains(&status) {
227 if route.bytes {
228 return Answer::Bytes(body);
229 }
230 return match serde_json::from_slice::<Value>(&body) {
231 Ok(Value::Null) => Answer::Null,
232 Ok(value) => Answer::Json(value),
233 Err(_) => Answer::Error { code: None, message: "the fallback store sent something that is not JSON".to_owned() },
234 };
235 }
236 if status == 404 && route.bytes {
237 return Answer::Null;
238 }
239 let said: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
240 let message = said.get("message").and_then(Value::as_str).map_or_else(|| format!("the fallback store answered {status}"), str::to_owned);
241 // 5xx may pass: like the binding's INTERNAL_ERROR.
242 let code = if status >= 500 {
243 Some("INTERNAL_ERROR".to_owned())
244 } else {
245 Some(said.get("code").and_then(Value::as_str).unwrap_or(if status == 404 { "NOT_FOUND" } else { "INVALID_ARGUMENT" }).to_owned())
246 };
247 Answer::Error { code, message }
248}
249
250/// Whether a call writes: refused while a switched namespace is read-only.
251pub fn writes(repo: Option<&str>, method: &str, args: &[Value]) -> bool {
252 match (repo, method) {
253 (None, "create" | "delete") => true,
254 (Some(_), "fork") => true,
255 (Some(_), "createToken") => arg(args, 0).as_str().unwrap_or("write") == "write",
256 _ => false,
257 }
258}
259
260#[cfg(test)]
261mod tests {
262 use super::*;
263
264 fn settings(namespaces: &str) -> Settings {
265 Settings::from_vars(Some("https://gitstore.example/"), Some("0123456789abcdef"), Some(namespaces), None).unwrap()
266 }
267
268 #[test]
269 fn nothing_is_switched_unless_the_store_and_namespaces_are_named() {
270 assert_eq!(Settings::from_vars(None, Some("0123456789abcdef"), Some("*"), None), None);
271 assert_eq!(Settings::from_vars(Some("https://x"), None, Some("*"), None), None);
272 // A secret too short to be one.
273 assert_eq!(Settings::from_vars(Some("https://x"), Some("short"), Some("*"), None), None);
274 assert_eq!(Settings::from_vars(Some("ftp://x"), Some("0123456789abcdef"), Some("*"), None), None);
275 let none = settings("");
276 assert!(!none.serves("g1t"));
277 let some = settings("g1t, g1t-us-1,bad name");
278 assert!(some.serves("g1t") && some.serves("g1t-us-1") && !some.serves("g1t-eu"));
279 assert!(settings("*").serves("anything"));
280 // Read-only unless told otherwise.
281 assert!(!some.writes);
282 let writable = Settings::from_vars(Some("https://x"), Some("0123456789abcdef"), Some("*"), Some("Allow")).unwrap();
283 assert!(writable.writes);
284 assert_eq!(some.url, "https://gitstore.example");
285 assert_eq!(some.remote("g1t", "acme--rocket"), "https://gitstore.example/git/g1t/acme--rocket.git");
286 }
287
288 #[test]
289 fn a_remote_names_its_key_as_an_artifacts_remote_does() {
290 let remote = settings("*").remote("g1t-us-1", "acme--rocket");
291 // store.rs reads keys back from remotes by their last two segments.
292 assert!(remote.ends_with("/g1t-us-1/acme--rocket.git"));
293 }
294
295 #[test]
296 fn calls_on_the_namespace_become_requests_for_its_key() {
297 let create = route("g1t", None, "create", &[json!("acme--rocket"), json!({ "description": "d", "setDefaultBranch": "trunk" })]).unwrap();
298 assert_eq!(create.method, "POST");
299 assert_eq!(create.path, "/api/repos");
300 assert_eq!(create.body.unwrap(), json!({ "name": "g1t/acme--rocket", "description": "d", "defaultBranch": "trunk" }));
301 let get = route("g1t", None, "get", &[json!("acme--rocket")]).unwrap();
302 assert_eq!((get.method, get.path.as_str()), ("GET", "/api/repos/g1t%2Facme--rocket"));
303 assert_eq!(route("g1t", None, "delete", &[json!("x1")]).unwrap().method, "DELETE");
304 assert_eq!(route("g1t", None, "list", &[json!("x1")]).unwrap_err().code, "NOT_SUPPORTED");
305 assert_eq!(route("g1t", None, "get", &[]).unwrap_err().code, "INVALID_ARGUMENT");
306 }
307
308 #[test]
309 fn calls_on_a_repository_become_requests_below_it() {
310 let base = "/api/repos/g1t-us-1%2Fpulls--pul_1";
311 let at = |method: &str, args: &[Value]| route("g1t-us-1", Some("pulls--pul_1"), method, args).unwrap();
312 assert_eq!(at("info", &[]).path, base);
313 let token = at("createToken", &[json!("read"), json!(300)]);
314 assert_eq!(token.body.unwrap(), json!({ "scope": "read", "ttl": 300 }));
315 assert_eq!(at("log", &[json!({ "ref": "fix/#1", "limit": 20 })]).path, format!("{base}/log?ref=fix%2F%231&limit=20"));
316 assert_eq!(at("readCommit", &[json!("abc")]).path, format!("{base}/commits/abc"));
317 assert_eq!(at("readTree", &[json!("abc")]).path, format!("{base}/trees/abc"));
318 let blob = at("readBlob", &[json!("abc")]);
319 assert!(blob.bytes);
320 let file = at("readFile", &[json!({ "ref": "main", "path": "src/a b.rs" })]);
321 assert_eq!(file.path, format!("{base}/file?ref=main&path=src%2Fa%20b.rs"));
322 assert!(file.bytes);
323 let fork = at("fork", &[json!("pulls--pul_2"), json!({ "defaultBranchOnly": true })]);
324 assert_eq!(fork.body.unwrap(), json!({ "name": "g1t-us-1/pulls--pul_2", "defaultBranchOnly": true }));
325 }
326
327 #[test]
328 fn answers_read_as_the_binding_would_have_returned_them() {
329 let json_route = route("g1t", Some("r"), "readTree", &[json!("a")]).unwrap();
330 let blob = route("g1t", Some("r"), "readBlob", &[json!("a")]).unwrap();
331 assert_eq!(answer(&json_route, 200, b"[]".to_vec()), Answer::Json(json!([])));
332 assert_eq!(answer(&json_route, 200, b"null".to_vec()), Answer::Null);
333 assert_eq!(answer(&blob, 200, b"hi".to_vec()), Answer::Bytes(b"hi".to_vec()));
334 assert_eq!(answer(&blob, 404, b"{}".to_vec()), Answer::Null);
335 assert_eq!(
336 answer(&json_route, 404, br#"{"code":"NOT_FOUND","message":"no repository g1t/r"}"#.to_vec()),
337 Answer::Error { code: Some("NOT_FOUND".into()), message: "no repository g1t/r".into() }
338 );
339 assert_eq!(
340 answer(&json_route, 409, br#"{"code":"ALREADY_EXISTS","message":"x"}"#.to_vec()),
341 Answer::Error { code: Some("ALREADY_EXISTS".into()), message: "x".into() }
342 );
343 // A 5xx may pass, as the binding's INTERNAL_ERROR.
344 assert!(matches!(answer(&json_route, 502, Vec::new()), Answer::Error { code: Some(code), .. } if code == "INTERNAL_ERROR"));
345 assert!(matches!(answer(&json_route, 200, b"<html>".to_vec()), Answer::Error { code: None, .. }));
346 }
347
348 #[test]
349 fn writes_are_told_apart_from_reads() {
350 assert!(writes(None, "create", &[json!("x")]));
351 assert!(writes(None, "delete", &[json!("x")]));
352 assert!(!writes(None, "get", &[json!("x")]));
353 assert!(writes(Some("r"), "fork", &[json!("y")]));
354 assert!(writes(Some("r"), "createToken", &[json!("write"), json!(60)]));
355 assert!(!writes(Some("r"), "createToken", &[json!("read"), json!(60)]));
356 assert!(!writes(Some("r"), "readBlob", &[json!("a")]));
357 }
358}