Skip to content
Sign up

flagon-io/g1t

public
0
360 linesCodeBlameRaw
1//! The cold fallback for the git store. CONTRIBUTING.md ("Operating
2//! g1t.sh") says when to switch to it; scripts/ops/restore-to-gitstore.mjs
3//! has every step.
4//!
5//! When Artifacts is down for a namespace, the repos service can serve that
6//! namespace from a self-hosted git store instead
7//! (`deploy/self-host/gitstore`), rebuilt from the nightly backups
8//! (`scripts/ops/restore-to-gitstore.mjs`). It is switched by
9//! configuration, not code:
10//!
11//! - `GIT_FALLBACK_URL`: where the git store answers, `https://...`.
12//! - `GIT_FALLBACK_SECRET` (a secret): the store's API secret.
13//! - `GIT_FALLBACK_NAMESPACES`: the namespaces served from it, comma
14//! separated, or `*` for all. Unset or empty: none, and nothing changes.
15//! - `GIT_FALLBACK_WRITES`: `refuse` (the default) or `allow`. While
16//! refused, a switched namespace is read-only: clones, fetches and pages
17//! work; pushes, merges and new repositories wait, told so in words.
18//!
19//! The store keeps each namespace's repositories under its own directory,
20//! so the key `g1t-us-1/acme--rocket` is `g1t-us-1/acme--rocket` there and
21//! `acme--rocket` (the default namespace) is `g1t/acme--rocket`. Its remotes
22//! read `<url>/git/<namespace>/<name>.git`, the shape Artifacts uses.
23//!
24//! This module is the plain part: the settings, and how each call the
25//! service makes on an Artifacts binding becomes a request to the store's
26//! API and back. store.rs makes the requests.
27
28use serde_json::{Value, json};
29
30/// The fallback store's settings, when it is configured at all.
31#[derive(Clone, Debug, PartialEq, Eq)]
32pub struct Settings {
33 /// Where the store answers, without a trailing slash.
34 pub url: String,
35 pub secret: String,
36 pub switched: Switched,
37 /// Whether a switched namespace takes writes.
38 pub writes: bool,
39}
40
41/// Which namespaces are served from the fallback store.
42#[derive(Clone, Debug, PartialEq, Eq)]
43pub enum Switched {
44 All,
45 Some(Vec<String>),
46}
47
48impl Settings {
49 /// The settings from the variables, or `None` when there is no store
50 /// to fall back to (no address, or no secret).
51 pub fn from_vars(url: Option<&str>, secret: Option<&str>, namespaces: Option<&str>, writes: Option<&str>) -> Option<Self> {
52 let url = url.map(str::trim).filter(|url| url.starts_with("https://") || url.starts_with("http://"))?;
53 let secret = secret.map(str::trim).filter(|secret| secret.len() >= 16)?;
54 let namespaces = namespaces.unwrap_or_default().trim();
55 let switched = if namespaces == "*" {
56 Switched::All
57 } else {
58 Switched::Some(
59 namespaces
60 .split(',')
61 .map(str::trim)
62 .filter(|name| crate::shards::valid_namespace(name))
63 .map(str::to_owned)
64 .collect(),
65 )
66 };
67 Some(Settings {
68 url: url.trim_end_matches('/').to_owned(),
69 secret: secret.to_owned(),
70 switched,
71 writes: writes.is_some_and(|value| value.trim().eq_ignore_ascii_case("allow")),
72 })
73 }
74
75 /// Whether `namespace` is served from the fallback store now.
76 pub fn serves(&self, namespace: &str) -> bool {
77 match &self.switched {
78 Switched::All => true,
79 Switched::Some(names) => names.iter().any(|name| name == namespace),
80 }
81 }
82
83 /// The remote git uses for `name` in `namespace`.
84 pub fn remote(&self, namespace: &str, name: &str) -> String {
85 format!("{}/git/{}.git", self.url, store_key(namespace, name))
86 }
87}
88
89/// A repository's key in the fallback store: always with its namespace.
90pub fn store_key(namespace: &str, name: &str) -> String {
91 format!("{namespace}/{name}")
92}
93
94/// Percent-encodes one path segment.
95fn segment(text: &str) -> String {
96 text.bytes()
97 .map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") })
98 .collect()
99}
100
101/// Percent-encodes a query value.
102fn query(pairs: &[(&str, String)]) -> String {
103 pairs
104 .iter()
105 .map(|(key, value)| format!("{key}={}", segment(value)))
106 .collect::<Vec<_>>()
107 .join("&")
108}
109
110/// How a call on the binding is asked of the store's API.
111#[derive(Debug, PartialEq)]
112pub struct Route {
113 pub method: &'static str,
114 /// Below the store's address: `/api/repos/...`.
115 pub path: String,
116 pub body: Option<Value>,
117 /// What the answer is: JSON, or bytes (a blob or a file).
118 pub bytes: bool,
119}
120
121/// A call the fallback cannot make, as the binding would have thrown it.
122#[derive(Debug, PartialEq)]
123pub struct Refused {
124 pub code: &'static str,
125 pub message: String,
126}
127
128fn arg<'a>(args: &'a [Value], at: usize) -> &'a Value {
129 args.get(at).unwrap_or(&Value::Null)
130}
131
132fn text(args: &[Value], at: usize) -> Result<String, Refused> {
133 arg(args, at).as_str().map(str::to_owned).ok_or_else(|| Refused {
134 code: "INVALID_ARGUMENT",
135 message: format!("argument {at} should be text"),
136 })
137}
138
139/// The request for `method(args)`: on the namespace when `repo` is `None`
140/// (`create`, `get`, `delete`), else on the repository named `repo` there.
141pub fn route(namespace: &str, repo: Option<&str>, method: &str, args: &[Value]) -> Result<Route, Refused> {
142 let json_route = |method: &'static str, path: String, body: Option<Value>| Route { method, path, body, bytes: false };
143 let Some(repo) = repo else {
144 let name = text(args, 0)?;
145 let key = store_key(namespace, &name);
146 return match method {
147 "create" => {
148 let options = arg(args, 1);
149 Ok(json_route(
150 "POST",
151 "/api/repos".to_owned(),
152 Some(json!({
153 "name": key,
154 "description": options.get("description").cloned().unwrap_or(Value::Null),
155 "defaultBranch": options.get("setDefaultBranch").cloned().unwrap_or(Value::Null),
156 })),
157 ))
158 }
159 "get" => Ok(json_route("GET", format!("/api/repos/{}", segment(&key)), None)),
160 "delete" => Ok(json_route("DELETE", format!("/api/repos/{}", segment(&key)), None)),
161 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
162 };
163 };
164 let base = format!("/api/repos/{}", segment(&store_key(namespace, repo)));
165 match method {
166 "info" => Ok(json_route("GET", base, None)),
167 "createToken" => Ok(json_route(
168 "POST",
169 format!("{base}/tokens"),
170 Some(json!({ "scope": arg(args, 0).as_str().unwrap_or("write"), "ttl": arg(args, 1).as_u64().unwrap_or(3_600) })),
171 )),
172 "log" => {
173 let options = arg(args, 0);
174 let mut pairs = Vec::new();
175 for name in ["ref", "limit", "offset"] {
176 match options.get(name) {
177 Some(Value::String(value)) => pairs.push((name, value.clone())),
178 Some(Value::Number(value)) => pairs.push((name, value.to_string())),
179 _ => {}
180 }
181 }
182 Ok(json_route("GET", format!("{base}/log?{}", query(&pairs)), None))
183 }
184 "readCommit" => Ok(json_route("GET", format!("{base}/commits/{}", segment(&text(args, 0)?)), None)),
185 "readTree" => Ok(json_route("GET", format!("{base}/trees/{}", segment(&text(args, 0)?)), None)),
186 "readBlob" => Ok(Route { method: "GET", path: format!("{base}/blobs/{}", segment(&text(args, 0)?)), body: None, bytes: true }),
187 "readFile" => {
188 let options = arg(args, 0);
189 let field = |name: &str| options.get(name).and_then(Value::as_str).unwrap_or_default().to_owned();
190 Ok(Route {
191 method: "GET",
192 path: format!("{base}/file?{}", query(&[("ref", field("ref")), ("path", field("path"))])),
193 body: None,
194 bytes: true,
195 })
196 }
197 "fork" => {
198 let target = text(args, 0)?;
199 let options = arg(args, 1);
200 Ok(json_route(
201 "POST",
202 format!("{base}/fork"),
203 Some(json!({
204 "name": store_key(namespace, &target),
205 "defaultBranchOnly": options.get("defaultBranchOnly").and_then(Value::as_bool).unwrap_or(true),
206 })),
207 ))
208 }
209 other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }),
210 }
211}
212
213/// What an answer from the store means for the call that asked.
214#[derive(Debug, PartialEq)]
215pub enum Answer {
216 /// JSON, as the binding would have returned it.
217 Json(Value),
218 /// A blob or a file's bytes.
219 Bytes(Vec<u8>),
220 /// Nothing there: a blob or file not found, as the binding's `null`.
221 Null,
222 /// An error, with the binding's code; `None` for one that may pass.
223 Error { code: Option<String>, message: String },
224}
225
226/// Reads an answer: `status` and `body` from the store, for `route`.
227pub fn answer(route: &Route, status: u16, body: Vec<u8>) -> Answer {
228 if (200..300).contains(&status) {
229 if route.bytes {
230 return Answer::Bytes(body);
231 }
232 return match serde_json::from_slice::<Value>(&body) {
233 Ok(Value::Null) => Answer::Null,
234 Ok(value) => Answer::Json(value),
235 Err(_) => Answer::Error { code: None, message: "the fallback store sent something that is not JSON".to_owned() },
236 };
237 }
238 if status == 404 && route.bytes {
239 return Answer::Null;
240 }
241 let said: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
242 let message = said.get("message").and_then(Value::as_str).map_or_else(|| format!("the fallback store answered {status}"), str::to_owned);
243 // 5xx may pass: like the binding's INTERNAL_ERROR.
244 let code = if status >= 500 {
245 Some("INTERNAL_ERROR".to_owned())
246 } else {
247 Some(said.get("code").and_then(Value::as_str).unwrap_or(if status == 404 { "NOT_FOUND" } else { "INVALID_ARGUMENT" }).to_owned())
248 };
249 Answer::Error { code, message }
250}
251
252/// Whether a call writes: refused while a switched namespace is read-only.
253pub fn writes(repo: Option<&str>, method: &str, args: &[Value]) -> bool {
254 match (repo, method) {
255 (None, "create" | "delete") => true,
256 (Some(_), "fork") => true,
257 (Some(_), "createToken") => arg(args, 0).as_str().unwrap_or("write") == "write",
258 _ => false,
259 }
260}
261
262#[cfg(test)]
263mod tests {
264 use super::*;
265
266 fn settings(namespaces: &str) -> Settings {
267 Settings::from_vars(Some("https://gitstore.example/"), Some("0123456789abcdef"), Some(namespaces), None).unwrap()
268 }
269
270 #[test]
271 fn nothing_is_switched_unless_the_store_and_namespaces_are_named() {
272 assert_eq!(Settings::from_vars(None, Some("0123456789abcdef"), Some("*"), None), None);
273 assert_eq!(Settings::from_vars(Some("https://x"), None, Some("*"), None), None);
274 // A secret too short to be one.
275 assert_eq!(Settings::from_vars(Some("https://x"), Some("short"), Some("*"), None), None);
276 assert_eq!(Settings::from_vars(Some("ftp://x"), Some("0123456789abcdef"), Some("*"), None), None);
277 let none = settings("");
278 assert!(!none.serves("g1t"));
279 let some = settings("g1t, g1t-us-1,bad name");
280 assert!(some.serves("g1t") && some.serves("g1t-us-1") && !some.serves("g1t-eu"));
281 assert!(settings("*").serves("anything"));
282 // Read-only unless told otherwise.
283 assert!(!some.writes);
284 let writable = Settings::from_vars(Some("https://x"), Some("0123456789abcdef"), Some("*"), Some("Allow")).unwrap();
285 assert!(writable.writes);
286 assert_eq!(some.url, "https://gitstore.example");
287 assert_eq!(some.remote("g1t", "acme--rocket"), "https://gitstore.example/git/g1t/acme--rocket.git");
288 }
289
290 #[test]
291 fn a_remote_names_its_key_as_an_artifacts_remote_does() {
292 let remote = settings("*").remote("g1t-us-1", "acme--rocket");
293 // store.rs reads keys back from remotes by their last two segments.
294 assert!(remote.ends_with("/g1t-us-1/acme--rocket.git"));
295 }
296
297 #[test]
298 fn calls_on_the_namespace_become_requests_for_its_key() {
299 let create = route("g1t", None, "create", &[json!("acme--rocket"), json!({ "description": "d", "setDefaultBranch": "trunk" })]).unwrap();
300 assert_eq!(create.method, "POST");
301 assert_eq!(create.path, "/api/repos");
302 assert_eq!(create.body.unwrap(), json!({ "name": "g1t/acme--rocket", "description": "d", "defaultBranch": "trunk" }));
303 let get = route("g1t", None, "get", &[json!("acme--rocket")]).unwrap();
304 assert_eq!((get.method, get.path.as_str()), ("GET", "/api/repos/g1t%2Facme--rocket"));
305 assert_eq!(route("g1t", None, "delete", &[json!("x1")]).unwrap().method, "DELETE");
306 assert_eq!(route("g1t", None, "list", &[json!("x1")]).unwrap_err().code, "NOT_SUPPORTED");
307 assert_eq!(route("g1t", None, "get", &[]).unwrap_err().code, "INVALID_ARGUMENT");
308 }
309
310 #[test]
311 fn calls_on_a_repository_become_requests_below_it() {
312 let base = "/api/repos/g1t-us-1%2Fpulls--pul_1";
313 let at = |method: &str, args: &[Value]| route("g1t-us-1", Some("pulls--pul_1"), method, args).unwrap();
314 assert_eq!(at("info", &[]).path, base);
315 let token = at("createToken", &[json!("read"), json!(300)]);
316 assert_eq!(token.body.unwrap(), json!({ "scope": "read", "ttl": 300 }));
317 assert_eq!(at("log", &[json!({ "ref": "fix/#1", "limit": 20 })]).path, format!("{base}/log?ref=fix%2F%231&limit=20"));
318 assert_eq!(at("readCommit", &[json!("abc")]).path, format!("{base}/commits/abc"));
319 assert_eq!(at("readTree", &[json!("abc")]).path, format!("{base}/trees/abc"));
320 let blob = at("readBlob", &[json!("abc")]);
321 assert!(blob.bytes);
322 let file = at("readFile", &[json!({ "ref": "main", "path": "src/a b.rs" })]);
323 assert_eq!(file.path, format!("{base}/file?ref=main&path=src%2Fa%20b.rs"));
324 assert!(file.bytes);
325 let fork = at("fork", &[json!("pulls--pul_2"), json!({ "defaultBranchOnly": true })]);
326 assert_eq!(fork.body.unwrap(), json!({ "name": "g1t-us-1/pulls--pul_2", "defaultBranchOnly": true }));
327 }
328
329 #[test]
330 fn answers_read_as_the_binding_would_have_returned_them() {
331 let json_route = route("g1t", Some("r"), "readTree", &[json!("a")]).unwrap();
332 let blob = route("g1t", Some("r"), "readBlob", &[json!("a")]).unwrap();
333 assert_eq!(answer(&json_route, 200, b"[]".to_vec()), Answer::Json(json!([])));
334 assert_eq!(answer(&json_route, 200, b"null".to_vec()), Answer::Null);
335 assert_eq!(answer(&blob, 200, b"hi".to_vec()), Answer::Bytes(b"hi".to_vec()));
336 assert_eq!(answer(&blob, 404, b"{}".to_vec()), Answer::Null);
337 assert_eq!(
338 answer(&json_route, 404, br#"{"code":"NOT_FOUND","message":"no repository g1t/r"}"#.to_vec()),
339 Answer::Error { code: Some("NOT_FOUND".into()), message: "no repository g1t/r".into() }
340 );
341 assert_eq!(
342 answer(&json_route, 409, br#"{"code":"ALREADY_EXISTS","message":"x"}"#.to_vec()),
343 Answer::Error { code: Some("ALREADY_EXISTS".into()), message: "x".into() }
344 );
345 // A 5xx may pass, as the binding's INTERNAL_ERROR.
346 assert!(matches!(answer(&json_route, 502, Vec::new()), Answer::Error { code: Some(code), .. } if code == "INTERNAL_ERROR"));
347 assert!(matches!(answer(&json_route, 200, b"<html>".to_vec()), Answer::Error { code: None, .. }));
348 }
349
350 #[test]
351 fn writes_are_told_apart_from_reads() {
352 assert!(writes(None, "create", &[json!("x")]));
353 assert!(writes(None, "delete", &[json!("x")]));
354 assert!(!writes(None, "get", &[json!("x")]));
355 assert!(writes(Some("r"), "fork", &[json!("y")]));
356 assert!(writes(Some("r"), "createToken", &[json!("write"), json!(60)]));
357 assert!(!writes(Some("r"), "createToken", &[json!("read"), json!(60)]));
358 assert!(!writes(Some("r"), "readBlob", &[json!("a")]));
359 }
360}