| 1 | //! The cold fallback for the git store (docs/ARTIFACTS.md, R12). |
| 2 | //! |
| 3 | //! When Artifacts is down for a namespace, the repos service can serve that |
| 4 | //! namespace from a self-hosted git store instead |
| 5 | //! (`deploy/self-host/gitstore`), rebuilt from the nightly backups |
| 6 | //! (`scripts/ops/restore-to-gitstore.mjs`). It is switched by |
| 7 | //! configuration, not code: |
| 8 | //! |
| 9 | //! - `GIT_FALLBACK_URL`: where the git store answers, `https://...`. |
| 10 | //! - `GIT_FALLBACK_SECRET` (a secret): the store's API secret. |
| 11 | //! - `GIT_FALLBACK_NAMESPACES`: the namespaces served from it, comma |
| 12 | //! separated, or `*` for all. Unset or empty: none, and nothing changes. |
| 13 | //! - `GIT_FALLBACK_WRITES`: `refuse` (the default) or `allow`. While |
| 14 | //! refused, a switched namespace is read-only: clones, fetches and pages |
| 15 | //! work; pushes, merges and new repositories wait, told so in words. |
| 16 | //! |
| 17 | //! The store keeps each namespace's repositories under its own directory, |
| 18 | //! so the key `g1t-us-1/acme--rocket` is `g1t-us-1/acme--rocket` there and |
| 19 | //! `acme--rocket` (the default namespace) is `g1t/acme--rocket`. Its remotes |
| 20 | //! read `<url>/git/<namespace>/<name>.git`, the shape Artifacts uses. |
| 21 | //! |
| 22 | //! This module is the plain part: the settings, and how each call the |
| 23 | //! service makes on an Artifacts binding becomes a request to the store's |
| 24 | //! API and back. store.rs makes the requests. |
| 25 | |
| 26 | use serde_json::{Value, json}; |
| 27 | |
| 28 | /// The fallback store's settings, when it is configured at all. |
| 29 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 30 | pub struct Settings { |
| 31 | /// Where the store answers, without a trailing slash. |
| 32 | pub url: String, |
| 33 | pub secret: String, |
| 34 | pub switched: Switched, |
| 35 | /// Whether a switched namespace takes writes. |
| 36 | pub writes: bool, |
| 37 | } |
| 38 | |
| 39 | /// Which namespaces are served from the fallback store. |
| 40 | #[derive(Clone, Debug, PartialEq, Eq)] |
| 41 | pub enum Switched { |
| 42 | All, |
| 43 | Some(Vec<String>), |
| 44 | } |
| 45 | |
| 46 | impl Settings { |
| 47 | /// The settings from the variables, or `None` when there is no store |
| 48 | /// to fall back to (no address, or no secret). |
| 49 | pub fn from_vars(url: Option<&str>, secret: Option<&str>, namespaces: Option<&str>, writes: Option<&str>) -> Option<Self> { |
| 50 | let url = url.map(str::trim).filter(|url| url.starts_with("https://") || url.starts_with("http://"))?; |
| 51 | let secret = secret.map(str::trim).filter(|secret| secret.len() >= 16)?; |
| 52 | let namespaces = namespaces.unwrap_or_default().trim(); |
| 53 | let switched = if namespaces == "*" { |
| 54 | Switched::All |
| 55 | } else { |
| 56 | Switched::Some( |
| 57 | namespaces |
| 58 | .split(',') |
| 59 | .map(str::trim) |
| 60 | .filter(|name| crate::shards::valid_namespace(name)) |
| 61 | .map(str::to_owned) |
| 62 | .collect(), |
| 63 | ) |
| 64 | }; |
| 65 | Some(Settings { |
| 66 | url: url.trim_end_matches('/').to_owned(), |
| 67 | secret: secret.to_owned(), |
| 68 | switched, |
| 69 | writes: writes.is_some_and(|value| value.trim().eq_ignore_ascii_case("allow")), |
| 70 | }) |
| 71 | } |
| 72 | |
| 73 | /// Whether `namespace` is served from the fallback store now. |
| 74 | pub fn serves(&self, namespace: &str) -> bool { |
| 75 | match &self.switched { |
| 76 | Switched::All => true, |
| 77 | Switched::Some(names) => names.iter().any(|name| name == namespace), |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | /// The remote git uses for `name` in `namespace`. |
| 82 | pub fn remote(&self, namespace: &str, name: &str) -> String { |
| 83 | format!("{}/git/{}.git", self.url, store_key(namespace, name)) |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | /// A repository's key in the fallback store: always with its namespace. |
| 88 | pub fn store_key(namespace: &str, name: &str) -> String { |
| 89 | format!("{namespace}/{name}") |
| 90 | } |
| 91 | |
| 92 | /// Percent-encodes one path segment. |
| 93 | fn segment(text: &str) -> String { |
| 94 | text.bytes() |
| 95 | .map(|b| if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { (b as char).to_string() } else { format!("%{b:02X}") }) |
| 96 | .collect() |
| 97 | } |
| 98 | |
| 99 | /// Percent-encodes a query value. |
| 100 | fn query(pairs: &[(&str, String)]) -> String { |
| 101 | pairs |
| 102 | .iter() |
| 103 | .map(|(key, value)| format!("{key}={}", segment(value))) |
| 104 | .collect::<Vec<_>>() |
| 105 | .join("&") |
| 106 | } |
| 107 | |
| 108 | /// How a call on the binding is asked of the store's API. |
| 109 | #[derive(Debug, PartialEq)] |
| 110 | pub struct Route { |
| 111 | pub method: &'static str, |
| 112 | /// Below the store's address: `/api/repos/...`. |
| 113 | pub path: String, |
| 114 | pub body: Option<Value>, |
| 115 | /// What the answer is: JSON, or bytes (a blob or a file). |
| 116 | pub bytes: bool, |
| 117 | } |
| 118 | |
| 119 | /// A call the fallback cannot make, as the binding would have thrown it. |
| 120 | #[derive(Debug, PartialEq)] |
| 121 | pub struct Refused { |
| 122 | pub code: &'static str, |
| 123 | pub message: String, |
| 124 | } |
| 125 | |
| 126 | fn arg<'a>(args: &'a [Value], at: usize) -> &'a Value { |
| 127 | args.get(at).unwrap_or(&Value::Null) |
| 128 | } |
| 129 | |
| 130 | fn text(args: &[Value], at: usize) -> Result<String, Refused> { |
| 131 | arg(args, at).as_str().map(str::to_owned).ok_or_else(|| Refused { |
| 132 | code: "INVALID_ARGUMENT", |
| 133 | message: format!("argument {at} should be text"), |
| 134 | }) |
| 135 | } |
| 136 | |
| 137 | /// The request for `method(args)`: on the namespace when `repo` is `None` |
| 138 | /// (`create`, `get`, `delete`), else on the repository named `repo` there. |
| 139 | pub fn route(namespace: &str, repo: Option<&str>, method: &str, args: &[Value]) -> Result<Route, Refused> { |
| 140 | let json_route = |method: &'static str, path: String, body: Option<Value>| Route { method, path, body, bytes: false }; |
| 141 | let Some(repo) = repo else { |
| 142 | let name = text(args, 0)?; |
| 143 | let key = store_key(namespace, &name); |
| 144 | return match method { |
| 145 | "create" => { |
| 146 | let options = arg(args, 1); |
| 147 | Ok(json_route( |
| 148 | "POST", |
| 149 | "/api/repos".to_owned(), |
| 150 | Some(json!({ |
| 151 | "name": key, |
| 152 | "description": options.get("description").cloned().unwrap_or(Value::Null), |
| 153 | "defaultBranch": options.get("setDefaultBranch").cloned().unwrap_or(Value::Null), |
| 154 | })), |
| 155 | )) |
| 156 | } |
| 157 | "get" => Ok(json_route("GET", format!("/api/repos/{}", segment(&key)), None)), |
| 158 | "delete" => Ok(json_route("DELETE", format!("/api/repos/{}", segment(&key)), None)), |
| 159 | other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }), |
| 160 | }; |
| 161 | }; |
| 162 | let base = format!("/api/repos/{}", segment(&store_key(namespace, repo))); |
| 163 | match method { |
| 164 | "info" => Ok(json_route("GET", base, None)), |
| 165 | "createToken" => Ok(json_route( |
| 166 | "POST", |
| 167 | format!("{base}/tokens"), |
| 168 | Some(json!({ "scope": arg(args, 0).as_str().unwrap_or("write"), "ttl": arg(args, 1).as_u64().unwrap_or(3_600) })), |
| 169 | )), |
| 170 | "log" => { |
| 171 | let options = arg(args, 0); |
| 172 | let mut pairs = Vec::new(); |
| 173 | for name in ["ref", "limit", "offset"] { |
| 174 | match options.get(name) { |
| 175 | Some(Value::String(value)) => pairs.push((name, value.clone())), |
| 176 | Some(Value::Number(value)) => pairs.push((name, value.to_string())), |
| 177 | _ => {} |
| 178 | } |
| 179 | } |
| 180 | Ok(json_route("GET", format!("{base}/log?{}", query(&pairs)), None)) |
| 181 | } |
| 182 | "readCommit" => Ok(json_route("GET", format!("{base}/commits/{}", segment(&text(args, 0)?)), None)), |
| 183 | "readTree" => Ok(json_route("GET", format!("{base}/trees/{}", segment(&text(args, 0)?)), None)), |
| 184 | "readBlob" => Ok(Route { method: "GET", path: format!("{base}/blobs/{}", segment(&text(args, 0)?)), body: None, bytes: true }), |
| 185 | "readFile" => { |
| 186 | let options = arg(args, 0); |
| 187 | let field = |name: &str| options.get(name).and_then(Value::as_str).unwrap_or_default().to_owned(); |
| 188 | Ok(Route { |
| 189 | method: "GET", |
| 190 | path: format!("{base}/file?{}", query(&[("ref", field("ref")), ("path", field("path"))])), |
| 191 | body: None, |
| 192 | bytes: true, |
| 193 | }) |
| 194 | } |
| 195 | "fork" => { |
| 196 | let target = text(args, 0)?; |
| 197 | let options = arg(args, 1); |
| 198 | Ok(json_route( |
| 199 | "POST", |
| 200 | format!("{base}/fork"), |
| 201 | Some(json!({ |
| 202 | "name": store_key(namespace, &target), |
| 203 | "defaultBranchOnly": options.get("defaultBranchOnly").and_then(Value::as_bool).unwrap_or(true), |
| 204 | })), |
| 205 | )) |
| 206 | } |
| 207 | other => Err(Refused { code: "NOT_SUPPORTED", message: format!("{other} is not offered by the fallback store") }), |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | /// What an answer from the store means for the call that asked. |
| 212 | #[derive(Debug, PartialEq)] |
| 213 | pub enum Answer { |
| 214 | /// JSON, as the binding would have returned it. |
| 215 | Json(Value), |
| 216 | /// A blob or a file's bytes. |
| 217 | Bytes(Vec<u8>), |
| 218 | /// Nothing there: a blob or file not found, as the binding's `null`. |
| 219 | Null, |
| 220 | /// An error, with the binding's code; `None` for one that may pass. |
| 221 | Error { code: Option<String>, message: String }, |
| 222 | } |
| 223 | |
| 224 | /// Reads an answer: `status` and `body` from the store, for `route`. |
| 225 | pub fn answer(route: &Route, status: u16, body: Vec<u8>) -> Answer { |
| 226 | if (200..300).contains(&status) { |
| 227 | if route.bytes { |
| 228 | return Answer::Bytes(body); |
| 229 | } |
| 230 | return match serde_json::from_slice::<Value>(&body) { |
| 231 | Ok(Value::Null) => Answer::Null, |
| 232 | Ok(value) => Answer::Json(value), |
| 233 | Err(_) => Answer::Error { code: None, message: "the fallback store sent something that is not JSON".to_owned() }, |
| 234 | }; |
| 235 | } |
| 236 | if status == 404 && route.bytes { |
| 237 | return Answer::Null; |
| 238 | } |
| 239 | let said: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); |
| 240 | let message = said.get("message").and_then(Value::as_str).map_or_else(|| format!("the fallback store answered {status}"), str::to_owned); |
| 241 | // 5xx may pass: like the binding's INTERNAL_ERROR. |
| 242 | let code = if status >= 500 { |
| 243 | Some("INTERNAL_ERROR".to_owned()) |
| 244 | } else { |
| 245 | Some(said.get("code").and_then(Value::as_str).unwrap_or(if status == 404 { "NOT_FOUND" } else { "INVALID_ARGUMENT" }).to_owned()) |
| 246 | }; |
| 247 | Answer::Error { code, message } |
| 248 | } |
| 249 | |
| 250 | /// Whether a call writes: refused while a switched namespace is read-only. |
| 251 | pub fn writes(repo: Option<&str>, method: &str, args: &[Value]) -> bool { |
| 252 | match (repo, method) { |
| 253 | (None, "create" | "delete") => true, |
| 254 | (Some(_), "fork") => true, |
| 255 | (Some(_), "createToken") => arg(args, 0).as_str().unwrap_or("write") == "write", |
| 256 | _ => false, |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | #[cfg(test)] |
| 261 | mod tests { |
| 262 | use super::*; |
| 263 | |
| 264 | fn settings(namespaces: &str) -> Settings { |
| 265 | Settings::from_vars(Some("https://gitstore.example/"), Some("0123456789abcdef"), Some(namespaces), None).unwrap() |
| 266 | } |
| 267 | |
| 268 | #[test] |
| 269 | fn nothing_is_switched_unless_the_store_and_namespaces_are_named() { |
| 270 | assert_eq!(Settings::from_vars(None, Some("0123456789abcdef"), Some("*"), None), None); |
| 271 | assert_eq!(Settings::from_vars(Some("https://x"), None, Some("*"), None), None); |
| 272 | // A secret too short to be one. |
| 273 | assert_eq!(Settings::from_vars(Some("https://x"), Some("short"), Some("*"), None), None); |
| 274 | assert_eq!(Settings::from_vars(Some("ftp://x"), Some("0123456789abcdef"), Some("*"), None), None); |
| 275 | let none = settings(""); |
| 276 | assert!(!none.serves("g1t")); |
| 277 | let some = settings("g1t, g1t-us-1,bad name"); |
| 278 | assert!(some.serves("g1t") && some.serves("g1t-us-1") && !some.serves("g1t-eu")); |
| 279 | assert!(settings("*").serves("anything")); |
| 280 | // Read-only unless told otherwise. |
| 281 | assert!(!some.writes); |
| 282 | let writable = Settings::from_vars(Some("https://x"), Some("0123456789abcdef"), Some("*"), Some("Allow")).unwrap(); |
| 283 | assert!(writable.writes); |
| 284 | assert_eq!(some.url, "https://gitstore.example"); |
| 285 | assert_eq!(some.remote("g1t", "acme--rocket"), "https://gitstore.example/git/g1t/acme--rocket.git"); |
| 286 | } |
| 287 | |
| 288 | #[test] |
| 289 | fn a_remote_names_its_key_as_an_artifacts_remote_does() { |
| 290 | let remote = settings("*").remote("g1t-us-1", "acme--rocket"); |
| 291 | // store.rs reads keys back from remotes by their last two segments. |
| 292 | assert!(remote.ends_with("/g1t-us-1/acme--rocket.git")); |
| 293 | } |
| 294 | |
| 295 | #[test] |
| 296 | fn calls_on_the_namespace_become_requests_for_its_key() { |
| 297 | let create = route("g1t", None, "create", &[json!("acme--rocket"), json!({ "description": "d", "setDefaultBranch": "trunk" })]).unwrap(); |
| 298 | assert_eq!(create.method, "POST"); |
| 299 | assert_eq!(create.path, "/api/repos"); |
| 300 | assert_eq!(create.body.unwrap(), json!({ "name": "g1t/acme--rocket", "description": "d", "defaultBranch": "trunk" })); |
| 301 | let get = route("g1t", None, "get", &[json!("acme--rocket")]).unwrap(); |
| 302 | assert_eq!((get.method, get.path.as_str()), ("GET", "/api/repos/g1t%2Facme--rocket")); |
| 303 | assert_eq!(route("g1t", None, "delete", &[json!("x1")]).unwrap().method, "DELETE"); |
| 304 | assert_eq!(route("g1t", None, "list", &[json!("x1")]).unwrap_err().code, "NOT_SUPPORTED"); |
| 305 | assert_eq!(route("g1t", None, "get", &[]).unwrap_err().code, "INVALID_ARGUMENT"); |
| 306 | } |
| 307 | |
| 308 | #[test] |
| 309 | fn calls_on_a_repository_become_requests_below_it() { |
| 310 | let base = "/api/repos/g1t-us-1%2Fpulls--pul_1"; |
| 311 | let at = |method: &str, args: &[Value]| route("g1t-us-1", Some("pulls--pul_1"), method, args).unwrap(); |
| 312 | assert_eq!(at("info", &[]).path, base); |
| 313 | let token = at("createToken", &[json!("read"), json!(300)]); |
| 314 | assert_eq!(token.body.unwrap(), json!({ "scope": "read", "ttl": 300 })); |
| 315 | assert_eq!(at("log", &[json!({ "ref": "fix/#1", "limit": 20 })]).path, format!("{base}/log?ref=fix%2F%231&limit=20")); |
| 316 | assert_eq!(at("readCommit", &[json!("abc")]).path, format!("{base}/commits/abc")); |
| 317 | assert_eq!(at("readTree", &[json!("abc")]).path, format!("{base}/trees/abc")); |
| 318 | let blob = at("readBlob", &[json!("abc")]); |
| 319 | assert!(blob.bytes); |
| 320 | let file = at("readFile", &[json!({ "ref": "main", "path": "src/a b.rs" })]); |
| 321 | assert_eq!(file.path, format!("{base}/file?ref=main&path=src%2Fa%20b.rs")); |
| 322 | assert!(file.bytes); |
| 323 | let fork = at("fork", &[json!("pulls--pul_2"), json!({ "defaultBranchOnly": true })]); |
| 324 | assert_eq!(fork.body.unwrap(), json!({ "name": "g1t-us-1/pulls--pul_2", "defaultBranchOnly": true })); |
| 325 | } |
| 326 | |
| 327 | #[test] |
| 328 | fn answers_read_as_the_binding_would_have_returned_them() { |
| 329 | let json_route = route("g1t", Some("r"), "readTree", &[json!("a")]).unwrap(); |
| 330 | let blob = route("g1t", Some("r"), "readBlob", &[json!("a")]).unwrap(); |
| 331 | assert_eq!(answer(&json_route, 200, b"[]".to_vec()), Answer::Json(json!([]))); |
| 332 | assert_eq!(answer(&json_route, 200, b"null".to_vec()), Answer::Null); |
| 333 | assert_eq!(answer(&blob, 200, b"hi".to_vec()), Answer::Bytes(b"hi".to_vec())); |
| 334 | assert_eq!(answer(&blob, 404, b"{}".to_vec()), Answer::Null); |
| 335 | assert_eq!( |
| 336 | answer(&json_route, 404, br#"{"code":"NOT_FOUND","message":"no repository g1t/r"}"#.to_vec()), |
| 337 | Answer::Error { code: Some("NOT_FOUND".into()), message: "no repository g1t/r".into() } |
| 338 | ); |
| 339 | assert_eq!( |
| 340 | answer(&json_route, 409, br#"{"code":"ALREADY_EXISTS","message":"x"}"#.to_vec()), |
| 341 | Answer::Error { code: Some("ALREADY_EXISTS".into()), message: "x".into() } |
| 342 | ); |
| 343 | // A 5xx may pass, as the binding's INTERNAL_ERROR. |
| 344 | assert!(matches!(answer(&json_route, 502, Vec::new()), Answer::Error { code: Some(code), .. } if code == "INTERNAL_ERROR")); |
| 345 | assert!(matches!(answer(&json_route, 200, b"<html>".to_vec()), Answer::Error { code: None, .. })); |
| 346 | } |
| 347 | |
| 348 | #[test] |
| 349 | fn writes_are_told_apart_from_reads() { |
| 350 | assert!(writes(None, "create", &[json!("x")])); |
| 351 | assert!(writes(None, "delete", &[json!("x")])); |
| 352 | assert!(!writes(None, "get", &[json!("x")])); |
| 353 | assert!(writes(Some("r"), "fork", &[json!("y")])); |
| 354 | assert!(writes(Some("r"), "createToken", &[json!("write"), json!(60)])); |
| 355 | assert!(!writes(Some("r"), "createToken", &[json!("read"), json!(60)])); |
| 356 | assert!(!writes(Some("r"), "readBlob", &[json!("a")])); |
| 357 | } |
| 358 | } |