Sign in

flagon-io/g1t

public

g1t/services/repos/src/shards.rs

172 lines7,447 bytesCodeBlame
1//! Which git store namespace a repository lives in.
2//!
3//! Cloudflare limits each Artifacts namespace to 2,000 control-plane
4//! requests per 10 seconds, and fixes its jurisdiction (US or EU) when it
5//! is made. So repositories can live in several namespaces, each reached
6//! through its own binding (`ARTIFACTS`, `ARTIFACTS_1`, ..., `ARTIFACTS_EU`),
7//! named in the `ARTIFACTS_NAMESPACES` variable:
8//!
9//! ```json
10//! { "ARTIFACTS": "g1t", "ARTIFACTS_1": "g1t-us-1", "ARTIFACTS_EU": "g1t-eu" }
11//! ```
12//!
13//! A repository's namespace is part of its store key in the registry's
14//! `store` column: `g1t-us-1/acme--rocket`. A key with no namespace
15//! (`acme--rocket`, every repository made before this) is in the namespace
16//! bound to `ARTIFACTS`. A pull request's working copy is always in its
17//! repository's namespace, since Artifacts forks within a namespace.
18//!
19//! New repositories go where `ARTIFACTS_NEW_REPOS` says (a comma-separated
20//! list of namespaces, spread by repository id), and to
21//! `ARTIFACTS_EU_NAMESPACE` for a workspace that keeps its data in the EU
22//! (not offered yet). Without either, everything stays in `ARTIFACTS`'s.
23
24/// The binding every installation has.
25pub const DEFAULT_BINDING: &str = "ARTIFACTS";
26/// Its namespace, unless `ARTIFACTS_NAMESPACES` says otherwise.
27pub const DEFAULT_NAMESPACE: &str = "g1t";
28
29/// Each binding and the namespace it reaches, the default first.
30pub fn bindings(config: Option<&str>) -> Vec<(String, String)> {
31 let mut out: Vec<(String, String)> = config
32 .and_then(|text| serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(text).ok())
33 .map(|map| {
34 map.into_iter()
35 .filter_map(|(binding, namespace)| Some((binding, namespace.as_str()?.trim().to_owned())))
36 .filter(|(_, namespace)| valid_namespace(namespace))
37 .collect()
38 })
39 .unwrap_or_default();
40 if !out.iter().any(|(binding, _)| binding == DEFAULT_BINDING) {
41 out.push((DEFAULT_BINDING.to_owned(), DEFAULT_NAMESPACE.to_owned()));
42 }
43 out.sort_by_key(|(binding, _)| (binding != DEFAULT_BINDING, binding.clone()));
44 out
45}
46
47/// Cloudflare's rule for names: 2 to 63 letters, digits, `.`, `_`, `-`,
48/// starting with a letter or digit.
49pub fn valid_namespace(name: &str) -> bool {
50 (2..=63).contains(&name.len())
51 && name.chars().next().is_some_and(|c| c.is_ascii_alphanumeric())
52 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))
53}
54
55/// A store key's namespace (`None`: the default one) and its name there.
56pub fn split(key: &str) -> (Option<&str>, &str) {
57 match key.split_once('/') {
58 Some((namespace, name)) => (Some(namespace), name),
59 None => (None, key),
60 }
61}
62
63/// The store key for `name` in `namespace`; the default one is left out,
64/// so keys made before sharding read the same.
65pub fn compose(namespace: Option<&str>, name: &str, default: &str) -> String {
66 match namespace {
67 Some(namespace) if namespace != default => format!("{namespace}/{name}"),
68 _ => name.to_owned(),
69 }
70}
71
72/// Where a workspace keeps its data.
73#[derive(Clone, Copy, Debug, PartialEq, Eq)]
74pub enum Residency {
75 Anywhere,
76 Eu,
77}
78
79/// Where new repositories go.
80#[derive(Clone, Debug, Default, PartialEq, Eq)]
81pub struct Placement {
82 /// Namespaces that take new repositories; empty for the default.
83 pub new_repos: Vec<String>,
84 pub eu: Option<String>,
85}
86
87impl Placement {
88 pub fn from_vars(new_repos: Option<&str>, eu: Option<&str>) -> Self {
89 Placement {
90 new_repos: new_repos
91 .unwrap_or_default()
92 .split(',')
93 .map(str::trim)
94 .filter(|name| valid_namespace(name))
95 .map(str::to_owned)
96 .collect(),
97 eu: eu.map(str::trim).filter(|name| valid_namespace(name)).map(str::to_owned),
98 }
99 }
100
101 /// The namespace for a new repository with this id, among those bound
102 /// (`bound`); `None` for the default. A namespace that is named but not
103 /// bound is passed over, so a half-made configuration cannot strand a
104 /// repository.
105 pub fn place(&self, repo_id: &str, residency: Residency, bound: &[String]) -> Option<String> {
106 if residency == Residency::Eu {
107 return self.eu.clone().filter(|eu| bound.contains(eu));
108 }
109 let usable: Vec<&String> = self.new_repos.iter().filter(|name| bound.contains(name)).collect();
110 if usable.is_empty() {
111 return None;
112 }
113 Some(usable[(fnv(repo_id) % usable.len() as u64) as usize].clone())
114 }
115}
116
117/// FNV-1a, to spread ids over namespaces the same way every time.
118fn fnv(text: &str) -> u64 {
119 text.bytes().fold(0xcbf2_9ce4_8422_2325, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3))
120}
121
122#[cfg(test)]
123mod tests {
124 use super::*;
125
126 #[test]
127 fn the_default_binding_is_always_there_and_first() {
128 assert_eq!(bindings(None), vec![("ARTIFACTS".to_owned(), "g1t".to_owned())]);
129 let configured = bindings(Some(r#"{"ARTIFACTS_EU":"g1t-eu","ARTIFACTS_1":"g1t-us-1","ARTIFACTS":"g1t","ARTIFACTS_2":"bad name!"}"#));
130 assert_eq!(
131 configured,
132 vec![
133 ("ARTIFACTS".to_owned(), "g1t".to_owned()),
134 ("ARTIFACTS_1".to_owned(), "g1t-us-1".to_owned()),
135 ("ARTIFACTS_EU".to_owned(), "g1t-eu".to_owned()),
136 ]
137 );
138 assert_eq!(bindings(Some("not json")), bindings(None));
139 }
140
141 #[test]
142 fn keys_name_their_namespace_unless_it_is_the_default() {
143 assert_eq!(split("acme--rocket"), (None, "acme--rocket"));
144 assert_eq!(split("g1t-us-1/acme--rocket"), (Some("g1t-us-1"), "acme--rocket"));
145 assert_eq!(compose(None, "acme--rocket", "g1t"), "acme--rocket");
146 assert_eq!(compose(Some("g1t"), "acme--rocket", "g1t"), "acme--rocket");
147 assert_eq!(compose(Some("g1t-us-1"), "pulls--pul_1", "g1t"), "g1t-us-1/pulls--pul_1");
148 }
149
150 #[test]
151 fn new_repositories_spread_over_the_bound_namespaces() {
152 let bound = vec!["g1t".to_owned(), "g1t-us-1".to_owned(), "g1t-us-2".to_owned(), "g1t-eu".to_owned()];
153 // Nothing configured: everything stays where it was.
154 assert_eq!(Placement::default().place("rep_1", Residency::Anywhere, &bound), None);
155 let placement = Placement::from_vars(Some("g1t-us-1, g1t-us-2,g1t-us-3"), Some("g1t-eu"));
156 let mut seen = std::collections::HashMap::new();
157 for n in 0..1000 {
158 let id = format!("rep_{n:05}");
159 let placed = placement.place(&id, Residency::Anywhere, &bound).unwrap();
160 // The same id always lands in the same place.
161 assert_eq!(placement.place(&id, Residency::Anywhere, &bound).unwrap(), placed);
162 *seen.entry(placed).or_insert(0) += 1;
163 }
164 // g1t-us-3 is named but not bound, so it takes nothing.
165 assert_eq!(seen.len(), 2);
166 assert!(seen.values().all(|count| *count > 400));
167 assert_eq!(placement.place("rep_1", Residency::Eu, &bound).as_deref(), Some("g1t-eu"));
168 // EU asked for but not bound: nowhere, so the caller can refuse.
169 assert_eq!(placement.place("rep_1", Residency::Eu, &bound[..3]), None);
170 assert!(valid_namespace("g1t") && !valid_namespace("-g1t") && !valid_namespace("x"));
171 }
172}