Sign in

flagon-io/g1t

public

g1t/services/repos/src/shards.rs

172 lines7,447 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Git storage hardened, pages in tens of milliseconds, honest security alerts, and costs reconciled daily1//! Which git store namespace a repository lives in.
2//!
3//! Cloudflare limits each Artifacts namespace to 2,000 control-plane
4//! requests per 10 seconds, and fixes its jurisdiction (US or EU) when it
5//! is made. So repositories can live in several namespaces, each reached
6//! through its own binding (`ARTIFACTS`, `ARTIFACTS_1`, ..., `ARTIFACTS_EU`),
7//! named in the `ARTIFACTS_NAMESPACES` variable:
8//!
9//! ```json
10//! { "ARTIFACTS": "g1t", "ARTIFACTS_1": "g1t-us-1", "ARTIFACTS_EU": "g1t-eu" }
11//! ```
12//!
13//! A repository's namespace is part of its store key in the registry's
14//! `store` column: `g1t-us-1/acme--rocket`. A key with no namespace
15//! (`acme--rocket`, every repository made before this) is in the namespace
16//! bound to `ARTIFACTS`. A pull request's working copy is always in its
17//! repository's namespace, since Artifacts forks within a namespace.
18//!
19//! New repositories go where `ARTIFACTS_NEW_REPOS` says (a comma-separated
20//! list of namespaces, spread by repository id), and to
21//! `ARTIFACTS_EU_NAMESPACE` for a workspace that keeps its data in the EU
22//! (not offered yet). Without either, everything stays in `ARTIFACTS`'s.
23
24/// The binding every installation has.
25pub const DEFAULT_BINDING: &str = "ARTIFACTS";
26/// Its namespace, unless `ARTIFACTS_NAMESPACES` says otherwise.
27pub const DEFAULT_NAMESPACE: &str = "g1t";
28
29/// Each binding and the namespace it reaches, the default first.
30pub fn bindings(config: Option<&str>) -> Vec<(String, String)> {
31 let mut out: Vec<(String, String)> = config
32 .and_then(|text| serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(text).ok())
33 .map(|map| {
34 map.into_iter()
35 .filter_map(|(binding, namespace)| Some((binding, namespace.as_str()?.trim().to_owned())))
36 .filter(|(_, namespace)| valid_namespace(namespace))
37 .collect()
38 })
39 .unwrap_or_default();
40 if !out.iter().any(|(binding, _)| binding == DEFAULT_BINDING) {
41 out.push((DEFAULT_BINDING.to_owned(), DEFAULT_NAMESPACE.to_owned()));
42 }
43 out.sort_by_key(|(binding, _)| (binding != DEFAULT_BINDING, binding.clone()));
44 out
45}
46
47/// Cloudflare's rule for names: 2 to 63 letters, digits, `.`, `_`, `-`,
48/// starting with a letter or digit.
49pub fn valid_namespace(name: &str) -> bool {
50 (2..=63).contains(&name.len())
51 && name.chars().next().is_some_and(|c| c.is_ascii_alphanumeric())
52 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))
53}
54
55/// A store key's namespace (`None`: the default one) and its name there.
56pub fn split(key: &str) -> (Option<&str>, &str) {
57 match key.split_once('/') {
58 Some((namespace, name)) => (Some(namespace), name),
59 None => (None, key),
60 }
61}
62
63/// The store key for `name` in `namespace`; the default one is left out,
64/// so keys made before sharding read the same.
65pub fn compose(namespace: Option<&str>, name: &str, default: &str) -> String {
66 match namespace {
67 Some(namespace) if namespace != default => format!("{namespace}/{name}"),
68 _ => name.to_owned(),
69 }
70}
71
72/// Where a workspace keeps its data.
73#[derive(Clone, Copy, Debug, PartialEq, Eq)]
74pub enum Residency {
75 Anywhere,
76 Eu,
77}
78
79/// Where new repositories go.
80#[derive(Clone, Debug, Default, PartialEq, Eq)]
81pub struct Placement {
82 /// Namespaces that take new repositories; empty for the default.
83 pub new_repos: Vec<String>,
84 pub eu: Option<String>,
85}
86
87impl Placement {
88 pub fn from_vars(new_repos: Option<&str>, eu: Option<&str>) -> Self {
89 Placement {
90 new_repos: new_repos
91 .unwrap_or_default()
92 .split(',')
93 .map(str::trim)
94 .filter(|name| valid_namespace(name))
95 .map(str::to_owned)
96 .collect(),
97 eu: eu.map(str::trim).filter(|name| valid_namespace(name)).map(str::to_owned),
98 }
99 }
100
101 /// The namespace for a new repository with this id, among those bound
102 /// (`bound`); `None` for the default. A namespace that is named but not
103 /// bound is passed over, so a half-made configuration cannot strand a
104 /// repository.
105 pub fn place(&self, repo_id: &str, residency: Residency, bound: &[String]) -> Option<String> {
106 if residency == Residency::Eu {
107 return self.eu.clone().filter(|eu| bound.contains(eu));
108 }
109 let usable: Vec<&String> = self.new_repos.iter().filter(|name| bound.contains(name)).collect();
110 if usable.is_empty() {
111 return None;
112 }
113 Some(usable[(fnv(repo_id) % usable.len() as u64) as usize].clone())
114 }
115}
116
117/// FNV-1a, to spread ids over namespaces the same way every time.
118fn fnv(text: &str) -> u64 {
119 text.bytes().fold(0xcbf2_9ce4_8422_2325, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3))
120}
121
122#[cfg(test)]
123mod tests {
124 use super::*;
125
126 #[test]
127 fn the_default_binding_is_always_there_and_first() {
128 assert_eq!(bindings(None), vec![("ARTIFACTS".to_owned(), "g1t".to_owned())]);
129 let configured = bindings(Some(r#"{"ARTIFACTS_EU":"g1t-eu","ARTIFACTS_1":"g1t-us-1","ARTIFACTS":"g1t","ARTIFACTS_2":"bad name!"}"#));
130 assert_eq!(
131 configured,
132 vec![
133 ("ARTIFACTS".to_owned(), "g1t".to_owned()),
134 ("ARTIFACTS_1".to_owned(), "g1t-us-1".to_owned()),
135 ("ARTIFACTS_EU".to_owned(), "g1t-eu".to_owned()),
136 ]
137 );
138 assert_eq!(bindings(Some("not json")), bindings(None));
139 }
140
141 #[test]
142 fn keys_name_their_namespace_unless_it_is_the_default() {
143 assert_eq!(split("acme--rocket"), (None, "acme--rocket"));
144 assert_eq!(split("g1t-us-1/acme--rocket"), (Some("g1t-us-1"), "acme--rocket"));
145 assert_eq!(compose(None, "acme--rocket", "g1t"), "acme--rocket");
146 assert_eq!(compose(Some("g1t"), "acme--rocket", "g1t"), "acme--rocket");
147 assert_eq!(compose(Some("g1t-us-1"), "pulls--pul_1", "g1t"), "g1t-us-1/pulls--pul_1");
148 }
149
150 #[test]
151 fn new_repositories_spread_over_the_bound_namespaces() {
152 let bound = vec!["g1t".to_owned(), "g1t-us-1".to_owned(), "g1t-us-2".to_owned(), "g1t-eu".to_owned()];
153 // Nothing configured: everything stays where it was.
154 assert_eq!(Placement::default().place("rep_1", Residency::Anywhere, &bound), None);
155 let placement = Placement::from_vars(Some("g1t-us-1, g1t-us-2,g1t-us-3"), Some("g1t-eu"));
156 let mut seen = std::collections::HashMap::new();
157 for n in 0..1000 {
158 let id = format!("rep_{n:05}");
159 let placed = placement.place(&id, Residency::Anywhere, &bound).unwrap();
160 // The same id always lands in the same place.
161 assert_eq!(placement.place(&id, Residency::Anywhere, &bound).unwrap(), placed);
162 *seen.entry(placed).or_insert(0) += 1;
163 }
164 // g1t-us-3 is named but not bound, so it takes nothing.
165 assert_eq!(seen.len(), 2);
166 assert!(seen.values().all(|count| *count > 400));
167 assert_eq!(placement.place("rep_1", Residency::Eu, &bound).as_deref(), Some("g1t-eu"));
168 // EU asked for but not bound: nowhere, so the caller can refuse.
169 assert_eq!(placement.place("rep_1", Residency::Eu, &bound[..3]), None);
170 assert!(valid_namespace("g1t") && !valid_namespace("-g1t") && !valid_namespace("x"));
171 }
172}