| 1 | //! Mirroring: a repository's links to copies of it on other hosts (see |
| 2 | //! `g1t_contracts::mirrors` for the model). |
| 3 | //! |
| 4 | //! Every linked repository has exactly one leader. A **mirror** follows a |
| 5 | //! remote that leads: it stands by as an exact, read-only copy, and nothing |
| 6 | //! runs on it. Someone can turn on **CI failover** (the remote keeps the |
| 7 | //! code, g1t runs its workflows) or **take over** (g1t leads for a while). |
| 8 | //! A takeover is **handed back** ref by ref: what only g1t changed is |
| 9 | //! pushed, a protected branch goes as a pull request, and a branch both |
| 10 | //! sides changed waits for a person's decision. A mirror can also be |
| 11 | //! **moved to g1t** for good, after which g1t no longer tracks the remote. |
| 12 | //! A repository g1t leads can be **mirrored to** any number of followers. |
| 13 | //! |
| 14 | //! This service keeps the links (`remotes`), each host's health |
| 15 | //! (`remote_hosts`) and a takeover's starting point (`remote_refs`), and |
| 16 | //! decides. The repos service keeps each repository's `RepoMirror`, set |
| 17 | //! here with `set_mirror`, so pushes and merges are refused or allowed |
| 18 | //! without asking. Git itself moves through the repos service (`mirror`, |
| 19 | //! `mirror_refs`, `mirror_apply`). |
| 20 | //! |
| 21 | //! Hosts are adapters: [`RemoteProvider`] names them, and the few places a |
| 22 | //! host's own API is used (credentials, whether a branch is protected, |
| 23 | //! opening a pull request) match on it. GitHub goes through g1t's GitHub |
| 24 | //! App; another g1t or any git host takes a username and token. A host |
| 25 | //! that sends no webhook is polled. |
| 26 | //! |
| 27 | //! Nothing happens on its own unless someone asked for it in the link's |
| 28 | //! settings: by default an unreachable remote is only shown on the |
| 29 | //! repository, and a takeover starts when someone starts it. |
| 30 | |
| 31 | use std::collections::{BTreeMap, BTreeSet, HashMap}; |
| 32 | |
| 33 | use g1t_contracts::access::{self, Capability}; |
| 34 | use g1t_contracts::events::{NewEvent, Publish}; |
| 35 | use g1t_contracts::identity::{ListMembersArgs, Member}; |
| 36 | use g1t_contracts::mirrors::*; |
| 37 | use g1t_contracts::repos::{ |
| 38 | GetByIdArgs, MirrorApplied, MirrorApplyArgs, MirrorArgs, MirrorDirection, MirrorRefs, MirrorRefsArgs, Mirrored, RefMove, |
| 39 | Repo, SetMirrorArgs, |
| 40 | }; |
| 41 | use g1t_contracts::time::rfc3339; |
| 42 | use g1t_contracts::{FailureCode, Outcome, Role, User, new_id}; |
| 43 | use g1t_kit::{args, now_ms, reply}; |
| 44 | use g1t_secrets::Sealer; |
| 45 | use serde::Deserialize; |
| 46 | use serde_json::Value; |
| 47 | use worker::wasm_bindgen::JsValue; |
| 48 | use worker::{Context, D1Database, Env, Fetcher, Method, Response, Result}; |
| 49 | |
| 50 | use crate::github::GithubApp; |
| 51 | use crate::http; |
| 52 | |
| 53 | const SOURCE: &str = "integrations"; |
| 54 | /// A host is unreachable after this many failed checks in a row, spanning |
| 55 | /// at least [`DOWN_AFTER_MS`]. |
| 56 | const DOWN_CHECKS: u32 = 3; |
| 57 | const DOWN_AFTER_MS: u64 = 2 * 60 * 1000; |
| 58 | /// And reachable again after this many good ones, spanning [`UP_AFTER_MS`]. |
| 59 | const UP_CHECKS: u32 = 3; |
| 60 | const UP_AFTER_MS: u64 = 5 * 60 * 1000; |
| 61 | /// How often a remote with no webhook is asked for news. |
| 62 | const POLL_MS: u64 = 5 * 60 * 1000; |
| 63 | /// Where g1t's commits go when the remote will not take them directly. |
| 64 | const HANDBACK_PREFIX: &str = "refs/heads/g1t/handback/"; |
| 65 | /// The most branches asked about protection in one plan. |
| 66 | const MAX_PROTECTION_CHECKS: usize = 20; |
| 67 | |
| 68 | fn fail<T>(code: FailureCode, message: impl Into<String>) -> Outcome<T> { |
| 69 | Outcome::fail(code, message) |
| 70 | } |
| 71 | |
| 72 | fn null_or(value: Option<&str>) -> JsValue { |
| 73 | value.map_or(JsValue::NULL, Into::into) |
| 74 | } |
| 75 | |
| 76 | // --- Pure parts, tested below ---------------------------------------------- |
| 77 | |
| 78 | /// The host of an https address, lowercased: `github.com`. |
| 79 | pub fn host_of(url: &str) -> String { |
| 80 | url.trim() |
| 81 | .trim_start_matches("https://") |
| 82 | .trim_start_matches("http://") |
| 83 | .split(['/', '?', '#']) |
| 84 | .next() |
| 85 | .unwrap_or_default() |
| 86 | .rsplit('@') |
| 87 | .next() |
| 88 | .unwrap_or_default() |
| 89 | .to_lowercase() |
| 90 | } |
| 91 | |
| 92 | /// A remote as people name it: `github.com/acme/web`. |
| 93 | pub fn display_name(url: &str) -> String { |
| 94 | let rest = url |
| 95 | .trim() |
| 96 | .trim_start_matches("https://") |
| 97 | .trim_start_matches("http://") |
| 98 | .trim_end_matches('/'); |
| 99 | let rest = rest.strip_suffix(".git").unwrap_or(rest); |
| 100 | let rest = rest.rsplit_once('@').map_or(rest, |(_, after)| after); |
| 101 | let (host, path) = rest.split_once('/').unwrap_or((rest, "")); |
| 102 | if path.is_empty() { host.to_lowercase() } else { format!("{}/{path}", host.to_lowercase()) } |
| 103 | } |
| 104 | |
| 105 | /// A clone address: https, with `.git` added when the host leaves it off, |
| 106 | /// and no credentials in it. |
| 107 | pub fn clean_clone_url(url: &str) -> Option<String> { |
| 108 | let url = url.trim().trim_end_matches('/'); |
| 109 | let rest = url.strip_prefix("https://")?; |
| 110 | if rest.contains('@') || rest.contains(' ') || !rest.contains('/') || rest.starts_with('/') { |
| 111 | return None; |
| 112 | } |
| 113 | Some(if url.ends_with(".git") { url.to_owned() } else { format!("{url}.git") }) |
| 114 | } |
| 115 | |
| 116 | /// The web address of a clone address. |
| 117 | pub fn web_url(clone_url: &str) -> String { |
| 118 | clone_url.strip_suffix(".git").unwrap_or(clone_url).to_owned() |
| 119 | } |
| 120 | |
| 121 | /// How a host's health moves with one more check. `now` in milliseconds. |
| 122 | #[derive(Clone, Debug, Default, PartialEq, Eq)] |
| 123 | pub struct HostHealth { |
| 124 | pub failures: u32, |
| 125 | pub successes: u32, |
| 126 | /// When the current run of failures or successes began. |
| 127 | pub streak_ms: u64, |
| 128 | pub unreachable_since: Option<u64>, |
| 129 | } |
| 130 | |
| 131 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 132 | pub enum Change { |
| 133 | None, |
| 134 | WentDown, |
| 135 | CameBack, |
| 136 | } |
| 137 | |
| 138 | impl HostHealth { |
| 139 | pub fn reachable(&self) -> bool { |
| 140 | self.unreachable_since.is_none() |
| 141 | } |
| 142 | |
| 143 | pub fn checked(&self, answered: bool, now: u64) -> (HostHealth, Change) { |
| 144 | let mut next = self.clone(); |
| 145 | if answered { |
| 146 | if next.successes == 0 { |
| 147 | next.streak_ms = now; |
| 148 | } |
| 149 | next.successes += 1; |
| 150 | next.failures = 0; |
| 151 | if next.unreachable_since.is_some() |
| 152 | && next.successes >= UP_CHECKS |
| 153 | && now.saturating_sub(next.streak_ms) >= UP_AFTER_MS |
| 154 | { |
| 155 | next.unreachable_since = None; |
| 156 | return (next, Change::CameBack); |
| 157 | } |
| 158 | } else { |
| 159 | if next.failures == 0 { |
| 160 | next.streak_ms = now; |
| 161 | } |
| 162 | next.failures += 1; |
| 163 | next.successes = 0; |
| 164 | if next.unreachable_since.is_none() |
| 165 | && next.failures >= DOWN_CHECKS |
| 166 | && now.saturating_sub(next.streak_ms) >= DOWN_AFTER_MS |
| 167 | { |
| 168 | next.unreachable_since = Some(now); |
| 169 | return (next, Change::WentDown); |
| 170 | } |
| 171 | } |
| 172 | (next, Change::None) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | /// The branch name in a ref: `main` for `refs/heads/main`. |
| 177 | fn branch_of(git_ref: &str) -> Option<&str> { |
| 178 | git_ref.strip_prefix("refs/heads/") |
| 179 | } |
| 180 | |
| 181 | /// The moves that carry out a hand-back plan, and the refs that go as pull |
| 182 | /// requests. `theirs_all` is every ref the remote has, so a hand-back |
| 183 | /// branch made before is moved from where it is. |
| 184 | pub fn hand_back_moves(plan: &HandbackPlan, theirs_all: &BTreeMap<String, String>) -> (Vec<RefMove>, Vec<String>) { |
| 185 | let mut moves = Vec::new(); |
| 186 | let mut pull_requests = Vec::new(); |
| 187 | for item in &plan.refs { |
| 188 | let push = |moves: &mut Vec<RefMove>| { |
| 189 | moves.push(RefMove { |
| 190 | direction: MirrorDirection::Push, |
| 191 | git_ref: item.git_ref.clone(), |
| 192 | to: None, |
| 193 | old: item.theirs.clone(), |
| 194 | new: item.ours.clone(), |
| 195 | }) |
| 196 | }; |
| 197 | let fetch = |moves: &mut Vec<RefMove>| { |
| 198 | moves.push(RefMove { |
| 199 | direction: MirrorDirection::Pull, |
| 200 | git_ref: item.git_ref.clone(), |
| 201 | to: None, |
| 202 | old: item.ours.clone(), |
| 203 | new: item.theirs.clone(), |
| 204 | }) |
| 205 | }; |
| 206 | let decision = match item.action { |
| 207 | RefAction::Diverged => item.decision, |
| 208 | RefAction::PullRequest => Some(RefDecision::PullRequest), |
| 209 | _ => None, |
| 210 | }; |
| 211 | match (item.action, decision) { |
| 212 | (RefAction::Same, _) => {} |
| 213 | (RefAction::Push, _) | (RefAction::Diverged, Some(RefDecision::KeepOurs)) => push(&mut moves), |
| 214 | (RefAction::Fetch, _) | (RefAction::Diverged, Some(RefDecision::KeepTheirs)) => fetch(&mut moves), |
| 215 | (_, Some(RefDecision::PullRequest)) => { |
| 216 | let Some(branch) = branch_of(&item.git_ref) else { |
| 217 | // Only branches can be pulled; a tag goes as it is. |
| 218 | push(&mut moves); |
| 219 | continue; |
| 220 | }; |
| 221 | if let Some(ours) = &item.ours { |
| 222 | let target = format!("{HANDBACK_PREFIX}{branch}"); |
| 223 | moves.push(RefMove { |
| 224 | direction: MirrorDirection::Push, |
| 225 | git_ref: item.git_ref.clone(), |
| 226 | to: Some(target.clone()), |
| 227 | old: theirs_all.get(&target).cloned(), |
| 228 | new: Some(ours.clone()), |
| 229 | }); |
| 230 | pull_requests.push(item.git_ref.clone()); |
| 231 | } |
| 232 | // g1t follows the remote's branch; its commits are on the |
| 233 | // hand-back branch and kept under refs/g1t/replaced/. |
| 234 | fetch(&mut moves); |
| 235 | } |
| 236 | (RefAction::Diverged, None) | (RefAction::PullRequest, _) => {} |
| 237 | } |
| 238 | } |
| 239 | (moves, pull_requests) |
| 240 | } |
| 241 | |
| 242 | /// Builds the plan for a hand-back from both sides' refs and what each ref |
| 243 | /// was when the takeover began. `protected` names branches the remote |
| 244 | /// protects. |
| 245 | pub fn plan_refs( |
| 246 | bases: &BTreeMap<String, (Option<String>, Option<RefDecision>)>, |
| 247 | ours: &BTreeMap<String, String>, |
| 248 | theirs: &BTreeMap<String, String>, |
| 249 | protected: &BTreeSet<String>, |
| 250 | ) -> Vec<RefPlan> { |
| 251 | let names: BTreeSet<&String> = bases.keys().chain(ours.keys()).chain(theirs.keys()).collect(); |
| 252 | names |
| 253 | .into_iter() |
| 254 | .filter(|name| !name.starts_with(HANDBACK_PREFIX)) |
| 255 | .filter(|name| name.starts_with("refs/heads/") || name.starts_with("refs/tags/")) |
| 256 | .filter_map(|name| { |
| 257 | let (base, decision) = bases.get(name).cloned().unwrap_or((None, None)); |
| 258 | let ours = ours.get(name).cloned(); |
| 259 | let theirs = theirs.get(name).cloned(); |
| 260 | // A ref g1t never had and the remote made since is copied in; a |
| 261 | // ref neither had at the start nor has now is nothing. |
| 262 | let action = ref_action(base.as_deref(), ours.as_deref(), theirs.as_deref(), protected.contains(name)); |
| 263 | (action != RefAction::Same).then(|| RefPlan { |
| 264 | git_ref: name.clone(), |
| 265 | base, |
| 266 | ours, |
| 267 | theirs, |
| 268 | action, |
| 269 | decision: if action == RefAction::Diverged { decision } else { None }, |
| 270 | }) |
| 271 | }) |
| 272 | .collect() |
| 273 | } |
| 274 | |
| 275 | // --- Rows ------------------------------------------------------------------- |
| 276 | |
| 277 | #[derive(Clone, Debug, Deserialize)] |
| 278 | pub(crate) struct RemoteRow { |
| 279 | pub id: String, |
| 280 | pub repo_id: String, |
| 281 | pub workspace: String, |
| 282 | pub repo: String, |
| 283 | pub provider: String, |
| 284 | pub role: String, |
| 285 | pub name: String, |
| 286 | pub url: String, |
| 287 | pub clone_url: String, |
| 288 | pub connection_id: Option<String>, |
| 289 | pub username: Option<String>, |
| 290 | pub credential: Option<String>, |
| 291 | pub state: String, |
| 292 | pub state_since: String, |
| 293 | pub state_by: Option<String>, |
| 294 | pub settings: String, |
| 295 | pub synced_at: Option<String>, |
| 296 | pub last_error: Option<String>, |
| 297 | pub created_at: String, |
| 298 | } |
| 299 | |
| 300 | impl RemoteRow { |
| 301 | fn provider(&self) -> RemoteProvider { |
| 302 | RemoteProvider::parse(&self.provider).unwrap_or(RemoteProvider::Git) |
| 303 | } |
| 304 | |
| 305 | fn leads(&self) -> bool { |
| 306 | self.role == RemoteRole::Leader.as_str() |
| 307 | } |
| 308 | |
| 309 | fn state(&self) -> RemoteState { |
| 310 | RemoteState::parse(&self.state) |
| 311 | } |
| 312 | |
| 313 | fn settings(&self) -> MirrorSettings { |
| 314 | serde_json::from_str(&self.settings).unwrap_or_default() |
| 315 | } |
| 316 | |
| 317 | fn host(&self) -> String { |
| 318 | host_of(&self.clone_url) |
| 319 | } |
| 320 | |
| 321 | /// `owner/name` on GitHub. |
| 322 | fn full_name(&self) -> &str { |
| 323 | self.url.trim_start_matches("https://github.com/") |
| 324 | } |
| 325 | |
| 326 | /// What the repos service keeps for a mirror of this remote. |
| 327 | fn repo_mirror(&self) -> Option<RepoMirror> { |
| 328 | if !self.leads() { |
| 329 | return None; |
| 330 | } |
| 331 | let settings = self.settings(); |
| 332 | Some(RepoMirror { |
| 333 | state: self.state().mirror().unwrap_or_default(), |
| 334 | remote: self.name.clone(), |
| 335 | url: self.url.clone(), |
| 336 | since: self.state_since.clone(), |
| 337 | warm: settings.keep_ci_warm, |
| 338 | github_workflows: settings.github_workflows, |
| 339 | hold_deploys: settings.hold_deploys, |
| 340 | }) |
| 341 | } |
| 342 | |
| 343 | fn contract(&self, health: Option<&HostRow>) -> Remote { |
| 344 | Remote { |
| 345 | id: self.id.clone(), |
| 346 | repo_id: self.repo_id.clone(), |
| 347 | repo: self.repo.clone(), |
| 348 | provider: self.provider(), |
| 349 | role: if self.leads() { RemoteRole::Leader } else { RemoteRole::Follower }, |
| 350 | name: self.name.clone(), |
| 351 | url: self.url.clone(), |
| 352 | state: self.state(), |
| 353 | state_since: self.state_since.clone(), |
| 354 | state_by: self.state_by.clone(), |
| 355 | reachable: health.is_none_or(|health| health.unreachable_since.is_none()), |
| 356 | unreachable_since: health.and_then(|health| health.unreachable_since.clone()), |
| 357 | synced_at: self.synced_at.clone(), |
| 358 | last_error: self.last_error.clone(), |
| 359 | settings: self.settings(), |
| 360 | created_at: self.created_at.clone(), |
| 361 | } |
| 362 | } |
| 363 | |
| 364 | fn link(&self) -> String { |
| 365 | format!("/{}/settings/mirroring", self.repo) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | #[derive(Clone, Debug, Default, Deserialize)] |
| 370 | pub(crate) struct HostRow { |
| 371 | pub host: String, |
| 372 | pub failures: u32, |
| 373 | pub successes: u32, |
| 374 | pub streak_ms: f64, |
| 375 | pub unreachable_since: Option<String>, |
| 376 | } |
| 377 | |
| 378 | impl HostRow { |
| 379 | fn health(&self) -> HostHealth { |
| 380 | HostHealth { |
| 381 | failures: self.failures, |
| 382 | successes: self.successes, |
| 383 | streak_ms: self.streak_ms as u64, |
| 384 | unreachable_since: self.unreachable_since.as_deref().and_then(crate::github::parse_time), |
| 385 | } |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | /// Who is acting: a person, or g1t on its own (an automatic takeover or |
| 390 | /// hand-back), which needs no role. |
| 391 | enum By<'a> { |
| 392 | Person(&'a User), |
| 393 | G1t, |
| 394 | } |
| 395 | |
| 396 | impl By<'_> { |
| 397 | fn name(&self) -> String { |
| 398 | match self { |
| 399 | By::Person(user) => user.username.clone(), |
| 400 | By::G1t => "g1t".to_owned(), |
| 401 | } |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | pub struct Mirrors { |
| 406 | db: D1Database, |
| 407 | sealer: Option<Sealer>, |
| 408 | repos: Fetcher, |
| 409 | identity: Fetcher, |
| 410 | events: Option<Fetcher>, |
| 411 | github: GithubApp, |
| 412 | } |
| 413 | |
| 414 | impl Mirrors { |
| 415 | pub fn new(env: &Env) -> Result<Self> { |
| 416 | Ok(Mirrors { |
| 417 | db: env.d1("DB")?, |
| 418 | sealer: env.secret("INTEGRATIONS_KEY").ok().and_then(|key| Sealer::new(&key.to_string())), |
| 419 | repos: env.service("REPOS")?, |
| 420 | identity: env.service("IDENTITY")?, |
| 421 | events: env.service("EVENTS").ok(), |
| 422 | github: GithubApp::new(env)?, |
| 423 | }) |
| 424 | } |
| 425 | |
| 426 | // --- Reading -------------------------------------------------------------- |
| 427 | |
| 428 | async fn rows(&self, repo_id: &str) -> Result<Vec<RemoteRow>> { |
| 429 | self.db |
| 430 | .prepare("SELECT * FROM remotes WHERE repo_id = ? ORDER BY role, created_at") |
| 431 | .bind(&[repo_id.into()])? |
| 432 | .all() |
| 433 | .await? |
| 434 | .results::<RemoteRow>() |
| 435 | } |
| 436 | |
| 437 | async fn row(&self, id: &str) -> Result<Option<RemoteRow>> { |
| 438 | self.db.prepare("SELECT * FROM remotes WHERE id = ?").bind(&[id.into()])?.first::<RemoteRow>(None).await |
| 439 | } |
| 440 | |
| 441 | async fn leader(&self, repo_id: &str) -> Result<Option<RemoteRow>> { |
| 442 | self.db |
| 443 | .prepare("SELECT * FROM remotes WHERE repo_id = ? AND role = 'leader'") |
| 444 | .bind(&[repo_id.into()])? |
| 445 | .first::<RemoteRow>(None) |
| 446 | .await |
| 447 | } |
| 448 | |
| 449 | async fn hosts(&self) -> Result<HashMap<String, HostRow>> { |
| 450 | Ok(self |
| 451 | .db |
| 452 | .prepare("SELECT * FROM remote_hosts") |
| 453 | .all() |
| 454 | .await? |
| 455 | .results::<HostRow>()? |
| 456 | .into_iter() |
| 457 | .map(|row| (row.host.clone(), row)) |
| 458 | .collect()) |
| 459 | } |
| 460 | |
| 461 | async fn repo(&self, repo_id: &str, viewer: Option<&User>) -> Result<Option<Repo>> { |
| 462 | let viewer = viewer.cloned().or_else(|| Some(User::system("g1t"))); |
| 463 | let found: Outcome<Repo> = |
| 464 | g1t_kit::call(&self.repos, "get_by_id", &GetByIdArgs { id: repo_id.to_owned(), viewer }).await?; |
| 465 | Ok(found.into_result().ok()) |
| 466 | } |
| 467 | |
| 468 | /// Why `actor` may not change `repo`'s links, if they may not. |
| 469 | fn refused<T>(actor: &User, repo: &Repo, capability: Capability) -> Option<Outcome<T>> { |
| 470 | let target = access::RepoRef { id: &repo.id, namespace: &repo.namespace, private: repo.is_private }; |
| 471 | match access::check(Some(actor), target, capability) { |
| 472 | Ok(()) => None, |
| 473 | Err(access::Denied::NotFound) => Some(fail(FailureCode::NotFound, "Repository not found.")), |
| 474 | Err(access::Denied::Forbidden) => { |
| 475 | Some(fail(FailureCode::Forbidden, access::needs(capability, &format!("{}/{}", repo.namespace, repo.name)))) |
| 476 | } |
| 477 | } |
| 478 | } |
| 479 | |
| 480 | /// The repository and its leader, for someone changing them. |
| 481 | async fn leader_for(&self, actor: &User, repo_id: &str) -> Result<std::result::Result<(Repo, RemoteRow), Outcome<MirrorView>>> { |
| 482 | let Some(repo) = self.repo(repo_id, Some(actor)).await? else { |
| 483 | return Ok(Err(fail(FailureCode::NotFound, "Repository not found."))); |
| 484 | }; |
| 485 | if let Some(refused) = Self::refused(actor, &repo, Capability::ManageIntegrations) { |
| 486 | return Ok(Err(refused)); |
| 487 | } |
| 488 | let Some(row) = self.leader(repo_id).await? else { |
| 489 | return Ok(Err(fail(FailureCode::NotFound, format!("{}/{} is not a mirror.", repo.namespace, repo.name)))); |
| 490 | }; |
| 491 | Ok(Ok((repo, row))) |
| 492 | } |
| 493 | |
| 494 | async fn view_of(&self, repo_id: &str, can_manage: bool, plan: Option<HandbackPlan>) -> Result<MirrorView> { |
| 495 | let hosts = self.hosts().await?; |
| 496 | Ok(MirrorView { |
| 497 | remotes: self.rows(repo_id).await?.iter().map(|row| row.contract(hosts.get(&row.host()))).collect(), |
| 498 | plan, |
| 499 | can_manage, |
| 500 | notes: Vec::new(), |
| 501 | }) |
| 502 | } |
| 503 | |
| 504 | pub async fn view(&self, a: MirrorViewArgs) -> Result<Outcome<MirrorView>> { |
| 505 | let Some(repo) = self.repo(&a.repo_id, a.viewer.as_ref()).await? else { |
| 506 | return Ok(fail(FailureCode::NotFound, "Repository not found.")); |
| 507 | }; |
| 508 | let can_manage = a |
| 509 | .viewer |
| 510 | .as_ref() |
| 511 | .is_some_and(|viewer| Self::refused::<()>(viewer, &repo, Capability::ManageIntegrations).is_none()); |
| 512 | Ok(Outcome::Ok(self.view_of(&repo.id, can_manage, None).await?)) |
| 513 | } |
| 514 | |
| 515 | pub async fn briefs(&self, a: MirrorBriefsArgs) -> Result<Vec<RemoteBrief>> { |
| 516 | let ids: Vec<&String> = a.repo_ids.iter().take(200).collect(); |
| 517 | if ids.is_empty() { |
| 518 | return Ok(Vec::new()); |
| 519 | } |
| 520 | let marks = vec!["?"; ids.len()].join(", "); |
| 521 | let bind: Vec<JsValue> = ids.iter().map(|id| id.as_str().into()).collect(); |
| 522 | let rows = self |
| 523 | .db |
| 524 | .prepare(format!("SELECT * FROM remotes WHERE repo_id IN ({marks}) ORDER BY role, created_at")) |
| 525 | .bind(&bind)? |
| 526 | .all() |
| 527 | .await? |
| 528 | .results::<RemoteRow>()?; |
| 529 | let hosts = self.hosts().await?; |
| 530 | Ok(rows |
| 531 | .iter() |
| 532 | .map(|row| RemoteBrief { |
| 533 | repo_id: row.repo_id.clone(), |
| 534 | role: if row.leads() { RemoteRole::Leader } else { RemoteRole::Follower }, |
| 535 | name: row.name.clone(), |
| 536 | state: row.state(), |
| 537 | reachable: hosts.get(&row.host()).is_none_or(|host| host.unreachable_since.is_none()), |
| 538 | }) |
| 539 | .collect()) |
| 540 | } |
| 541 | |
| 542 | // --- Talking to hosts ----------------------------------------------------- |
| 543 | |
| 544 | /// The user and token a remote is opened with. |
| 545 | async fn credential(&self, row: &RemoteRow) -> Result<std::result::Result<(Option<String>, String), String>> { |
| 546 | match row.provider() { |
| 547 | RemoteProvider::Github => { |
| 548 | let Some(installation) = row.connection_id.as_deref().and_then(|id| id.parse::<u64>().ok()) else { |
| 549 | return Ok(Err("This link has no GitHub installation.".to_owned())); |
| 550 | }; |
| 551 | Ok(self.github.installation_token(installation).await?.map(|token| (None, token))) |
| 552 | } |
| 553 | RemoteProvider::G1t | RemoteProvider::Git => { |
| 554 | let (Some(sealer), Some(sealed)) = (&self.sealer, row.credential.as_deref()) else { |
| 555 | return Ok(Err("This link has no token. Add one in its settings.".to_owned())); |
| 556 | }; |
| 557 | match sealer.open(sealed, &format!("rmt:{}", row.id)) { |
| 558 | Some(token) => Ok(Ok((row.username.clone(), token))), |
| 559 | None => Ok(Err("This link's token could not be read. Add it again.".to_owned())), |
| 560 | } |
| 561 | } |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | /// Which of `branches` the remote protects. Hosts that cannot say |
| 566 | /// protect none: a refused push still goes as a pull request. |
| 567 | async fn protected(&self, row: &RemoteRow, token: &str, branches: &[String]) -> Result<BTreeSet<String>> { |
| 568 | let mut protected = BTreeSet::new(); |
| 569 | if row.provider() != RemoteProvider::Github { |
| 570 | return Ok(protected); |
| 571 | } |
| 572 | for git_ref in branches.iter().take(MAX_PROTECTION_CHECKS) { |
| 573 | let Some(branch) = branch_of(git_ref) else { continue }; |
| 574 | let Ok(answer) = self |
| 575 | .github |
| 576 | .api(Method::Get, &format!("/repos/{}/branches/{}", row.full_name(), branch), token, None) |
| 577 | .await |
| 578 | else { |
| 579 | continue; |
| 580 | }; |
| 581 | if answer.ok() && answer.json()["protected"].as_bool() == Some(true) { |
| 582 | protected.insert(git_ref.clone()); |
| 583 | } |
| 584 | } |
| 585 | Ok(protected) |
| 586 | } |
| 587 | |
| 588 | /// Opens a pull request on the remote from a hand-back branch. Returns |
| 589 | /// a line for people: where it is, or how to open it by hand. |
| 590 | async fn open_pull_request(&self, row: &RemoteRow, token: &str, git_ref: &str, repo: &str) -> Result<String> { |
| 591 | let branch = branch_of(git_ref).unwrap_or(git_ref); |
| 592 | let head = format!("g1t/handback/{branch}"); |
| 593 | let by_hand = format!("{branch}: g1t's commits are on {head} on {}. Open a pull request from it into {branch}.", row.name); |
| 594 | if row.provider() != RemoteProvider::Github { |
| 595 | return Ok(by_hand); |
| 596 | } |
| 597 | let body = serde_json::json!({ |
| 598 | "title": format!("Changes made on g1t while {} was unreachable", row.name), |
| 599 | "head": head, |
| 600 | "base": branch, |
| 601 | "body": format!( |
| 602 | "g1t led [{repo}](https://g1t.sh/{repo}) while this repository could not be reached. These are the commits made there on `{branch}`.\n\nThe branch is protected here, so they come as a pull request rather than a push." |
| 603 | ), |
| 604 | "maintainer_can_modify": true, |
| 605 | }); |
| 606 | let Ok(answer) = self.github.api(Method::Post, &format!("/repos/{}/pulls", row.full_name()), token, Some(body)).await |
| 607 | else { |
| 608 | return Ok(by_hand); |
| 609 | }; |
| 610 | let said = answer.json(); |
| 611 | Ok(if answer.ok() { |
| 612 | format!("{branch}: opened {}", said["html_url"].as_str().unwrap_or("a pull request")) |
| 613 | } else if answer.status == 422 && said.to_string().contains("already exists") { |
| 614 | format!("{branch}: the pull request from {head} was already open and now has the new commits.") |
| 615 | } else { |
| 616 | format!("{by_hand} ({})", answer.problem("GitHub")) |
| 617 | }) |
| 618 | } |
| 619 | |
| 620 | async fn repos_refs(&self, row: &RemoteRow, username: Option<String>, token: String) -> Result<Outcome<MirrorRefs>> { |
| 621 | g1t_kit::call( |
| 622 | &self.repos, |
| 623 | "mirror_refs", |
| 624 | &MirrorRefsArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username }, |
| 625 | ) |
| 626 | .await |
| 627 | } |
| 628 | |
| 629 | /// g1t's own refs, asking nothing of the remote: a takeover starts while |
| 630 | /// it is away, when not even a token can be had from it. |
| 631 | async fn our_refs(&self, row: &RemoteRow) -> Result<Outcome<MirrorRefs>> { |
| 632 | g1t_kit::call( |
| 633 | &self.repos, |
| 634 | "mirror_refs", |
| 635 | &MirrorRefsArgs { repo_id: row.repo_id.clone(), url: String::new(), token: String::new(), username: None }, |
| 636 | ) |
| 637 | .await |
| 638 | } |
| 639 | |
| 640 | async fn apply(&self, row: &RemoteRow, username: Option<String>, token: String, moves: Vec<RefMove>) -> Result<Outcome<MirrorApplied>> { |
| 641 | g1t_kit::call( |
| 642 | &self.repos, |
| 643 | "mirror_apply", |
| 644 | &MirrorApplyArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username, moves }, |
| 645 | ) |
| 646 | .await |
| 647 | } |
| 648 | |
| 649 | /// Copies refs in the link's direction: a mirror catches up, a follower |
| 650 | /// is pushed to. `Err` is a reason, already recorded on the link. |
| 651 | async fn sync(&self, row: &RemoteRow) -> Result<std::result::Result<Mirrored, String>> { |
| 652 | let direction = if row.leads() { MirrorDirection::Pull } else { MirrorDirection::Push }; |
| 653 | let (username, token) = match self.credential(row).await? { |
| 654 | Ok(credential) => credential, |
| 655 | Err(reason) => { |
| 656 | self.note(row, Some(&reason)).await?; |
| 657 | return Ok(Err(reason)); |
| 658 | } |
| 659 | }; |
| 660 | let done: Outcome<Mirrored> = g1t_kit::call( |
| 661 | &self.repos, |
| 662 | "mirror", |
| 663 | &MirrorArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username, direction }, |
| 664 | ) |
| 665 | .await?; |
| 666 | match done { |
| 667 | Outcome::Ok(mirrored) => { |
| 668 | self.note(row, None).await?; |
| 669 | self.host_checked(&row.host(), true, None).await?; |
| 670 | Ok(Ok(mirrored)) |
| 671 | } |
| 672 | Outcome::Fail(failure) => { |
| 673 | let unreachable = failure.code == FailureCode::Unavailable; |
| 674 | if unreachable { |
| 675 | self.host_checked(&row.host(), false, Some(&failure.message)).await?; |
| 676 | } else { |
| 677 | self.note(row, Some(&failure.message)).await?; |
| 678 | } |
| 679 | if !row.leads() && !unreachable { |
| 680 | self.set_state(row, RemoteState::Stuck, None).await?; |
| 681 | } |
| 682 | Ok(Err(failure.message)) |
| 683 | } |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | async fn note(&self, row: &RemoteRow, problem: Option<&str>) -> Result<()> { |
| 688 | let statement = match problem { |
| 689 | Some(problem) => self |
| 690 | .db |
| 691 | .prepare("UPDATE remotes SET last_error = ?2 WHERE id = ?1") |
| 692 | .bind(&[row.id.as_str().into(), problem.into()])?, |
| 693 | None => self |
| 694 | .db |
| 695 | .prepare("UPDATE remotes SET last_error = NULL, synced_at = ?2 WHERE id = ?1") |
| 696 | .bind(&[row.id.as_str().into(), rfc3339(now_ms()).into()])?, |
| 697 | }; |
| 698 | statement.run().await?; |
| 699 | if problem.is_none() && row.state() == RemoteState::Stuck { |
| 700 | self.set_state(row, RemoteState::Following, None).await?; |
| 701 | } |
| 702 | Ok(()) |
| 703 | } |
| 704 | |
| 705 | // --- State ------------------------------------------------------------------ |
| 706 | |
| 707 | /// Records a link's state, and tells the repos service what a mirror |
| 708 | /// is now. A link the repos service has not heard of yet is retried by |
| 709 | /// the cron (`recorded`). |
| 710 | async fn set_state(&self, row: &RemoteRow, state: RemoteState, by: Option<&str>) -> Result<RemoteRow> { |
| 711 | let now = rfc3339(now_ms()); |
| 712 | self.db |
| 713 | .prepare("UPDATE remotes SET state = ?2, state_since = ?3, state_by = ?4, recorded = 0 WHERE id = ?1") |
| 714 | .bind(&[row.id.as_str().into(), state.as_str().into(), now.as_str().into(), null_or(by)])? |
| 715 | .run() |
| 716 | .await?; |
| 717 | let row = RemoteRow { |
| 718 | state: state.as_str().to_owned(), |
| 719 | state_since: now, |
| 720 | state_by: by.map(str::to_owned), |
| 721 | ..row.clone() |
| 722 | }; |
| 723 | self.record(&row).await?; |
| 724 | Ok(row) |
| 725 | } |
| 726 | |
| 727 | /// Tells the repos service a repository's mirror, as this link has it. |
| 728 | async fn record(&self, row: &RemoteRow) -> Result<()> { |
| 729 | if !row.leads() { |
| 730 | return Ok(()); |
| 731 | } |
| 732 | let done: Outcome<Repo> = g1t_kit::call( |
| 733 | &self.repos, |
| 734 | "set_mirror", |
| 735 | &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: row.repo_mirror() }, |
| 736 | ) |
| 737 | .await?; |
| 738 | if done.into_result().is_ok() { |
| 739 | self.db.prepare("UPDATE remotes SET recorded = 1 WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 740 | } |
| 741 | Ok(()) |
| 742 | } |
| 743 | |
| 744 | async fn publish(&self, kind: &'static str, row: &RemoteRow, by: Option<&str>, title: String, detail: Option<String>, notify: Vec<String>) { |
| 745 | let Some(events) = &self.events else { return }; |
| 746 | let event = NewEvent { |
| 747 | kind, |
| 748 | source: SOURCE, |
| 749 | repo_id: Some(row.repo_id.clone()), |
| 750 | actor: None, |
| 751 | data: MirrorEvent { |
| 752 | repo_id: row.repo_id.clone(), |
| 753 | repo: row.repo.clone(), |
| 754 | remote_id: row.id.clone(), |
| 755 | remote: row.name.clone(), |
| 756 | state: (kind != "mirror.moved_in").then(|| row.state.clone()), |
| 757 | from: None, |
| 758 | by: by.map(str::to_owned), |
| 759 | title, |
| 760 | detail, |
| 761 | notify, |
| 762 | link: row.link(), |
| 763 | }, |
| 764 | }; |
| 765 | let sent: Result<Value> = g1t_kit::call(events, "publish", &Publish { events: vec![event] }).await; |
| 766 | if let Err(error) = sent { |
| 767 | worker::console_error!("mirrors: publishing {kind} for {} failed: {error}", row.repo); |
| 768 | } |
| 769 | } |
| 770 | |
| 771 | /// The workspace's owners, when the link asks for the inbox. |
| 772 | async fn told(&self, row: &RemoteRow, except: Option<&str>) -> Vec<String> { |
| 773 | if row.settings().notify != Notify::Inbox { |
| 774 | return Vec::new(); |
| 775 | } |
| 776 | let members: Result<Outcome<Vec<Member>>> = g1t_kit::call( |
| 777 | &self.identity, |
| 778 | "list_members", |
| 779 | &ListMembersArgs { slug: row.workspace.clone(), viewer: Some(User::system(&row.workspace)) }, |
| 780 | ) |
| 781 | .await; |
| 782 | match members { |
| 783 | Ok(Outcome::Ok(members)) => members |
| 784 | .into_iter() |
| 785 | .filter(|member| member.role == Role::Owner) |
| 786 | .map(|member| member.username) |
| 787 | .filter(|name| except.is_none_or(|except| !name.eq_ignore_ascii_case(except))) |
| 788 | .collect(), |
| 789 | _ => Vec::new(), |
| 790 | } |
| 791 | } |
| 792 | |
| 793 | // --- Takeover, CI failover, hand-back, moving in ------------------------------ |
| 794 | |
| 795 | pub async fn take_over(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { |
| 796 | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { |
| 797 | Ok(found) => found, |
| 798 | Err(refused) => return Ok(refused), |
| 799 | }; |
| 800 | self.start_takeover(row, By::Person(&a.actor)).await |
| 801 | } |
| 802 | |
| 803 | async fn start_takeover(&self, row: RemoteRow, by: By<'_>) -> Result<Outcome<MirrorView>> { |
| 804 | match row.state() { |
| 805 | RemoteState::Takeover => return Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)), |
| 806 | RemoteState::HandingBack => { |
| 807 | return Ok(fail(FailureCode::Conflict, "It is being handed back. Wait for that to finish, or for it to stop.")); |
| 808 | } |
| 809 | _ => {} |
| 810 | } |
| 811 | // The starting point: what g1t holds now, which is what it last |
| 812 | // copied from the remote. |
| 813 | let refs = match self.our_refs(&row).await? { |
| 814 | Outcome::Ok(refs) => refs, |
| 815 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 816 | }; |
| 817 | let mut statements = vec![self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?]; |
| 818 | for (name, hash) in &refs.ours { |
| 819 | statements.push( |
| 820 | self.db |
| 821 | .prepare("INSERT INTO remote_refs (remote_id, ref, base) VALUES (?, ?, ?)") |
| 822 | .bind(&[row.id.as_str().into(), name.as_str().into(), hash.as_str().into()])?, |
| 823 | ); |
| 824 | } |
| 825 | self.db.batch(statements).await?; |
| 826 | let name = by.name(); |
| 827 | let row = self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; |
| 828 | let notify = self.told(&row, Some(&name)).await; |
| 829 | let title = match by { |
| 830 | By::G1t => format!("g1t took over {} while {} is unreachable", row.repo, row.name), |
| 831 | By::Person(_) => format!("{name} took over {} from {}", row.repo, row.name), |
| 832 | }; |
| 833 | self.publish("mirror.state_changed", &row, Some(&name), title, None, notify).await; |
| 834 | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)) |
| 835 | } |
| 836 | |
| 837 | pub async fn ci(&self, a: MirrorCiArgs) -> Result<Outcome<MirrorView>> { |
| 838 | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { |
| 839 | Ok(found) => found, |
| 840 | Err(refused) => return Ok(refused), |
| 841 | }; |
| 842 | let next = match (row.state(), a.on) { |
| 843 | (RemoteState::Standby, true) => RemoteState::Ci, |
| 844 | (RemoteState::Ci, false) => RemoteState::Standby, |
| 845 | (RemoteState::Ci, true) | (RemoteState::Standby, false) => { |
| 846 | return Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)); |
| 847 | } |
| 848 | _ => return Ok(fail(FailureCode::Conflict, "g1t leads this repository now: its workflows already run here.")), |
| 849 | }; |
| 850 | let row = self.set_state(&row, next, Some(&a.actor.username)).await?; |
| 851 | let title = if a.on { |
| 852 | format!("{} started running {}'s workflows on g1t", a.actor.username, row.name) |
| 853 | } else { |
| 854 | format!("{} ended CI failover for {}", a.actor.username, row.repo) |
| 855 | }; |
| 856 | self.publish("mirror.state_changed", &row, Some(&a.actor.username), title, None, Vec::new()).await; |
| 857 | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)) |
| 858 | } |
| 859 | |
| 860 | /// What handing back would do now. |
| 861 | async fn plan(&self, row: &RemoteRow) -> Result<Outcome<HandbackPlan>> { |
| 862 | let (username, token) = match self.credential(row).await? { |
| 863 | Ok(credential) => credential, |
| 864 | Err(reason) => return Ok(fail(FailureCode::Conflict, reason)), |
| 865 | }; |
| 866 | let refs = match self.repos_refs(row, username, token.clone()).await? { |
| 867 | Outcome::Ok(refs) => refs, |
| 868 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 869 | }; |
| 870 | let bases = self.bases(row).await?; |
| 871 | let Some(theirs) = refs.theirs else { |
| 872 | // The remote is away: what g1t changed, as it would go. |
| 873 | let refs = plan_refs(&bases, &refs.ours, &bases_as_theirs(&bases), &BTreeSet::new()); |
| 874 | return Ok(Outcome::Ok(HandbackPlan::new(refs, false))); |
| 875 | }; |
| 876 | let candidates: Vec<String> = plan_refs(&bases, &refs.ours, &theirs, &BTreeSet::new()) |
| 877 | .into_iter() |
| 878 | .filter(|plan| plan.action == RefAction::Push && plan.ours.is_some() && plan.theirs.is_some()) |
| 879 | .map(|plan| plan.git_ref) |
| 880 | .collect(); |
| 881 | let protected = self.protected(row, &token, &candidates).await?; |
| 882 | Ok(Outcome::Ok(HandbackPlan::new(plan_refs(&bases, &refs.ours, &theirs, &protected), true))) |
| 883 | } |
| 884 | |
| 885 | async fn bases(&self, row: &RemoteRow) -> Result<BTreeMap<String, (Option<String>, Option<RefDecision>)>> { |
| 886 | #[derive(Deserialize)] |
| 887 | struct Base { |
| 888 | #[serde(rename = "ref")] |
| 889 | git_ref: String, |
| 890 | base: Option<String>, |
| 891 | decision: Option<String>, |
| 892 | } |
| 893 | Ok(self |
| 894 | .db |
| 895 | .prepare("SELECT ref, base, decision FROM remote_refs WHERE remote_id = ?") |
| 896 | .bind(&[row.id.as_str().into()])? |
| 897 | .all() |
| 898 | .await? |
| 899 | .results::<Base>()? |
| 900 | .into_iter() |
| 901 | .map(|base| { |
| 902 | let decision = base.decision.and_then(|text| serde_json::from_value(Value::String(text)).ok()); |
| 903 | (base.git_ref, (base.base, decision)) |
| 904 | }) |
| 905 | .collect()) |
| 906 | } |
| 907 | |
| 908 | pub async fn hand_back_plan(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { |
| 909 | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { |
| 910 | Ok(found) => found, |
| 911 | Err(refused) => return Ok(refused), |
| 912 | }; |
| 913 | if !matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { |
| 914 | return Ok(fail(FailureCode::Conflict, "g1t has not taken over, so there is nothing to hand back.")); |
| 915 | } |
| 916 | let plan = match self.plan(&row).await? { |
| 917 | Outcome::Ok(plan) => plan, |
| 918 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 919 | }; |
| 920 | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, Some(plan)).await?)) |
| 921 | } |
| 922 | |
| 923 | pub async fn hand_back(&self, a: MirrorHandBackArgs) -> Result<Outcome<MirrorView>> { |
| 924 | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { |
| 925 | Ok(found) => found, |
| 926 | Err(refused) => return Ok(refused), |
| 927 | }; |
| 928 | let mut statements = Vec::new(); |
| 929 | for (git_ref, decision) in &a.decisions { |
| 930 | let decision = serde_json::to_value(decision).ok().and_then(|value| value.as_str().map(str::to_owned)); |
| 931 | statements.push( |
| 932 | self.db |
| 933 | .prepare( |
| 934 | "INSERT INTO remote_refs (remote_id, ref, decision) VALUES (?1, ?2, ?3) |
| 935 | ON CONFLICT (remote_id, ref) DO UPDATE SET decision = excluded.decision", |
| 936 | ) |
| 937 | .bind(&[row.id.as_str().into(), git_ref.as_str().into(), null_or(decision.as_deref())])?, |
| 938 | ); |
| 939 | } |
| 940 | if !statements.is_empty() { |
| 941 | self.db.batch(statements).await?; |
| 942 | } |
| 943 | self.finish_takeover(row, By::Person(&a.actor)).await |
| 944 | } |
| 945 | |
| 946 | /// Hands a takeover back: freezes the repository, moves each ref as the |
| 947 | /// plan says, and returns to standing by. If anything is refused, g1t |
| 948 | /// keeps the lead and says why, so nobody is stuck. |
| 949 | async fn finish_takeover(&self, row: RemoteRow, by: By<'_>) -> Result<Outcome<MirrorView>> { |
| 950 | if !matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { |
| 951 | return Ok(fail(FailureCode::Conflict, "g1t has not taken over, so there is nothing to hand back.")); |
| 952 | } |
| 953 | let plan = match self.plan(&row).await? { |
| 954 | Outcome::Ok(plan) => plan, |
| 955 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 956 | }; |
| 957 | if !plan.reachable { |
| 958 | return Ok(fail(FailureCode::Unavailable, format!("{} is not answering yet. Hand back once it is.", row.name))); |
| 959 | } |
| 960 | if !plan.ready { |
| 961 | return Ok(Outcome::Fail(undecided(&plan))); |
| 962 | } |
| 963 | let name = by.name(); |
| 964 | // Nothing moves on g1t while it goes back. |
| 965 | let row = self.set_state(&row, RemoteState::HandingBack, Some(&name)).await?; |
| 966 | let plan = match self.plan(&row).await? { |
| 967 | Outcome::Ok(plan) if plan.ready => plan, |
| 968 | Outcome::Ok(plan) => { |
| 969 | let row = self.set_state(&row, RemoteState::Takeover, row.state_by.as_deref()).await?; |
| 970 | let _ = row; |
| 971 | return Ok(Outcome::Fail(undecided(&plan))); |
| 972 | } |
| 973 | Outcome::Fail(failure) => { |
| 974 | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; |
| 975 | return Ok(Outcome::Fail(failure)); |
| 976 | } |
| 977 | }; |
| 978 | let (username, token) = match self.credential(&row).await? { |
| 979 | Ok(credential) => credential, |
| 980 | Err(reason) => { |
| 981 | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; |
| 982 | return Ok(fail(FailureCode::Conflict, reason)); |
| 983 | } |
| 984 | }; |
| 985 | let theirs_all = match self.repos_refs(&row, username.clone(), token.clone()).await? { |
| 986 | Outcome::Ok(refs) => refs.theirs.unwrap_or_default(), |
| 987 | Outcome::Fail(_) => BTreeMap::new(), |
| 988 | }; |
| 989 | let (moves, mut pull_requests) = hand_back_moves(&plan, &theirs_all); |
| 990 | let mut problems = Vec::new(); |
| 991 | if !moves.is_empty() { |
| 992 | let applied = match self.apply(&row, username.clone(), token.clone(), moves).await? { |
| 993 | Outcome::Ok(applied) => applied, |
| 994 | Outcome::Fail(failure) => { |
| 995 | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; |
| 996 | self.note(&row, Some(&failure.message)).await?; |
| 997 | return Ok(Outcome::Fail(failure)); |
| 998 | } |
| 999 | }; |
| 1000 | // A push the remote refused (a protected branch it did not say |
| 1001 | // was protected) goes as a pull request instead. |
| 1002 | let refused: Vec<RefPlan> = applied |
| 1003 | .moved |
| 1004 | .iter() |
| 1005 | .filter(|moved| moved.direction == MirrorDirection::Push && moved.problem.is_some()) |
| 1006 | .filter_map(|moved| { |
| 1007 | plan.refs |
| 1008 | .iter() |
| 1009 | .find(|item| item.git_ref == moved.git_ref && branch_of(&item.git_ref).is_some() && item.ours.is_some()) |
| 1010 | .cloned() |
| 1011 | }) |
| 1012 | .collect(); |
| 1013 | for moved in applied.moved.iter().filter(|moved| moved.problem.is_some()) { |
| 1014 | if !refused.iter().any(|item| item.git_ref == moved.git_ref) { |
| 1015 | problems.push(format!("{}: {}", moved.git_ref, moved.problem.as_deref().unwrap_or_default())); |
| 1016 | } |
| 1017 | } |
| 1018 | if !refused.is_empty() { |
| 1019 | let retry = HandbackPlan::new( |
| 1020 | refused |
| 1021 | .into_iter() |
| 1022 | .map(|item| RefPlan { action: RefAction::PullRequest, ..item }) |
| 1023 | .collect(), |
| 1024 | true, |
| 1025 | ); |
| 1026 | let (moves, more) = hand_back_moves(&retry, &theirs_all); |
| 1027 | match self.apply(&row, username.clone(), token.clone(), moves).await? { |
| 1028 | Outcome::Ok(applied) => { |
| 1029 | for moved in applied.moved.iter().filter(|moved| moved.problem.is_some()) { |
| 1030 | problems.push(format!("{}: {}", moved.git_ref, moved.problem.as_deref().unwrap_or_default())); |
| 1031 | } |
| 1032 | pull_requests.extend(more); |
| 1033 | } |
| 1034 | Outcome::Fail(failure) => problems.push(failure.message), |
| 1035 | } |
| 1036 | } |
| 1037 | } |
| 1038 | if !problems.is_empty() { |
| 1039 | let reason = format!("Some branches did not go back: {}", problems.join("; ")); |
| 1040 | let row = self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; |
| 1041 | self.note(&row, Some(&reason)).await?; |
| 1042 | return Ok(fail(FailureCode::Conflict, format!("{reason}. g1t still leads; try again or decide differently."))); |
| 1043 | } |
| 1044 | let mut notes = Vec::new(); |
| 1045 | for git_ref in &pull_requests { |
| 1046 | notes.push(self.open_pull_request(&row, &token, git_ref, &row.repo).await?); |
| 1047 | } |
| 1048 | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1049 | let row = self.set_state(&row, RemoteState::Standby, Some(&name)).await?; |
| 1050 | // Anything else the remote has, g1t now follows. |
| 1051 | if let Err(reason) = self.sync(&row).await? { |
| 1052 | notes.push(format!("Catching up after handing back: {reason}")); |
| 1053 | } |
| 1054 | let notify = self.told(&row, Some(&name)).await; |
| 1055 | let detail = (!notes.is_empty()).then(|| notes.join("\n")); |
| 1056 | self.publish( |
| 1057 | "mirror.state_changed", |
| 1058 | &row, |
| 1059 | Some(&name), |
| 1060 | format!("{} was handed back to {}", row.repo, row.name), |
| 1061 | detail, |
| 1062 | notify, |
| 1063 | ) |
| 1064 | .await; |
| 1065 | let mut view = self.view_of(&row.repo_id, true, None).await?; |
| 1066 | view.notes = notes; |
| 1067 | Ok(Outcome::Ok(view)) |
| 1068 | } |
| 1069 | |
| 1070 | pub async fn move_in(&self, a: MirrorMoveInArgs) -> Result<Outcome<MirrorView>> { |
| 1071 | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { |
| 1072 | Ok(found) => found, |
| 1073 | Err(refused) => return Ok(refused), |
| 1074 | }; |
| 1075 | if row.state() == RemoteState::HandingBack { |
| 1076 | return Ok(fail(FailureCode::Conflict, "It is being handed back. Move it to g1t once that is done.")); |
| 1077 | } |
| 1078 | let done: Outcome<Repo> = |
| 1079 | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; |
| 1080 | if let Outcome::Fail(failure) = done { |
| 1081 | return Ok(Outcome::Fail(failure)); |
| 1082 | } |
| 1083 | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1084 | let mut notes = Vec::new(); |
| 1085 | if a.keep_remote_updated { |
| 1086 | let now = rfc3339(now_ms()); |
| 1087 | self.db |
| 1088 | .prepare( |
| 1089 | "UPDATE remotes SET role = 'follower', state = 'following', state_since = ?2, state_by = ?3, recorded = 1 |
| 1090 | WHERE id = ?1", |
| 1091 | ) |
| 1092 | .bind(&[row.id.as_str().into(), now.as_str().into(), a.actor.username.as_str().into()])? |
| 1093 | .run() |
| 1094 | .await?; |
| 1095 | if let Some(follower) = self.row(&row.id).await? |
| 1096 | && let Err(reason) = self.sync(&follower).await? |
| 1097 | { |
| 1098 | notes.push(format!("{} could not be brought up to date yet: {reason}", row.name)); |
| 1099 | } |
| 1100 | } else { |
| 1101 | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1102 | } |
| 1103 | if row.provider() == RemoteProvider::Github { |
| 1104 | let mode = if a.keep_remote_updated { "push" } else { "import" }; |
| 1105 | self.db |
| 1106 | .prepare("UPDATE github_repos SET mode = ? WHERE repo_id = ?") |
| 1107 | .bind(&[mode.into(), row.repo_id.as_str().into()])? |
| 1108 | .run() |
| 1109 | .await?; |
| 1110 | } |
| 1111 | let title = if a.keep_remote_updated { |
| 1112 | format!("{} moved {} to g1t; {} now follows it", a.actor.username, row.repo, row.name) |
| 1113 | } else { |
| 1114 | format!("{} moved {} to g1t and stopped tracking {}", a.actor.username, row.repo, row.name) |
| 1115 | }; |
| 1116 | self.publish("mirror.moved_in", &row, Some(&a.actor.username), title, None, Vec::new()).await; |
| 1117 | let mut view = self.view_of(&row.repo_id, true, None).await?; |
| 1118 | view.notes = notes; |
| 1119 | Ok(Outcome::Ok(view)) |
| 1120 | } |
| 1121 | |
| 1122 | pub async fn sync_now(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { |
| 1123 | let Some(repo) = self.repo(&a.repo_id, Some(&a.actor)).await? else { |
| 1124 | return Ok(fail(FailureCode::NotFound, "Repository not found.")); |
| 1125 | }; |
| 1126 | // Syncing brings commits in, as pushing does. |
| 1127 | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::Push) { |
| 1128 | return Ok(refused); |
| 1129 | } |
| 1130 | let rows = self.rows(&repo.id).await?; |
| 1131 | if rows.is_empty() { |
| 1132 | return Ok(fail(FailureCode::NotFound, "This repository is not linked to another host.")); |
| 1133 | } |
| 1134 | let mut problems = Vec::new(); |
| 1135 | for row in &rows { |
| 1136 | if row.leads() && !row.state().mirror().is_some_and(MirrorState::follows) { |
| 1137 | continue; |
| 1138 | } |
| 1139 | if let Err(reason) = self.sync(row).await? { |
| 1140 | problems.push(format!("{}: {reason}", row.name)); |
| 1141 | } |
| 1142 | } |
| 1143 | if !problems.is_empty() { |
| 1144 | return Ok(fail(FailureCode::Conflict, problems.join("; "))); |
| 1145 | } |
| 1146 | Ok(Outcome::Ok(self.view_of(&repo.id, true, None).await?)) |
| 1147 | } |
| 1148 | |
| 1149 | pub async fn settings(&self, a: MirrorSettingsArgs) -> Result<Outcome<Remote>> { |
| 1150 | let Some(row) = self.row(&a.remote_id).await? else { |
| 1151 | return Ok(fail(FailureCode::NotFound, "That link was not found.")); |
| 1152 | }; |
| 1153 | let Some(repo) = self.repo(&row.repo_id, Some(&a.actor)).await? else { |
| 1154 | return Ok(fail(FailureCode::NotFound, "Repository not found.")); |
| 1155 | }; |
| 1156 | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { |
| 1157 | return Ok(refused); |
| 1158 | } |
| 1159 | let (least, most) = TAKE_OVER_AFTER_MINUTES; |
| 1160 | if a.settings.take_over_after.is_some_and(|minutes| minutes < least || minutes > most) { |
| 1161 | return Ok(fail(FailureCode::Invalid, format!("Take over after between {least} minutes and {} hours.", most / 60))); |
| 1162 | } |
| 1163 | let settings = serde_json::to_string(&a.settings).unwrap_or_else(|_| "{}".to_owned()); |
| 1164 | self.db |
| 1165 | .prepare("UPDATE remotes SET settings = ?2, recorded = 0 WHERE id = ?1") |
| 1166 | .bind(&[row.id.as_str().into(), settings.as_str().into()])? |
| 1167 | .run() |
| 1168 | .await?; |
| 1169 | let row = RemoteRow { settings, ..row }; |
| 1170 | self.record(&row).await?; |
| 1171 | let hosts = self.hosts().await?; |
| 1172 | Ok(Outcome::Ok(row.contract(hosts.get(&row.host())))) |
| 1173 | } |
| 1174 | |
| 1175 | pub async fn add(&self, a: MirrorAddArgs) -> Result<Outcome<Remote>> { |
| 1176 | if a.provider == RemoteProvider::Github { |
| 1177 | return Ok(fail(FailureCode::Invalid, "Link GitHub repositories through the GitHub App, from New → Import from GitHub.")); |
| 1178 | } |
| 1179 | let Some(repo) = self.repo(&a.repo_id, Some(&a.actor)).await? else { |
| 1180 | return Ok(fail(FailureCode::NotFound, "Repository not found.")); |
| 1181 | }; |
| 1182 | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { |
| 1183 | return Ok(refused); |
| 1184 | } |
| 1185 | let Some(clone_url) = clean_clone_url(&a.url) else { |
| 1186 | return Ok(fail(FailureCode::Invalid, "Give the remote's https address, such as https://g1t.sh/acme/web.git.")); |
| 1187 | }; |
| 1188 | let Some(token) = a.token.as_deref().map(str::trim).filter(|token| !token.is_empty()) else { |
| 1189 | return Ok(fail(FailureCode::Invalid, "Give a token that can read and push to the remote.")); |
| 1190 | }; |
| 1191 | let Some(sealer) = &self.sealer else { |
| 1192 | return Ok(fail(FailureCode::Conflict, "Tokens cannot be kept on this g1t: INTEGRATIONS_KEY is not set.")); |
| 1193 | }; |
| 1194 | if a.role == RemoteRole::Leader && self.leader(&repo.id).await?.is_some() { |
| 1195 | return Ok(fail(FailureCode::Conflict, "This repository already mirrors a remote. A repository follows one leader.")); |
| 1196 | } |
| 1197 | let now = now_ms(); |
| 1198 | let id = new_id("rmt", now); |
| 1199 | let row = RemoteRow { |
| 1200 | id: id.clone(), |
| 1201 | repo_id: repo.id.clone(), |
| 1202 | workspace: repo.namespace.clone(), |
| 1203 | repo: format!("{}/{}", repo.namespace, repo.name), |
| 1204 | provider: a.provider.as_str().to_owned(), |
| 1205 | role: a.role.as_str().to_owned(), |
| 1206 | name: display_name(&clone_url), |
| 1207 | url: web_url(&clone_url), |
| 1208 | clone_url: clone_url.clone(), |
| 1209 | connection_id: None, |
| 1210 | username: a.username.map(|name| name.trim().to_owned()).filter(|name| !name.is_empty()), |
| 1211 | credential: Some(sealer.seal(token, &format!("rmt:{id}"))), |
| 1212 | state: if a.role == RemoteRole::Leader { "standby" } else { "following" }.to_owned(), |
| 1213 | state_since: rfc3339(now), |
| 1214 | state_by: Some(a.actor.username.clone()), |
| 1215 | settings: "{}".to_owned(), |
| 1216 | synced_at: None, |
| 1217 | last_error: None, |
| 1218 | created_at: rfc3339(now), |
| 1219 | }; |
| 1220 | if row.name.eq_ignore_ascii_case(&format!("g1t.sh/{}", row.repo)) { |
| 1221 | return Ok(fail(FailureCode::Invalid, "That is this repository.")); |
| 1222 | } |
| 1223 | // A leader fills an empty repository; one with history of its own |
| 1224 | // would lose it. |
| 1225 | if a.role == RemoteRole::Leader { |
| 1226 | let refs = match self.repos_refs(&row, row.username.clone(), token.to_owned()).await? { |
| 1227 | Outcome::Ok(refs) => refs, |
| 1228 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), |
| 1229 | }; |
| 1230 | if !refs.ours.is_empty() { |
| 1231 | return Ok(fail( |
| 1232 | FailureCode::Conflict, |
| 1233 | "Only an empty repository can become a mirror: it is filled from the remote. Make a new repository for it.", |
| 1234 | )); |
| 1235 | } |
| 1236 | if let Some(reason) = refs.unreachable { |
| 1237 | return Ok(fail(FailureCode::Unavailable, reason)); |
| 1238 | } |
| 1239 | } |
| 1240 | self.db |
| 1241 | .prepare( |
| 1242 | "INSERT INTO remotes |
| 1243 | (id, repo_id, workspace, repo, provider, role, name, url, clone_url, username, credential, |
| 1244 | state, state_since, state_by, settings, created_by, created_at) |
| 1245 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, '{}', ?15, ?13)", |
| 1246 | ) |
| 1247 | .bind(&[ |
| 1248 | row.id.as_str().into(), |
| 1249 | row.repo_id.as_str().into(), |
| 1250 | row.workspace.as_str().into(), |
| 1251 | row.repo.as_str().into(), |
| 1252 | row.provider.as_str().into(), |
| 1253 | row.role.as_str().into(), |
| 1254 | row.name.as_str().into(), |
| 1255 | row.url.as_str().into(), |
| 1256 | row.clone_url.as_str().into(), |
| 1257 | null_or(row.username.as_deref()), |
| 1258 | null_or(row.credential.as_deref()), |
| 1259 | row.state.as_str().into(), |
| 1260 | row.state_since.as_str().into(), |
| 1261 | a.actor.username.as_str().into(), |
| 1262 | a.actor.id.as_str().into(), |
| 1263 | ])? |
| 1264 | .run() |
| 1265 | .await?; |
| 1266 | self.record(&row).await?; |
| 1267 | let synced = self.sync(&row).await?; |
| 1268 | let row = self.row(&id).await?.unwrap_or(row); |
| 1269 | if let Err(reason) = synced { |
| 1270 | let hosts = self.hosts().await?; |
| 1271 | let mut remote = row.contract(hosts.get(&row.host())); |
| 1272 | remote.last_error = Some(reason); |
| 1273 | return Ok(Outcome::Ok(remote)); |
| 1274 | } |
| 1275 | let hosts = self.hosts().await?; |
| 1276 | Ok(Outcome::Ok(row.contract(hosts.get(&row.host())))) |
| 1277 | } |
| 1278 | |
| 1279 | pub async fn remove(&self, a: MirrorRemoveArgs) -> Result<Outcome<bool>> { |
| 1280 | let Some(row) = self.row(&a.remote_id).await? else { |
| 1281 | return Ok(Outcome::Ok(false)); |
| 1282 | }; |
| 1283 | let Some(repo) = self.repo(&row.repo_id, Some(&a.actor)).await? else { |
| 1284 | return Ok(fail(FailureCode::NotFound, "Repository not found.")); |
| 1285 | }; |
| 1286 | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { |
| 1287 | return Ok(refused); |
| 1288 | } |
| 1289 | if row.leads() && matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { |
| 1290 | return Ok(fail( |
| 1291 | FailureCode::Conflict, |
| 1292 | "g1t leads this repository for now. Hand it back, or move it to g1t, before unlinking.", |
| 1293 | )); |
| 1294 | } |
| 1295 | if row.leads() { |
| 1296 | let done: Outcome<Repo> = |
| 1297 | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; |
| 1298 | if let Outcome::Fail(failure) = done { |
| 1299 | return Ok(Outcome::Fail(failure)); |
| 1300 | } |
| 1301 | } |
| 1302 | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1303 | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1304 | if row.provider() == RemoteProvider::Github { |
| 1305 | self.db |
| 1306 | .prepare("UPDATE github_repos SET mode = 'import' WHERE repo_id = ?") |
| 1307 | .bind(&[row.repo_id.as_str().into()])? |
| 1308 | .run() |
| 1309 | .await?; |
| 1310 | } |
| 1311 | Ok(Outcome::Ok(true)) |
| 1312 | } |
| 1313 | |
| 1314 | /// A remote g1t can no longer reach for good (uninstalled, deleted). A |
| 1315 | /// mirror standing by becomes an ordinary repository with what it has, |
| 1316 | /// since it can no longer follow; a follower is let go; a takeover |
| 1317 | /// keeps going and is told why, so nothing is lost. |
| 1318 | pub async fn link_gone(&self, id: &str, why: &str) -> Result<()> { |
| 1319 | let Some(row) = self.row(id).await? else { return Ok(()) }; |
| 1320 | if row.leads() && matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { |
| 1321 | let reason = format!("{} {why}. g1t keeps leading; move it to g1t to keep it here.", row.name); |
| 1322 | return self.note(&row, Some(&reason)).await; |
| 1323 | } |
| 1324 | if row.leads() { |
| 1325 | let _: Outcome<Repo> = |
| 1326 | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; |
| 1327 | } |
| 1328 | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1329 | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1330 | let title = if row.leads() { |
| 1331 | format!("{} stopped mirroring {}: it {why}. g1t keeps what it has.", row.repo, row.name) |
| 1332 | } else { |
| 1333 | format!("{} stopped pushing to {}: it {why}.", row.repo, row.name) |
| 1334 | }; |
| 1335 | let notify = self.told(&row, None).await; |
| 1336 | self.publish("mirror.moved_in", &row, Some("g1t"), title, None, notify).await; |
| 1337 | Ok(()) |
| 1338 | } |
| 1339 | |
| 1340 | // --- What hosts and g1t say ----------------------------------------------- |
| 1341 | |
| 1342 | /// A push on GitHub, from the App's webhook. |
| 1343 | pub async fn on_github_push(&self, payload: &Value) -> Result<()> { |
| 1344 | let Some(id) = payload["repository"]["id"].as_u64() else { return Ok(()) }; |
| 1345 | let rows = self |
| 1346 | .db |
| 1347 | .prepare("SELECT * FROM remotes WHERE provider = 'github' AND external_id = ?") |
| 1348 | .bind(&[id.to_string().into()])? |
| 1349 | .all() |
| 1350 | .await? |
| 1351 | .results::<RemoteRow>()?; |
| 1352 | for row in rows { |
| 1353 | if let Err(error) = self.on_remote_push(&row, payload).await { |
| 1354 | worker::console_error!("mirrors: a push on {} was not followed: {error}", row.name); |
| 1355 | } |
| 1356 | } |
| 1357 | Ok(()) |
| 1358 | } |
| 1359 | |
| 1360 | /// A push on a remote: a mirror standing by catches up; during a |
| 1361 | /// takeover nothing moves (the hand-back will see it); a follower takes |
| 1362 | /// in fast-forwards, or overwrites, as its settings say. |
| 1363 | async fn on_remote_push(&self, row: &RemoteRow, payload: &Value) -> Result<()> { |
| 1364 | if row.leads() { |
| 1365 | if row.state().mirror().is_some_and(MirrorState::follows) |
| 1366 | && let Err(reason) = self.sync(row).await? |
| 1367 | { |
| 1368 | worker::console_log!("mirrors: {} did not catch up with {}: {reason}", row.repo, row.name); |
| 1369 | } |
| 1370 | return Ok(()); |
| 1371 | } |
| 1372 | let (Some(git_ref), Some(after)) = (payload["ref"].as_str(), payload["after"].as_str()) else { |
| 1373 | return Ok(()); |
| 1374 | }; |
| 1375 | let before = payload["before"].as_str().filter(|hash| !hash.chars().all(|c| c == '0')); |
| 1376 | let deleted = payload["deleted"].as_bool() == Some(true) || after.chars().all(|c| c == '0'); |
| 1377 | let (username, token) = match self.credential(row).await? { |
| 1378 | Ok(credential) => credential, |
| 1379 | Err(reason) => return self.note(row, Some(&reason)).await, |
| 1380 | }; |
| 1381 | let refs = match self.repos_refs(row, username.clone(), token.clone()).await? { |
| 1382 | Outcome::Ok(refs) => refs, |
| 1383 | Outcome::Fail(failure) => return self.note(row, Some(&failure.message)).await, |
| 1384 | }; |
| 1385 | let ours = refs.ours.get(git_ref).map(String::as_str); |
| 1386 | // g1t's own push coming back: nothing to do. |
| 1387 | if ours == Some(after) || (deleted && ours.is_none()) { |
| 1388 | return Ok(()); |
| 1389 | } |
| 1390 | if row.settings().remote_pushes == RemotePushes::Overwrite { |
| 1391 | let _ = self.sync(row).await?; |
| 1392 | return Ok(()); |
| 1393 | } |
| 1394 | let forced = payload["forced"].as_bool() == Some(true); |
| 1395 | if !deleted && !forced && before == ours { |
| 1396 | let moves = vec![RefMove { |
| 1397 | direction: MirrorDirection::Pull, |
| 1398 | git_ref: git_ref.to_owned(), |
| 1399 | to: None, |
| 1400 | old: ours.map(str::to_owned), |
| 1401 | new: Some(after.to_owned()), |
| 1402 | }]; |
| 1403 | if let Outcome::Ok(applied) = self.apply(row, username, token, moves).await? |
| 1404 | && applied.moved.iter().all(|moved| moved.problem.is_none()) |
| 1405 | { |
| 1406 | return self.note(row, None).await; |
| 1407 | } |
| 1408 | } |
| 1409 | let branch = branch_of(git_ref).unwrap_or(git_ref); |
| 1410 | let reason = format!( |
| 1411 | "{branch} changed on {} in a way g1t cannot follow. Sync to push g1t's {branch} over it, or bring that change into g1t first.", |
| 1412 | row.name |
| 1413 | ); |
| 1414 | self.note(row, Some(&reason)).await?; |
| 1415 | self.set_state(row, RemoteState::Stuck, None).await?; |
| 1416 | Ok(()) |
| 1417 | } |
| 1418 | |
| 1419 | /// A push on g1t: each follower is sent it. A stuck follower waits for |
| 1420 | /// someone to sync it, so a change made on it is not pushed over. |
| 1421 | pub async fn on_event(&self, event: &g1t_contracts::events::Event) -> Result<()> { |
| 1422 | if event.kind != "git.push" { |
| 1423 | return Ok(()); |
| 1424 | } |
| 1425 | let Some(repo_id) = event.repo_id.as_deref() else { return Ok(()) }; |
| 1426 | for row in self.rows(repo_id).await? { |
| 1427 | if row.leads() || row.state() != RemoteState::Following { |
| 1428 | continue; |
| 1429 | } |
| 1430 | if let Err(reason) = self.sync(&row).await? { |
| 1431 | worker::console_log!("mirrors: {} not pushed to {}: {reason}", row.repo, row.name); |
| 1432 | } |
| 1433 | } |
| 1434 | Ok(()) |
| 1435 | } |
| 1436 | |
| 1437 | /// Records one check of a host, and acts when it goes down or comes back. |
| 1438 | async fn host_checked(&self, host: &str, answered: bool, problem: Option<&str>) -> Result<()> { |
| 1439 | let now = now_ms(); |
| 1440 | let row = self |
| 1441 | .db |
| 1442 | .prepare("SELECT * FROM remote_hosts WHERE host = ?") |
| 1443 | .bind(&[host.into()])? |
| 1444 | .first::<HostRow>(None) |
| 1445 | .await? |
| 1446 | .unwrap_or_else(|| HostRow { host: host.to_owned(), ..HostRow::default() }); |
| 1447 | let (next, change) = row.health().checked(answered, now); |
| 1448 | self.db |
| 1449 | .prepare( |
| 1450 | "INSERT INTO remote_hosts (host, failures, successes, streak_ms, unreachable_since, checked_ms, last_problem) |
| 1451 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) |
| 1452 | ON CONFLICT (host) DO UPDATE SET failures = excluded.failures, successes = excluded.successes, |
| 1453 | streak_ms = excluded.streak_ms, unreachable_since = excluded.unreachable_since, |
| 1454 | checked_ms = excluded.checked_ms, last_problem = COALESCE(excluded.last_problem, remote_hosts.last_problem)", |
| 1455 | ) |
| 1456 | .bind(&[ |
| 1457 | host.into(), |
| 1458 | next.failures.into(), |
| 1459 | next.successes.into(), |
| 1460 | (next.streak_ms as f64).into(), |
| 1461 | null_or(next.unreachable_since.map(rfc3339).as_deref()), |
| 1462 | (now as f64).into(), |
| 1463 | null_or(problem), |
| 1464 | ])? |
| 1465 | .run() |
| 1466 | .await?; |
| 1467 | if change == Change::None { |
| 1468 | return Ok(()); |
| 1469 | } |
| 1470 | let rows = self |
| 1471 | .db |
| 1472 | .prepare("SELECT * FROM remotes WHERE role = 'leader'") |
| 1473 | .all() |
| 1474 | .await? |
| 1475 | .results::<RemoteRow>()?; |
| 1476 | for row in rows.into_iter().filter(|row| row.host() == host) { |
| 1477 | let (kind, title) = match change { |
| 1478 | Change::WentDown => ("mirror.unreachable", format!("{} is not answering. {} keeps its copy.", row.name, row.repo)), |
| 1479 | _ => ("mirror.reachable", format!("{} answers again", row.name)), |
| 1480 | }; |
| 1481 | let notify = self.told(&row, None).await; |
| 1482 | let detail = (change == Change::WentDown && row.state().mirror().is_some_and(MirrorState::follows)) |
| 1483 | .then(|| format!("Take over {} to keep working on g1t until it is back.", row.repo)); |
| 1484 | self.publish(kind, &row, None, title, detail, notify).await; |
| 1485 | } |
| 1486 | Ok(()) |
| 1487 | } |
| 1488 | |
| 1489 | // --- Every minute ----------------------------------------------------------- |
| 1490 | |
| 1491 | pub async fn on_minute(&self) -> Result<()> { |
| 1492 | let now = now_ms(); |
| 1493 | // Mirrors the repos service has not heard of yet. |
| 1494 | let unrecorded = self |
| 1495 | .db |
| 1496 | .prepare("SELECT * FROM remotes WHERE recorded = 0 LIMIT 50") |
| 1497 | .all() |
| 1498 | .await? |
| 1499 | .results::<RemoteRow>()?; |
| 1500 | for row in &unrecorded { |
| 1501 | if row.leads() { |
| 1502 | self.record(row).await?; |
| 1503 | } else { |
| 1504 | self.db.prepare("UPDATE remotes SET recorded = 1 WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 1505 | } |
| 1506 | } |
| 1507 | let leaders = self |
| 1508 | .db |
| 1509 | .prepare("SELECT * FROM remotes WHERE role = 'leader'") |
| 1510 | .all() |
| 1511 | .await? |
| 1512 | .results::<RemoteRow>()?; |
| 1513 | // One check per host a mirror follows. |
| 1514 | let mut by_host: BTreeMap<String, &RemoteRow> = BTreeMap::new(); |
| 1515 | for row in &leaders { |
| 1516 | by_host.entry(row.host()).or_insert(row); |
| 1517 | } |
| 1518 | for (host, row) in &by_host { |
| 1519 | let answered = self.probe(row).await?; |
| 1520 | self.host_checked(host, answered.is_ok(), answered.err().as_deref()).await?; |
| 1521 | } |
| 1522 | let hosts = self.hosts().await?; |
| 1523 | for row in &leaders { |
| 1524 | let health = hosts.get(&row.host()).map(HostRow::health).unwrap_or_default(); |
| 1525 | let settings = row.settings(); |
| 1526 | match row.state() { |
| 1527 | // Taking over on its own, only when asked to. |
| 1528 | RemoteState::Standby | RemoteState::Ci => { |
| 1529 | if let (Some(minutes), Some(since)) = (settings.take_over_after, health.unreachable_since) |
| 1530 | && now.saturating_sub(since) >= u64::from(minutes) * 60_000 |
| 1531 | { |
| 1532 | if let Outcome::Fail(failure) = self.start_takeover(row.clone(), By::G1t).await? { |
| 1533 | worker::console_log!("mirrors: {} not taken over: {}", row.repo, failure.message); |
| 1534 | } |
| 1535 | continue; |
| 1536 | } |
| 1537 | // Hosts with no webhook are polled. |
| 1538 | if row.provider() != RemoteProvider::Github && health.reachable() { |
| 1539 | let polled = self |
| 1540 | .db |
| 1541 | .prepare("UPDATE remotes SET polled_ms = ?2 WHERE id = ?1 AND polled_ms < ?3 RETURNING id") |
| 1542 | .bind(&[row.id.as_str().into(), (now as f64).into(), (now.saturating_sub(POLL_MS) as f64).into()])? |
| 1543 | .first::<Value>(None) |
| 1544 | .await?; |
| 1545 | if polled.is_some() { |
| 1546 | let _ = self.sync(row).await?; |
| 1547 | } |
| 1548 | } |
| 1549 | } |
| 1550 | // Handing back on its own once the remote is back and it is |
| 1551 | // clean, when asked to. |
| 1552 | RemoteState::Takeover if health.reachable() && settings.hand_back == HandBack::WhenClean && row.state_by.as_deref() == Some("g1t") => { |
| 1553 | if let Outcome::Ok(plan) = self.plan(row).await? |
| 1554 | && plan.clean() |
| 1555 | && let Outcome::Fail(failure) = self.finish_takeover(row.clone(), By::G1t).await? |
| 1556 | { |
| 1557 | worker::console_log!("mirrors: {} not handed back: {}", row.repo, failure.message); |
| 1558 | } |
| 1559 | } |
| 1560 | _ => {} |
| 1561 | } |
| 1562 | } |
| 1563 | Ok(()) |
| 1564 | } |
| 1565 | |
| 1566 | /// Whether a remote's host answers `info/refs`: any answer but a server |
| 1567 | /// error counts, since a refused credential is the link's problem, not |
| 1568 | /// the host's. |
| 1569 | async fn probe(&self, row: &RemoteRow) -> Result<std::result::Result<(), String>> { |
| 1570 | let (username, token) = match self.credential(row).await? { |
| 1571 | Ok(credential) => credential, |
| 1572 | // No credential: ask without one; the host still answers. |
| 1573 | Err(_) => (None, String::new()), |
| 1574 | }; |
| 1575 | let authorization = if token.is_empty() { |
| 1576 | String::new() |
| 1577 | } else { |
| 1578 | let user = username.unwrap_or_else(|| "x-access-token".to_owned()); |
| 1579 | format!("Basic {}", base64(&format!("{user}:{token}"))) |
| 1580 | }; |
| 1581 | let url = format!("{}/info/refs?service=git-upload-pack", row.clone_url); |
| 1582 | let mut headers = vec![("accept", "*/*")]; |
| 1583 | if !authorization.is_empty() { |
| 1584 | headers.push(("authorization", authorization.as_str())); |
| 1585 | } |
| 1586 | match http::send(Method::Get, &url, &headers, None).await { |
| 1587 | Ok(answer) if answer.status < 500 && answer.status != 429 && answer.status != 408 => Ok(Ok(())), |
| 1588 | Ok(answer) => Ok(Err(format!("{} answered {}.", row.host(), answer.status))), |
| 1589 | Err(_) => Ok(Err(format!("{} could not be reached.", row.host()))), |
| 1590 | } |
| 1591 | } |
| 1592 | } |
| 1593 | |
| 1594 | fn base64(text: &str) -> String { |
| 1595 | const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; |
| 1596 | let bytes = text.as_bytes(); |
| 1597 | let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); |
| 1598 | for chunk in bytes.chunks(3) { |
| 1599 | let n = (chunk[0] as u32) << 16 | (*chunk.get(1).unwrap_or(&0) as u32) << 8 | *chunk.get(2).unwrap_or(&0) as u32; |
| 1600 | for (i, shift) in [18, 12, 6, 0].into_iter().enumerate() { |
| 1601 | out.push(if i <= chunk.len() { ALPHABET[(n >> shift) as usize & 63] as char } else { '=' }); |
| 1602 | } |
| 1603 | } |
| 1604 | out |
| 1605 | } |
| 1606 | |
| 1607 | /// When the remote is away, the plan assumes it is where the takeover |
| 1608 | /// started. |
| 1609 | fn bases_as_theirs(bases: &BTreeMap<String, (Option<String>, Option<RefDecision>)>) -> BTreeMap<String, String> { |
| 1610 | bases.iter().filter_map(|(name, (base, _))| base.clone().map(|base| (name.clone(), base))).collect() |
| 1611 | } |
| 1612 | |
| 1613 | fn undecided(plan: &HandbackPlan) -> g1t_contracts::Failure { |
| 1614 | let waiting: Vec<&str> = plan |
| 1615 | .refs |
| 1616 | .iter() |
| 1617 | .filter(|item| item.action == RefAction::Diverged && item.decision.is_none()) |
| 1618 | .map(|item| branch_of(&item.git_ref).unwrap_or(&item.git_ref)) |
| 1619 | .collect(); |
| 1620 | g1t_contracts::Failure { |
| 1621 | code: FailureCode::Conflict, |
| 1622 | message: format!( |
| 1623 | "Changed on both sides: {}. Decide for each whether to keep g1t's, keep the remote's, or send g1t's as a pull request.", |
| 1624 | waiting.join(", ") |
| 1625 | ), |
| 1626 | } |
| 1627 | } |
| 1628 | |
| 1629 | /// Answers the `mirror_*` methods; `None` for any other. |
| 1630 | pub async fn route(method: &str, body: &Value, env: &Env, _ctx: &Context) -> Option<Result<Response>> { |
| 1631 | if !method.starts_with("mirror_") { |
| 1632 | return None; |
| 1633 | } |
| 1634 | Some(handle(method, body.clone(), env).await) |
| 1635 | } |
| 1636 | |
| 1637 | async fn handle(method: &str, body: Value, env: &Env) -> Result<Response> { |
| 1638 | let mirrors = Mirrors::new(env)?; |
| 1639 | match method { |
| 1640 | "mirror_view" => reply(&mirrors.view(args(body)?).await?), |
| 1641 | "mirror_briefs" => reply(&mirrors.briefs(args(body)?).await?), |
| 1642 | "mirror_take_over" => reply(&mirrors.take_over(args(body)?).await?), |
| 1643 | "mirror_ci" => reply(&mirrors.ci(args(body)?).await?), |
| 1644 | "mirror_hand_back_plan" => reply(&mirrors.hand_back_plan(args(body)?).await?), |
| 1645 | "mirror_hand_back" => reply(&mirrors.hand_back(args(body)?).await?), |
| 1646 | "mirror_move_in" => reply(&mirrors.move_in(args(body)?).await?), |
| 1647 | "mirror_sync" => reply(&mirrors.sync_now(args(body)?).await?), |
| 1648 | "mirror_settings" => reply(&mirrors.settings(args(body)?).await?), |
| 1649 | "mirror_add" => reply(&mirrors.add(args(body)?).await?), |
| 1650 | "mirror_remove" => reply(&mirrors.remove(args(body)?).await?), |
| 1651 | _ => Response::error("Unknown method", 404), |
| 1652 | } |
| 1653 | } |
| 1654 | |
| 1655 | #[cfg(test)] |
| 1656 | mod tests { |
| 1657 | use super::*; |
| 1658 | |
| 1659 | #[test] |
| 1660 | fn remotes_are_named_as_people_say_them() { |
| 1661 | assert_eq!(host_of("https://GitHub.com/acme/web.git"), "github.com"); |
| 1662 | assert_eq!(host_of("https://user@git.acme.internal/x"), "git.acme.internal"); |
| 1663 | assert_eq!(display_name("https://github.com/acme/web.git"), "github.com/acme/web"); |
| 1664 | assert_eq!(display_name("https://g1t.sh/acme/web/"), "g1t.sh/acme/web"); |
| 1665 | assert_eq!(clean_clone_url("https://g1t.sh/acme/web").as_deref(), Some("https://g1t.sh/acme/web.git")); |
| 1666 | assert_eq!(clean_clone_url("https://git.example/a.git/").as_deref(), Some("https://git.example/a.git")); |
| 1667 | assert_eq!(clean_clone_url("http://git.example/a"), None, "https only"); |
| 1668 | assert_eq!(clean_clone_url("https://me:secret@git.example/a"), None, "no credentials in the address"); |
| 1669 | assert_eq!(clean_clone_url("https://git.example"), None); |
| 1670 | assert_eq!(web_url("https://github.com/acme/web.git"), "https://github.com/acme/web"); |
| 1671 | } |
| 1672 | |
| 1673 | #[test] |
| 1674 | fn a_host_goes_down_slowly_and_comes_back_slower() { |
| 1675 | let minute = 60_000; |
| 1676 | let mut health = HostHealth::default(); |
| 1677 | let mut changes = Vec::new(); |
| 1678 | for at in [0, minute, 2 * minute] { |
| 1679 | let (next, change) = health.checked(false, at); |
| 1680 | health = next; |
| 1681 | changes.push(change); |
| 1682 | } |
| 1683 | assert_eq!(changes, [Change::None, Change::None, Change::WentDown], "three failures over two minutes"); |
| 1684 | assert!(!health.reachable()); |
| 1685 | // One blip does not bring it back, nor do three quick answers. |
| 1686 | let (next, change) = health.checked(true, 3 * minute); |
| 1687 | assert_eq!(change, Change::None); |
| 1688 | let (next, _) = next.checked(true, 3 * minute + 1); |
| 1689 | let (next, change) = next.checked(true, 3 * minute + 2); |
| 1690 | assert_eq!(change, Change::None, "three answers in a moment are not five minutes"); |
| 1691 | let (back, change) = next.checked(true, 8 * minute); |
| 1692 | assert_eq!(change, Change::CameBack); |
| 1693 | assert!(back.reachable()); |
| 1694 | // A failure in between starts the count again. |
| 1695 | let (flaky, _) = HostHealth::default().checked(false, 0); |
| 1696 | let (flaky, _) = flaky.checked(true, minute); |
| 1697 | let (flaky, _) = flaky.checked(false, 2 * minute); |
| 1698 | let (_, change) = flaky.checked(false, 3 * minute); |
| 1699 | assert_eq!(change, Change::None); |
| 1700 | } |
| 1701 | |
| 1702 | fn refs(pairs: &[(&str, &str)]) -> BTreeMap<String, String> { |
| 1703 | pairs.iter().map(|(name, hash)| (name.to_string(), hash.to_string())).collect() |
| 1704 | } |
| 1705 | |
| 1706 | #[test] |
| 1707 | fn a_hand_back_plan_reads_each_ref_against_where_the_takeover_began() { |
| 1708 | let bases: BTreeMap<_, _> = [ |
| 1709 | ("refs/heads/main", "a"), |
| 1710 | ("refs/heads/fix", "f"), |
| 1711 | ("refs/heads/docs", "d"), |
| 1712 | ("refs/heads/quiet", "q"), |
| 1713 | ] |
| 1714 | .into_iter() |
| 1715 | .map(|(name, base)| (name.to_owned(), (Some(base.to_owned()), None))) |
| 1716 | .collect(); |
| 1717 | let ours = refs(&[ |
| 1718 | ("refs/heads/main", "a2"), |
| 1719 | ("refs/heads/fix", "f2"), |
| 1720 | ("refs/heads/docs", "d2"), |
| 1721 | ("refs/heads/quiet", "q"), |
| 1722 | ("refs/heads/agent", "n"), |
| 1723 | ]); |
| 1724 | let theirs = refs(&[ |
| 1725 | ("refs/heads/main", "a"), |
| 1726 | ("refs/heads/fix", "f"), |
| 1727 | ("refs/heads/docs", "d3"), |
| 1728 | ("refs/heads/quiet", "q"), |
| 1729 | ("refs/heads/g1t/handback/old", "x"), |
| 1730 | ]); |
| 1731 | let protected = BTreeSet::from(["refs/heads/main".to_owned()]); |
| 1732 | let plan = plan_refs(&bases, &ours, &theirs, &protected); |
| 1733 | let actions: Vec<(&str, RefAction)> = plan.iter().map(|item| (item.git_ref.as_str(), item.action)).collect(); |
| 1734 | assert_eq!(actions, [ |
| 1735 | ("refs/heads/agent", RefAction::Push), |
| 1736 | ("refs/heads/docs", RefAction::Diverged), |
| 1737 | ("refs/heads/fix", RefAction::Push), |
| 1738 | ("refs/heads/main", RefAction::PullRequest), |
| 1739 | ]); |
| 1740 | let plan = HandbackPlan::new(plan, true); |
| 1741 | assert!(!plan.ready, "docs needs a decision"); |
| 1742 | |
| 1743 | let (moves, pull_requests) = hand_back_moves(&plan, &theirs); |
| 1744 | assert_eq!(pull_requests, ["refs/heads/main"]); |
| 1745 | let summary: Vec<(MirrorDirection, &str, Option<&str>)> = |
| 1746 | moves.iter().map(|m| (m.direction, m.git_ref.as_str(), m.to.as_deref())).collect(); |
| 1747 | assert_eq!(summary, [ |
| 1748 | (MirrorDirection::Push, "refs/heads/agent", None), |
| 1749 | (MirrorDirection::Push, "refs/heads/fix", None), |
| 1750 | (MirrorDirection::Push, "refs/heads/main", Some("refs/heads/g1t/handback/main")), |
| 1751 | (MirrorDirection::Pull, "refs/heads/main", None), |
| 1752 | ], "an undecided ref does not move"); |
| 1753 | let main_back = &moves[3]; |
| 1754 | assert_eq!((main_back.old.as_deref(), main_back.new.as_deref()), (Some("a2"), Some("a")), "g1t follows the remote's main"); |
| 1755 | let push_fix = &moves[1]; |
| 1756 | assert_eq!((push_fix.old.as_deref(), push_fix.new.as_deref()), (Some("f"), Some("f2")), "only from where the remote is"); |
| 1757 | } |
| 1758 | |
| 1759 | #[test] |
| 1760 | fn decisions_settle_a_diverged_ref() { |
| 1761 | let plan = |decision| { |
| 1762 | HandbackPlan::new( |
| 1763 | vec![RefPlan { |
| 1764 | git_ref: "refs/heads/docs".into(), |
| 1765 | base: Some("d".into()), |
| 1766 | ours: Some("d2".into()), |
| 1767 | theirs: Some("d3".into()), |
| 1768 | action: RefAction::Diverged, |
| 1769 | decision: Some(decision), |
| 1770 | }], |
| 1771 | true, |
| 1772 | ) |
| 1773 | }; |
| 1774 | let theirs = refs(&[("refs/heads/g1t/handback/docs", "old")]); |
| 1775 | let (moves, _) = hand_back_moves(&plan(RefDecision::KeepOurs), &theirs); |
| 1776 | assert_eq!((moves[0].direction, moves[0].old.as_deref(), moves[0].new.as_deref()), (MirrorDirection::Push, Some("d3"), Some("d2"))); |
| 1777 | let (moves, _) = hand_back_moves(&plan(RefDecision::KeepTheirs), &theirs); |
| 1778 | assert_eq!((moves[0].direction, moves[0].old.as_deref(), moves[0].new.as_deref()), (MirrorDirection::Pull, Some("d2"), Some("d3"))); |
| 1779 | let (moves, pull_requests) = hand_back_moves(&plan(RefDecision::PullRequest), &theirs); |
| 1780 | assert_eq!(pull_requests, ["refs/heads/docs"]); |
| 1781 | assert_eq!(moves[0].old.as_deref(), Some("old"), "an old hand-back branch is moved from where it is"); |
| 1782 | assert_eq!(moves.len(), 2); |
| 1783 | } |
| 1784 | } |