Mirroring core: one leader per repository, standby mirrors, takeover and hand-back
A mirror stands by as a read-only copy of the remote it follows (repos 0017: repos.mirror), refusing pushes, merges, issues and agents until someone takes over. Integrations keeps the links (0006: remotes, remote_refs, remote_hosts), checks hosts every minute, and hands a takeover back ref by ref: pushes, pull requests for protected branches, decisions for diverged ones. A mirror can be moved to g1t for good. Catching up keeps commits a force-push dropped under refs/g1t/replaced/. GitHub links become remotes; another g1t or any git host links with a token.
31 files+3942−1720/31 viewed
| 4282 | 4282 | is_private: input["private"].as_bool() == Some(true), | |
| 4283 | 4283 | import_url: optional_text(input, "import_url"), | |
| 4284 | 4284 | import_token: None, | |
| 4285 | + | mirror: None, | |
| 4285 | 4286 | }, | |
| 4286 | 4287 | ) | |
| 4287 | 4288 | .await |
| 110 | 110 | /// [`CAUSED_BY_JOB`]). Absent otherwise. | |
| 111 | 111 | #[serde(rename = "causedByJob", skip_serializing_if = "Option::is_none")] | |
| 112 | 112 | pub caused_by_job: Option<String>, | |
| 113 | + | /// Set when the push was copied in from the remote a mirror follows, | |
| 114 | + | /// not made on g1t. Absent otherwise. | |
| 115 | + | #[serde(skip_serializing_if = "std::ops::Not::not")] | |
| 116 | + | pub mirrored: bool, | |
| 117 | + | /// The repository's mirror state when the push landed (see | |
| 118 | + | /// [`crate::mirrors`]): workflows and deployments follow it. Absent for | |
| 119 | + | /// a repository that leads. | |
| 120 | + | #[serde(skip_serializing_if = "Option::is_none")] | |
| 121 | + | pub mirror: Option<crate::mirrors::RepoMirror>, | |
| 113 | 122 | } | |
| 114 | 123 | ||
| 115 | 124 | /// The payload of `issue.opened`, `issue.updated`, `issue.assigned`, | |
| ⋯ | |||
| 1036 | 1045 | default_branch: false, | |
| 1037 | 1046 | unscanned: false, | |
| 1038 | 1047 | caused_by_job: job_run_of(&actor).map(str::to_owned), | |
| 1048 | + | mirrored: false, | |
| 1049 | + | mirror: None, | |
| 1039 | 1050 | }; | |
| 1040 | 1051 | let push = serde_json::to_value(push).unwrap(); | |
| 1041 | 1052 | assert_eq!(caused_by_job(&push), Some("run_9")); | |
| ⋯ | |||
| 1053 | 1064 | default_branch: false, | |
| 1054 | 1065 | unscanned, | |
| 1055 | 1066 | caused_by_job: None, | |
| 1067 | + | mirrored: false, | |
| 1068 | + | mirror: None, | |
| 1056 | 1069 | }; | |
| 1057 | 1070 | let quiet = serde_json::to_value(push(false)).unwrap(); | |
| 1058 | 1071 | assert!(quiet.get("unscanned").is_none()); | |
| 26 | 26 | pub mod inbox; | |
| 27 | 27 | pub mod integrations; | |
| 28 | 28 | pub mod members; | |
| 29 | + | pub mod mirrors; | |
| 29 | 30 | mod ids; | |
| 30 | 31 | mod names; | |
| 31 | 32 | mod outcome; |
| 1 | + | //! Mirroring: a repository kept in step with copies of it on other hosts. | |
| 2 | + | //! | |
| 3 | + | //! Every linked repository has exactly one leader, where work happens; the | |
| 4 | + | //! others follow it. A **mirror** follows a remote that leads (GitHub, | |
| 5 | + | //! another g1t, any git host). While it stands by it is an exact, read-only | |
| 6 | + | //! copy that runs nothing. Someone can **take over**: g1t leads for a | |
| 7 | + | //! while, then **hands back**, sending what was done to the remote. A | |
| 8 | + | //! repository g1t leads can be **mirrored to** any number of followers. | |
| 9 | + | //! | |
| 10 | + | //! The integrations service keeps the links (`remotes`) and decides; the | |
| 11 | + | //! repos service keeps each repository's [`RepoMirror`], so pushes and | |
| 12 | + | //! merges are refused or allowed without asking anyone. | |
| 13 | + | ||
| 14 | + | use std::collections::BTreeMap; | |
| 15 | + | ||
| 16 | + | use serde::{Deserialize, Serialize}; | |
| 17 | + | ||
| 18 | + | use crate::User; | |
| 19 | + | ||
| 20 | + | /// Where a mirror stands with the remote it follows. | |
| 21 | + | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] | |
| 22 | + | #[serde(rename_all = "snake_case")] | |
| 23 | + | pub enum MirrorState { | |
| 24 | + | /// A quiet copy: it follows every push and runs nothing. | |
| 25 | + | #[default] | |
| 26 | + | Standby, | |
| 27 | + | /// The remote keeps the code; g1t runs its workflows (CI failover). | |
| 28 | + | Ci, | |
| 29 | + | /// g1t leads for now: pushes, pull requests, agents and workflows work. | |
| 30 | + | Takeover, | |
| 31 | + | /// Sending what was done during a takeover back to the remote. The | |
| 32 | + | /// repository is read-only until it is done. | |
| 33 | + | HandingBack, | |
| 34 | + | } | |
| 35 | + | ||
| 36 | + | impl MirrorState { | |
| 37 | + | pub fn as_str(self) -> &'static str { | |
| 38 | + | match self { | |
| 39 | + | MirrorState::Standby => "standby", | |
| 40 | + | MirrorState::Ci => "ci", | |
| 41 | + | MirrorState::Takeover => "takeover", | |
| 42 | + | MirrorState::HandingBack => "handing_back", | |
| 43 | + | } | |
| 44 | + | } | |
| 45 | + | ||
| 46 | + | pub fn parse(text: &str) -> Option<MirrorState> { | |
| 47 | + | Some(match text { | |
| 48 | + | "standby" => MirrorState::Standby, | |
| 49 | + | "ci" => MirrorState::Ci, | |
| 50 | + | "takeover" => MirrorState::Takeover, | |
| 51 | + | "handing_back" => MirrorState::HandingBack, | |
| 52 | + | _ => return None, | |
| 53 | + | }) | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | /// Whether g1t leads, so the repository takes writes. | |
| 57 | + | pub fn leads(self) -> bool { | |
| 58 | + | self == MirrorState::Takeover | |
| 59 | + | } | |
| 60 | + | ||
| 61 | + | /// Whether g1t copies the remote's pushes in. | |
| 62 | + | pub fn follows(self) -> bool { | |
| 63 | + | matches!(self, MirrorState::Standby | MirrorState::Ci) | |
| 64 | + | } | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | /// A repository's tie to the remote it mirrors, as the repos service keeps | |
| 68 | + | /// it on [`crate::repos::Repo`]. Absent for a repository that leads. | |
| 69 | + | #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] | |
| 70 | + | #[serde(rename_all = "camelCase")] | |
| 71 | + | pub struct RepoMirror { | |
| 72 | + | pub state: MirrorState, | |
| 73 | + | /// The remote, for people: `github.com/acme/web`. | |
| 74 | + | pub remote: String, | |
| 75 | + | /// Its web address. | |
| 76 | + | pub url: String, | |
| 77 | + | /// RFC 3339: when it entered this state. | |
| 78 | + | pub since: String, | |
| 79 | + | /// Run `.g1t/workflows` on pushes copied in while it stands by. | |
| 80 | + | #[serde(default)] | |
| 81 | + | pub warm: bool, | |
| 82 | + | /// Run `.github/workflows` as well, in CI failover and during a takeover. | |
| 83 | + | #[serde(default)] | |
| 84 | + | pub github_workflows: bool, | |
| 85 | + | /// Hold jobs that name an `environment:` for approval, in CI failover | |
| 86 | + | /// and during a takeover, so nothing deploys twice. | |
| 87 | + | #[serde(default)] | |
| 88 | + | pub hold_deploys: bool, | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | impl RepoMirror { | |
| 92 | + | /// Whether the repository takes pushes, merges, issues and agents. | |
| 93 | + | pub fn writable(&self) -> bool { | |
| 94 | + | self.state.leads() | |
| 95 | + | } | |
| 96 | + | } | |
| 97 | + | ||
| 98 | + | /// Why a mirror refuses a write, for people and for `git push`. | |
| 99 | + | pub fn mirror_message(namespace: &str, name: &str, mirror: &RepoMirror) -> String { | |
| 100 | + | match mirror.state { | |
| 101 | + | MirrorState::HandingBack => format!( | |
| 102 | + | "{namespace}/{name} is handing back to {}. It takes changes again once that is done.", | |
| 103 | + | mirror.remote | |
| 104 | + | ), | |
| 105 | + | _ => format!( | |
| 106 | + | "{namespace}/{name} is a mirror of {remote}, so it is read-only here. Push to {remote}, or take over in Settings → Mirroring to work on g1t.", | |
| 107 | + | remote = mirror.remote | |
| 108 | + | ), | |
| 109 | + | } | |
| 110 | + | } | |
| 111 | + | ||
| 112 | + | /// The kinds of host a remote can be. Each is an adapter in integrations | |
| 113 | + | /// (`src/remotes.rs`); a new host is one more arm. | |
| 114 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 115 | + | #[serde(rename_all = "snake_case")] | |
| 116 | + | pub enum RemoteProvider { | |
| 117 | + | /// Through g1t's GitHub App. | |
| 118 | + | Github, | |
| 119 | + | /// Another g1t: g1t.sh, or one run by its owner. | |
| 120 | + | G1t, | |
| 121 | + | /// Any git host over HTTPS, with a username and token. | |
| 122 | + | Git, | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | impl RemoteProvider { | |
| 126 | + | pub fn as_str(self) -> &'static str { | |
| 127 | + | match self { | |
| 128 | + | RemoteProvider::Github => "github", | |
| 129 | + | RemoteProvider::G1t => "g1t", | |
| 130 | + | RemoteProvider::Git => "git", | |
| 131 | + | } | |
| 132 | + | } | |
| 133 | + | ||
| 134 | + | pub fn parse(text: &str) -> Option<RemoteProvider> { | |
| 135 | + | Some(match text { | |
| 136 | + | "github" => RemoteProvider::Github, | |
| 137 | + | "g1t" => RemoteProvider::G1t, | |
| 138 | + | "git" => RemoteProvider::Git, | |
| 139 | + | _ => return None, | |
| 140 | + | }) | |
| 141 | + | } | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | /// Which side leads. | |
| 145 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 146 | + | #[serde(rename_all = "snake_case")] | |
| 147 | + | pub enum RemoteRole { | |
| 148 | + | /// The remote leads; the repository on g1t is its mirror. | |
| 149 | + | Leader, | |
| 150 | + | /// g1t leads; the remote is kept in step with it. | |
| 151 | + | Follower, | |
| 152 | + | } | |
| 153 | + | ||
| 154 | + | impl RemoteRole { | |
| 155 | + | pub fn as_str(self) -> &'static str { | |
| 156 | + | match self { | |
| 157 | + | RemoteRole::Leader => "leader", | |
| 158 | + | RemoteRole::Follower => "follower", | |
| 159 | + | } | |
| 160 | + | } | |
| 161 | + | } | |
| 162 | + | ||
| 163 | + | /// A link's state. A leader is `standby`, `ci`, `takeover` or | |
| 164 | + | /// `handing_back` (see [`MirrorState`]); a follower is `following`, or | |
| 165 | + | /// `stuck` when the remote refused what g1t sent. | |
| 166 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 167 | + | #[serde(rename_all = "snake_case")] | |
| 168 | + | pub enum RemoteState { | |
| 169 | + | Standby, | |
| 170 | + | Ci, | |
| 171 | + | Takeover, | |
| 172 | + | HandingBack, | |
| 173 | + | Following, | |
| 174 | + | Stuck, | |
| 175 | + | } | |
| 176 | + | ||
| 177 | + | impl RemoteState { | |
| 178 | + | pub fn as_str(self) -> &'static str { | |
| 179 | + | match self { | |
| 180 | + | RemoteState::Standby => "standby", | |
| 181 | + | RemoteState::Ci => "ci", | |
| 182 | + | RemoteState::Takeover => "takeover", | |
| 183 | + | RemoteState::HandingBack => "handing_back", | |
| 184 | + | RemoteState::Following => "following", | |
| 185 | + | RemoteState::Stuck => "stuck", | |
| 186 | + | } | |
| 187 | + | } | |
| 188 | + | ||
| 189 | + | pub fn parse(text: &str) -> RemoteState { | |
| 190 | + | match text { | |
| 191 | + | "ci" => RemoteState::Ci, | |
| 192 | + | "takeover" => RemoteState::Takeover, | |
| 193 | + | "handing_back" => RemoteState::HandingBack, | |
| 194 | + | "following" => RemoteState::Following, | |
| 195 | + | "stuck" => RemoteState::Stuck, | |
| 196 | + | _ => RemoteState::Standby, | |
| 197 | + | } | |
| 198 | + | } | |
| 199 | + | ||
| 200 | + | /// The mirror state of a leader in this state. | |
| 201 | + | pub fn mirror(self) -> Option<MirrorState> { | |
| 202 | + | MirrorState::parse(self.as_str()) | |
| 203 | + | } | |
| 204 | + | } | |
| 205 | + | ||
| 206 | + | /// Who hears that a remote stopped answering. Nobody is woken up unless | |
| 207 | + | /// they asked to be. | |
| 208 | + | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] | |
| 209 | + | #[serde(rename_all = "snake_case")] | |
| 210 | + | pub enum Notify { | |
| 211 | + | /// The repository shows it, and that is all. | |
| 212 | + | #[default] | |
| 213 | + | Banner, | |
| 214 | + | /// Also an inbox item for the repository's admins. | |
| 215 | + | Inbox, | |
| 216 | + | } | |
| 217 | + | ||
| 218 | + | /// When a takeover is handed back once the remote answers again. | |
| 219 | + | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] | |
| 220 | + | #[serde(rename_all = "snake_case")] | |
| 221 | + | pub enum HandBack { | |
| 222 | + | /// Only when someone hands it back. | |
| 223 | + | Ask, | |
| 224 | + | /// On its own when every branch goes back without a decision. | |
| 225 | + | #[default] | |
| 226 | + | WhenClean, | |
| 227 | + | } | |
| 228 | + | ||
| 229 | + | /// What a follower does about pushes made on the remote itself. | |
| 230 | + | #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] | |
| 231 | + | #[serde(rename_all = "snake_case")] | |
| 232 | + | pub enum RemotePushes { | |
| 233 | + | /// Fast-forwards are taken in; anything else is shown as diverged. | |
| 234 | + | #[default] | |
| 235 | + | Adopt, | |
| 236 | + | /// g1t's branches are pushed over them (what they pointed at is kept). | |
| 237 | + | Overwrite, | |
| 238 | + | } | |
| 239 | + | ||
| 240 | + | /// The levers on a link. Every one defaults to doing nothing on its own. | |
| 241 | + | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 242 | + | #[serde(rename_all = "camelCase", default)] | |
| 243 | + | pub struct MirrorSettings { | |
| 244 | + | pub notify: Notify, | |
| 245 | + | /// Take over on its own once the remote has not answered for this many | |
| 246 | + | /// minutes. `None`: only when someone takes over. | |
| 247 | + | pub take_over_after: Option<u32>, | |
| 248 | + | pub hand_back: HandBack, | |
| 249 | + | /// Run `.g1t/workflows` on pushes copied in while standing by. | |
| 250 | + | pub keep_ci_warm: bool, | |
| 251 | + | /// Run `.github/workflows` in CI failover and during a takeover. | |
| 252 | + | pub github_workflows: bool, | |
| 253 | + | /// Hold deploy jobs for approval in CI failover and during a takeover. | |
| 254 | + | pub hold_deploys: bool, | |
| 255 | + | /// For followers. | |
| 256 | + | pub remote_pushes: RemotePushes, | |
| 257 | + | } | |
| 258 | + | ||
| 259 | + | impl Default for MirrorSettings { | |
| 260 | + | fn default() -> Self { | |
| 261 | + | MirrorSettings { | |
| 262 | + | notify: Notify::Banner, | |
| 263 | + | take_over_after: None, | |
| 264 | + | hand_back: HandBack::WhenClean, | |
| 265 | + | keep_ci_warm: false, | |
| 266 | + | github_workflows: true, | |
| 267 | + | hold_deploys: true, | |
| 268 | + | remote_pushes: RemotePushes::Adopt, | |
| 269 | + | } | |
| 270 | + | } | |
| 271 | + | } | |
| 272 | + | ||
| 273 | + | /// The shortest and longest wait a person may set before an automatic | |
| 274 | + | /// takeover. | |
| 275 | + | pub const TAKE_OVER_AFTER_MINUTES: (u32, u32) = (5, 24 * 60); | |
| 276 | + | ||
| 277 | + | /// A repository's link to a remote. | |
| 278 | + | #[derive(Clone, Debug, Serialize, Deserialize)] | |
| 279 | + | #[serde(rename_all = "camelCase")] | |
| 280 | + | pub struct Remote { | |
| 281 | + | pub id: String, | |
| 282 | + | pub repo_id: String, | |
| 283 | + | /// `workspace/name` on g1t. | |
| 284 | + | pub repo: String, | |
| 285 | + | pub provider: RemoteProvider, | |
| 286 | + | pub role: RemoteRole, | |
| 287 | + | /// For people: `github.com/acme/web`. | |
| 288 | + | pub name: String, | |
| 289 | + | /// Its web address. | |
| 290 | + | pub url: String, | |
| 291 | + | pub state: RemoteState, | |
| 292 | + | pub state_since: String, | |
| 293 | + | /// Who put it in this state: a username, or `g1t` when it was automatic. | |
| 294 | + | pub state_by: Option<String>, | |
| 295 | + | /// Whether its host answers. A remote that refuses g1t's credential | |
| 296 | + | /// still answers: that is [`Remote::last_error`]. | |
| 297 | + | pub reachable: bool, | |
| 298 | + | pub unreachable_since: Option<String>, | |
| 299 | + | pub synced_at: Option<String>, | |
| 300 | + | pub last_error: Option<String>, | |
| 301 | + | pub settings: MirrorSettings, | |
| 302 | + | pub created_at: String, | |
| 303 | + | } | |
| 304 | + | ||
| 305 | + | /// What happens to one ref when a takeover is handed back. | |
| 306 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 307 | + | #[serde(rename_all = "snake_case")] | |
| 308 | + | pub enum RefAction { | |
| 309 | + | /// Nothing moved on either side, or both moved to the same commit. | |
| 310 | + | Same, | |
| 311 | + | /// Only g1t moved: pushed to the remote. | |
| 312 | + | Push, | |
| 313 | + | /// Only the remote moved: copied in. | |
| 314 | + | Fetch, | |
| 315 | + | /// g1t moved, but the remote protects the branch: sent as a pull | |
| 316 | + | /// request from `g1t/handback/<branch>`, and g1t follows the remote. | |
| 317 | + | PullRequest, | |
| 318 | + | /// Both moved, apart. Waits for a [`RefDecision`]. | |
| 319 | + | Diverged, | |
| 320 | + | } | |
| 321 | + | ||
| 322 | + | /// A person's decision for a diverged ref. | |
| 323 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 324 | + | #[serde(rename_all = "snake_case")] | |
| 325 | + | pub enum RefDecision { | |
| 326 | + | /// g1t's commit is pushed over the remote's. | |
| 327 | + | KeepOurs, | |
| 328 | + | /// The remote's commit is taken; g1t's is kept under `refs/g1t/replaced/`. | |
| 329 | + | KeepTheirs, | |
| 330 | + | /// g1t's commits go to the remote as a pull request. | |
| 331 | + | PullRequest, | |
| 332 | + | } | |
| 333 | + | ||
| 334 | + | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 335 | + | #[serde(rename_all = "camelCase")] | |
| 336 | + | pub struct RefPlan { | |
| 337 | + | #[serde(rename = "ref")] | |
| 338 | + | pub git_ref: String, | |
| 339 | + | /// The commit both sides agreed on when the takeover began. | |
| 340 | + | pub base: Option<String>, | |
| 341 | + | pub ours: Option<String>, | |
| 342 | + | pub theirs: Option<String>, | |
| 343 | + | pub action: RefAction, | |
| 344 | + | /// For a diverged ref, what someone decided. | |
| 345 | + | pub decision: Option<RefDecision>, | |
| 346 | + | } | |
| 347 | + | ||
| 348 | + | /// What handing a takeover back would do, ref by ref. | |
| 349 | + | #[derive(Clone, Debug, Default, Serialize, Deserialize)] | |
| 350 | + | #[serde(rename_all = "camelCase")] | |
| 351 | + | pub struct HandbackPlan { | |
| 352 | + | pub refs: Vec<RefPlan>, | |
| 353 | + | /// Whether the remote answered, so the plan reflects it. | |
| 354 | + | pub reachable: bool, | |
| 355 | + | /// Whether it can go back now: the remote answers and every diverged | |
| 356 | + | /// ref has a decision. | |
| 357 | + | pub ready: bool, | |
| 358 | + | } | |
| 359 | + | ||
| 360 | + | /// Decides one ref of a hand-back. `base` is what both sides agreed on when | |
| 361 | + | /// the takeover began. A ref only g1t moved goes back by push, or by pull | |
| 362 | + | /// request when the remote protects it. | |
| 363 | + | pub fn ref_action(base: Option<&str>, ours: Option<&str>, theirs: Option<&str>, protected: bool) -> RefAction { | |
| 364 | + | if ours == theirs { | |
| 365 | + | return RefAction::Same; | |
| 366 | + | } | |
| 367 | + | let ours_moved = ours != base; | |
| 368 | + | let theirs_moved = theirs != base; | |
| 369 | + | match (ours_moved, theirs_moved) { | |
| 370 | + | (false, _) => RefAction::Fetch, | |
| 371 | + | (true, false) if protected && ours.is_some() && theirs.is_some() => RefAction::PullRequest, | |
| 372 | + | (true, false) => RefAction::Push, | |
| 373 | + | (true, true) => RefAction::Diverged, | |
| 374 | + | } | |
| 375 | + | } | |
| 376 | + | ||
| 377 | + | impl HandbackPlan { | |
| 378 | + | pub fn new(refs: Vec<RefPlan>, reachable: bool) -> Self { | |
| 379 | + | let ready = reachable | |
| 380 | + | && refs | |
| 381 | + | .iter() | |
| 382 | + | .all(|plan| plan.action != RefAction::Diverged || plan.decision.is_some()); | |
| 383 | + | HandbackPlan { refs, reachable, ready } | |
| 384 | + | } | |
| 385 | + | ||
| 386 | + | /// Whether it needs nobody: no ref is diverged. | |
| 387 | + | pub fn clean(&self) -> bool { | |
| 388 | + | self.reachable && !self.refs.iter().any(|plan| plan.action == RefAction::Diverged) | |
| 389 | + | } | |
| 390 | + | } | |
| 391 | + | ||
| 392 | + | /// Everything about a repository's links, for its pages and API. | |
| 393 | + | #[derive(Clone, Debug, Default, Serialize, Deserialize)] | |
| 394 | + | #[serde(rename_all = "camelCase")] | |
| 395 | + | pub struct MirrorView { | |
| 396 | + | pub remotes: Vec<Remote>, | |
| 397 | + | /// While a takeover is on or being handed back. | |
| 398 | + | pub plan: Option<HandbackPlan>, | |
| 399 | + | /// Whether the viewer may change the links and take over. | |
| 400 | + | pub can_manage: bool, | |
| 401 | + | /// What the last action did that people should know: pull requests it | |
| 402 | + | /// opened, branches it could not bring up to date. | |
| 403 | + | #[serde(default, skip_serializing_if = "Vec::is_empty")] | |
| 404 | + | pub notes: Vec<String>, | |
| 405 | + | } | |
| 406 | + | ||
| 407 | + | /// A repository's links in brief, for lists. | |
| 408 | + | #[derive(Clone, Debug, Serialize, Deserialize)] | |
| 409 | + | #[serde(rename_all = "camelCase")] | |
| 410 | + | pub struct RemoteBrief { | |
| 411 | + | pub repo_id: String, | |
| 412 | + | pub role: RemoteRole, | |
| 413 | + | pub name: String, | |
| 414 | + | pub state: RemoteState, | |
| 415 | + | pub reachable: bool, | |
| 416 | + | } | |
| 417 | + | ||
| 418 | + | /// The payload of the `mirror.*` events: `mirror.unreachable` (a mirror's | |
| 419 | + | /// remote stopped answering), `mirror.reachable` (it answers again), | |
| 420 | + | /// `mirror.state_changed` (stood by, CI failover, taken over, handing back, | |
| 421 | + | /// handed back) and `mirror.moved_in` (moved to g1t for good). | |
| 422 | + | #[derive(Clone, Debug, Default, Serialize, Deserialize)] | |
| 423 | + | #[serde(rename_all = "camelCase")] | |
| 424 | + | pub struct MirrorEvent { | |
| 425 | + | pub repo_id: String, | |
| 426 | + | /// `workspace/name`. | |
| 427 | + | pub repo: String, | |
| 428 | + | pub remote_id: String, | |
| 429 | + | /// `github.com/acme/web`. | |
| 430 | + | pub remote: String, | |
| 431 | + | /// A `RemoteState` as text; absent once moved in. | |
| 432 | + | #[serde(skip_serializing_if = "Option::is_none")] | |
| 433 | + | pub state: Option<String>, | |
| 434 | + | #[serde(skip_serializing_if = "Option::is_none")] | |
| 435 | + | pub from: Option<String>, | |
| 436 | + | /// Who did it: a username, or `g1t` when it happened on its own. | |
| 437 | + | #[serde(skip_serializing_if = "Option::is_none")] | |
| 438 | + | pub by: Option<String>, | |
| 439 | + | /// What happened, for people. | |
| 440 | + | pub title: String, | |
| 441 | + | #[serde(skip_serializing_if = "Option::is_none")] | |
| 442 | + | pub detail: Option<String>, | |
| 443 | + | /// Usernames to tell in their inbox (the workspace's owners, when the | |
| 444 | + | /// link's settings ask for it). | |
| 445 | + | #[serde(default, skip_serializing_if = "Vec::is_empty")] | |
| 446 | + | pub notify: Vec<String>, | |
| 447 | + | /// The repository's mirroring settings page. | |
| 448 | + | pub link: String, | |
| 449 | + | } | |
| 450 | + | ||
| 451 | + | // --- Methods of the integrations service ---------------------------------- | |
| 452 | + | ||
| 453 | + | /// `mirror_view`: a repository's links, as the viewer may see them. | |
| 454 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 455 | + | #[serde(rename_all = "camelCase")] | |
| 456 | + | pub struct MirrorViewArgs { | |
| 457 | + | pub viewer: Option<User>, | |
| 458 | + | pub repo_id: String, | |
| 459 | + | } | |
| 460 | + | ||
| 461 | + | /// `mirror_briefs`: the links of these repositories. Returns `[RemoteBrief]`. | |
| 462 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 463 | + | #[serde(rename_all = "camelCase")] | |
| 464 | + | pub struct MirrorBriefsArgs { | |
| 465 | + | pub repo_ids: Vec<String>, | |
| 466 | + | } | |
| 467 | + | ||
| 468 | + | /// `mirror_take_over`, `mirror_hand_back_plan`, `mirror_sync`: one | |
| 469 | + | /// repository's mirror. Return `Outcome<MirrorView>`. | |
| 470 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 471 | + | #[serde(rename_all = "camelCase")] | |
| 472 | + | pub struct MirrorActArgs { | |
| 473 | + | pub actor: User, | |
| 474 | + | pub repo_id: String, | |
| 475 | + | } | |
| 476 | + | ||
| 477 | + | /// `mirror_ci`: starts or ends CI failover. Returns `Outcome<MirrorView>`. | |
| 478 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 479 | + | #[serde(rename_all = "camelCase")] | |
| 480 | + | pub struct MirrorCiArgs { | |
| 481 | + | pub actor: User, | |
| 482 | + | pub repo_id: String, | |
| 483 | + | pub on: bool, | |
| 484 | + | } | |
| 485 | + | ||
| 486 | + | /// `mirror_hand_back`: sends a takeover back, with decisions for diverged | |
| 487 | + | /// refs. Refused while a diverged ref has none. Returns | |
| 488 | + | /// `Outcome<MirrorView>`. | |
| 489 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 490 | + | #[serde(rename_all = "camelCase")] | |
| 491 | + | pub struct MirrorHandBackArgs { | |
| 492 | + | pub actor: User, | |
| 493 | + | pub repo_id: String, | |
| 494 | + | #[serde(default)] | |
| 495 | + | pub decisions: BTreeMap<String, RefDecision>, | |
| 496 | + | } | |
| 497 | + | ||
| 498 | + | /// `mirror_settings`: changes a link's levers. Returns `Outcome<Remote>`. | |
| 499 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 500 | + | #[serde(rename_all = "camelCase")] | |
| 501 | + | pub struct MirrorSettingsArgs { | |
| 502 | + | pub actor: User, | |
| 503 | + | pub remote_id: String, | |
| 504 | + | pub settings: MirrorSettings, | |
| 505 | + | } | |
| 506 | + | ||
| 507 | + | /// `mirror_add`: links a repository to a remote on another g1t or any git | |
| 508 | + | /// host. GitHub links are made by `github_import`. A leader can only be | |
| 509 | + | /// added to an empty repository, which is then filled from it. Returns | |
| 510 | + | /// `Outcome<Remote>`. | |
| 511 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 512 | + | #[serde(rename_all = "camelCase")] | |
| 513 | + | pub struct MirrorAddArgs { | |
| 514 | + | pub actor: User, | |
| 515 | + | pub repo_id: String, | |
| 516 | + | pub provider: RemoteProvider, | |
| 517 | + | pub role: RemoteRole, | |
| 518 | + | /// The remote's https clone address. | |
| 519 | + | pub url: String, | |
| 520 | + | #[serde(default)] | |
| 521 | + | pub username: Option<String>, | |
| 522 | + | /// A token for it. Kept sealed; never shown again. | |
| 523 | + | #[serde(default)] | |
| 524 | + | pub token: Option<String>, | |
| 525 | + | } | |
| 526 | + | ||
| 527 | + | /// `mirror_move_in`: moves a mirror to g1t for good. The repository stops | |
| 528 | + | /// being a mirror and g1t stops tracking the remote: pushes made there no | |
| 529 | + | /// longer come here. Allowed while it stands by, in CI failover, or during | |
| 530 | + | /// a takeover (what g1t holds is kept as it is, nothing is handed back). | |
| 531 | + | /// With `keep_remote_updated`, the remote becomes a follower instead of | |
| 532 | + | /// being unlinked: g1t pushes to it from then on. Returns | |
| 533 | + | /// `Outcome<MirrorView>`. | |
| 534 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 535 | + | #[serde(rename_all = "camelCase")] | |
| 536 | + | pub struct MirrorMoveInArgs { | |
| 537 | + | pub actor: User, | |
| 538 | + | pub repo_id: String, | |
| 539 | + | #[serde(default)] | |
| 540 | + | pub keep_remote_updated: bool, | |
| 541 | + | } | |
| 542 | + | ||
| 543 | + | /// `mirror_remove`: unlinks a remote. A mirror becomes an ordinary | |
| 544 | + | /// repository with what it has. Refused during a takeover. Returns | |
| 545 | + | /// `Outcome<bool>`. | |
| 546 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 547 | + | #[serde(rename_all = "camelCase")] | |
| 548 | + | pub struct MirrorRemoveArgs { | |
| 549 | + | pub actor: User, | |
| 550 | + | pub remote_id: String, | |
| 551 | + | } | |
| 552 | + | ||
| 553 | + | #[cfg(test)] | |
| 554 | + | mod tests { | |
| 555 | + | use super::*; | |
| 556 | + | ||
| 557 | + | #[test] | |
| 558 | + | fn hand_back_decides_each_ref_from_the_base() { | |
| 559 | + | let (a, b, c) = (Some("a"), Some("b"), Some("c")); | |
| 560 | + | assert_eq!(ref_action(a, a, a, false), RefAction::Same); | |
| 561 | + | assert_eq!(ref_action(a, b, b, false), RefAction::Same, "both moved to the same commit"); | |
| 562 | + | assert_eq!(ref_action(a, b, a, false), RefAction::Push); | |
| 563 | + | assert_eq!(ref_action(a, b, a, true), RefAction::PullRequest, "a protected branch goes as a pull request"); | |
| 564 | + | assert_eq!(ref_action(a, a, b, false), RefAction::Fetch); | |
| 565 | + | assert_eq!(ref_action(a, b, c, false), RefAction::Diverged); | |
| 566 | + | assert_eq!(ref_action(None, b, None, true), RefAction::Push, "a new branch is pushed"); | |
| 567 | + | assert_eq!(ref_action(a, None, a, false), RefAction::Push, "a deleted branch is deleted there"); | |
| 568 | + | assert_eq!(ref_action(None, None, b, false), RefAction::Fetch, "a branch made there is copied in"); | |
| 569 | + | } | |
| 570 | + | ||
| 571 | + | #[test] | |
| 572 | + | fn a_plan_is_ready_once_every_diverged_ref_is_decided() { | |
| 573 | + | let diverged = |decision| RefPlan { | |
| 574 | + | git_ref: "refs/heads/docs".into(), | |
| 575 | + | base: Some("a".into()), | |
| 576 | + | ours: Some("b".into()), | |
| 577 | + | theirs: Some("c".into()), | |
| 578 | + | action: RefAction::Diverged, | |
| 579 | + | decision, | |
| 580 | + | }; | |
| 581 | + | let plan = HandbackPlan::new(vec![diverged(None)], true); | |
| 582 | + | assert!(!plan.ready && !plan.clean()); | |
| 583 | + | let plan = HandbackPlan::new(vec![diverged(Some(RefDecision::KeepTheirs))], true); | |
| 584 | + | assert!(plan.ready && !plan.clean()); | |
| 585 | + | assert!(!HandbackPlan::new(Vec::new(), false).ready, "not while the remote is away"); | |
| 586 | + | assert!(HandbackPlan::new(Vec::new(), true).clean()); | |
| 587 | + | } | |
| 588 | + | ||
| 589 | + | #[test] | |
| 590 | + | fn only_a_takeover_takes_writes() { | |
| 591 | + | let mut mirror = RepoMirror { remote: "github.com/acme/web".into(), ..RepoMirror::default() }; | |
| 592 | + | for (state, writable) in [ | |
| 593 | + | (MirrorState::Standby, false), | |
| 594 | + | (MirrorState::Ci, false), | |
| 595 | + | (MirrorState::Takeover, true), | |
| 596 | + | (MirrorState::HandingBack, false), | |
| 597 | + | ] { | |
| 598 | + | mirror.state = state; | |
| 599 | + | assert_eq!(mirror.writable(), writable, "{state:?}"); | |
| 600 | + | assert_eq!(MirrorState::parse(state.as_str()), Some(state)); | |
| 601 | + | assert_eq!(RemoteState::parse(state.as_str()).mirror(), Some(state)); | |
| 602 | + | } | |
| 603 | + | assert!(mirror_message("acme", "web", &RepoMirror { remote: "github.com/acme/web".into(), ..RepoMirror::default() }) | |
| 604 | + | .contains("is a mirror of github.com/acme/web")); | |
| 605 | + | } | |
| 606 | + | ||
| 607 | + | #[test] | |
| 608 | + | fn settings_default_to_doing_nothing_on_their_own() { | |
| 609 | + | let settings: MirrorSettings = serde_json::from_str("{}").unwrap(); | |
| 610 | + | assert_eq!(settings, MirrorSettings::default()); | |
| 611 | + | assert_eq!(settings.notify, Notify::Banner); | |
| 612 | + | assert_eq!(settings.take_over_after, None); | |
| 613 | + | assert!(!settings.keep_ci_warm); | |
| 614 | + | } | |
| 615 | + | } |
| 27 | 27 | /// A sensitive change needs the person to prove it is them again: a | |
| 28 | 28 | /// recent sign-in, or their password. See `identity::Reauth`. | |
| 29 | 29 | ReauthRequired, | |
| 30 | + | /// Another host g1t depends on for this did not answer. | |
| 31 | + | Unavailable, | |
| 30 | 32 | } | |
| 31 | 33 | ||
| 32 | 34 | impl FailureCode { | |
| ⋯ | |||
| 44 | 46 | | FailureCode::OssPoolEmpty => 402, | |
| 45 | 47 | FailureCode::Paused => 409, | |
| 46 | 48 | FailureCode::ReauthRequired => 403, | |
| 49 | + | FailureCode::Unavailable => 503, | |
| 47 | 50 | } | |
| 48 | 51 | } | |
| 49 | 52 | } | |
| 37 | 37 | /// RFC 3339: when it was archived, made read-only. Null when it is not. | |
| 38 | 38 | #[serde(default)] | |
| 39 | 39 | pub archived_at: Option<String>, | |
| 40 | + | /// Set when it mirrors a remote that leads (see [`crate::mirrors`]). | |
| 41 | + | /// Unless g1t has taken over, it is read-only. | |
| 42 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 43 | + | pub mirror: Option<crate::mirrors::RepoMirror>, | |
| 40 | 44 | } | |
| 41 | 45 | ||
| 42 | 46 | impl Repo { | |
| 43 | 47 | pub fn archived(&self) -> bool { | |
| 44 | 48 | self.archived_at.is_some() | |
| 45 | 49 | } | |
| 50 | + | ||
| 51 | + | /// Whether it is a mirror that does not take writes now. | |
| 52 | + | pub fn mirror_read_only(&self) -> bool { | |
| 53 | + | self.mirror.as_ref().is_some_and(|mirror| !mirror.writable()) | |
| 54 | + | } | |
| 55 | + | ||
| 56 | + | /// Why it takes no pushes, merges, issues or agents now: archived, or | |
| 57 | + | /// a mirror standing by. `None` when it takes them. | |
| 58 | + | pub fn read_only_reason(&self) -> Option<String> { | |
| 59 | + | if self.archived() { | |
| 60 | + | return Some(archived_message(&self.namespace, &self.name)); | |
| 61 | + | } | |
| 62 | + | self.mirror | |
| 63 | + | .as_ref() | |
| 64 | + | .filter(|mirror| !mirror.writable()) | |
| 65 | + | .map(|mirror| crate::mirrors::mirror_message(&self.namespace, &self.name, mirror)) | |
| 66 | + | } | |
| 46 | 67 | } | |
| 47 | 68 | ||
| 48 | 69 | /// `storage_options` (no arguments, `{}`): what a workspace may choose | |
| ⋯ | |||
| 328 | 349 | /// only the default branch. Set only by the integrations service. | |
| 329 | 350 | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 330 | 351 | pub import_token: Option<String>, | |
| 352 | + | /// Set by the integrations service for a mirror: it is read-only from | |
| 353 | + | /// the start. See [`crate::mirrors`]. | |
| 354 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 355 | + | pub mirror: Option<crate::mirrors::RepoMirror>, | |
| 331 | 356 | } | |
| 332 | 357 | ||
| 333 | 358 | /// `mirror`: makes a repository's branches and tags match another git | |
| ⋯ | |||
| 340 | 365 | /// The other host's https address, such as | |
| 341 | 366 | /// `https://github.com/owner/repo.git`. | |
| 342 | 367 | pub url: String, | |
| 343 | − | /// A GitHub installation access token for it. Opaque: any length. | |
| 368 | + | /// A token for it, such as a GitHub installation access token. Opaque: | |
| 369 | + | /// any length. | |
| 344 | 370 | pub token: String, | |
| 371 | + | /// The user the token is sent as, by basic authentication. | |
| 372 | + | /// `x-access-token` (GitHub's) when absent. | |
| 373 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 374 | + | pub username: Option<String>, | |
| 345 | 375 | pub direction: MirrorDirection, | |
| 346 | 376 | } | |
| 347 | 377 | ||
| ⋯ | |||
| 363 | 393 | /// Full ref names created or moved. | |
| 364 | 394 | pub updated: Vec<String>, | |
| 365 | 395 | pub deleted: Vec<String>, | |
| 396 | + | /// Refs a pull moved somewhere their old commit is not part of (a | |
| 397 | + | /// force-push on the remote), each as the `refs/g1t/replaced/...` ref | |
| 398 | + | /// that keeps the old commit. | |
| 399 | + | #[serde(default)] | |
| 400 | + | pub replaced: Vec<String>, | |
| 401 | + | } | |
| 402 | + | ||
| 403 | + | /// `mirror_refs`: a repository's branches and tags, and another host's (only | |
| 404 | + | /// the repository's when `url` is empty). | |
| 405 | + | /// Services only. Returns `Outcome<MirrorRefs>`; when the other host does | |
| 406 | + | /// not answer, `theirs` is absent and `unreachable` says why. It fails | |
| 407 | + | /// when the other host answers and refuses. | |
| 408 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 409 | + | #[serde(rename_all = "camelCase")] | |
| 410 | + | pub struct MirrorRefsArgs { | |
| 411 | + | pub repo_id: String, | |
| 412 | + | pub url: String, | |
| 413 | + | pub token: String, | |
| 414 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 415 | + | pub username: Option<String>, | |
| 416 | + | } | |
| 417 | + | ||
| 418 | + | #[derive(Clone, Debug, Default, Serialize, Deserialize)] | |
| 419 | + | #[serde(rename_all = "camelCase")] | |
| 420 | + | pub struct MirrorRefs { | |
| 421 | + | /// Full ref name to commit (or tag) id, on g1t. | |
| 422 | + | pub ours: std::collections::BTreeMap<String, String>, | |
| 423 | + | /// The same, on the other host; absent when it did not answer. | |
| 424 | + | pub theirs: Option<std::collections::BTreeMap<String, String>>, | |
| 425 | + | /// Why the other host's refs are absent. | |
| 426 | + | #[serde(default)] | |
| 427 | + | pub unreachable: Option<String>, | |
| 428 | + | } | |
| 429 | + | ||
| 430 | + | /// One ref moved by `mirror_apply`, from one side to the other. | |
| 431 | + | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 432 | + | #[serde(rename_all = "camelCase")] | |
| 433 | + | pub struct RefMove { | |
| 434 | + | pub direction: MirrorDirection, | |
| 435 | + | /// The ref on the side it comes from. | |
| 436 | + | #[serde(rename = "ref")] | |
| 437 | + | pub git_ref: String, | |
| 438 | + | /// The ref it is written to; the same name when absent. | |
| 439 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 440 | + | pub to: Option<String>, | |
| 441 | + | /// What the target must hold now for the move to happen; absent when it | |
| 442 | + | /// must not exist. | |
| 443 | + | #[serde(default)] | |
| 444 | + | pub old: Option<String>, | |
| 445 | + | /// What it is moved to; absent to delete it. | |
| 446 | + | #[serde(default)] | |
| 447 | + | pub new: Option<String>, | |
| 448 | + | } | |
| 449 | + | ||
| 450 | + | /// `mirror_apply`: moves the given refs, each only if its target still | |
| 451 | + | /// holds `old`. Pulled refs that lose their old commit keep it under | |
| 452 | + | /// `refs/g1t/replaced/`. Services only. Returns `Outcome<MirrorApplied>`. | |
| 453 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 454 | + | #[serde(rename_all = "camelCase")] | |
| 455 | + | pub struct MirrorApplyArgs { | |
| 456 | + | pub repo_id: String, | |
| 457 | + | pub url: String, | |
| 458 | + | pub token: String, | |
| 459 | + | #[serde(default, skip_serializing_if = "Option::is_none")] | |
| 460 | + | pub username: Option<String>, | |
| 461 | + | pub moves: Vec<RefMove>, | |
| 462 | + | } | |
| 463 | + | ||
| 464 | + | #[derive(Clone, Debug, Default, Serialize, Deserialize)] | |
| 465 | + | #[serde(rename_all = "camelCase")] | |
| 466 | + | pub struct MirrorApplied { | |
| 467 | + | pub moved: Vec<RefMoved>, | |
| 468 | + | } | |
| 469 | + | ||
| 470 | + | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] | |
| 471 | + | #[serde(rename_all = "camelCase")] | |
| 472 | + | pub struct RefMoved { | |
| 473 | + | #[serde(rename = "ref")] | |
| 474 | + | pub git_ref: String, | |
| 475 | + | pub direction: MirrorDirection, | |
| 476 | + | /// Why it did not move; absent when it did. | |
| 477 | + | #[serde(default)] | |
| 478 | + | pub problem: Option<String>, | |
| 479 | + | /// The `refs/g1t/replaced/...` ref that keeps what it pointed at. | |
| 480 | + | #[serde(default)] | |
| 481 | + | pub replaced: Option<String>, | |
| 482 | + | } | |
| 483 | + | ||
| 484 | + | /// `set_mirror`: records a repository's [`crate::mirrors::RepoMirror`], or | |
| 485 | + | /// clears it. Services only (integrations). Returns `Outcome<Repo>`. | |
| 486 | + | #[derive(Debug, Serialize, Deserialize)] | |
| 487 | + | #[serde(rename_all = "camelCase")] | |
| 488 | + | pub struct SetMirrorArgs { | |
| 489 | + | pub repo_id: String, | |
| 490 | + | pub mirror: Option<crate::mirrors::RepoMirror>, | |
| 366 | 491 | } | |
| 367 | 492 | ||
| 368 | 493 | /// `update`: changes whichever of a repository's details are given. | |
| 1 | + | # Mirroring: keeping remotes in sync | |
| 2 | + | ||
| 3 | + | > **2026-10-08:** "bidirectional synchronization from GitHub or whatever | |
| 4 | + | > providers … as well as setting up a different remote (think self-hosted g1t | |
| 5 | + | > and mirroring to the hosted g1t.sh platform) so that it will keep remotes in | |
| 6 | + | > sync with each other." And: "GitHub is down, I want to run GitHub's | |
| 7 | + | > workflows on g1t, but only until GitHub is back up. But what if I push on | |
| 8 | + | > g1t?" And: "It needs to be highly clear when a repository is mirrored, and | |
| 9 | + | > things are disabled … It should still work as a repository when it's in | |
| 10 | + | > mirror mode." | |
| 11 | + | ||
| 12 | + | This replaces the three loose modes shipped in Projects step 5 (import, | |
| 13 | + | mirror, push, see PLAN.md "Projects") with one model that has a clear answer | |
| 14 | + | for every push, wherever it lands. | |
| 15 | + | ||
| 16 | + | ## What exists today | |
| 17 | + | ||
| 18 | + | - GitHub only, through the `g1t-sh` App. `github_repos.mode` is one of | |
| 19 | + | `import | mirror | push` (integrations `0003_github.sql`). | |
| 20 | + | - **Mirror** pulls on GitHub's push webhook and calls `mirror::copy` with | |
| 21 | + | `Prune::Yes`, which *overwrites* g1t's refs and deletes g1t-only branches. | |
| 22 | + | Anything pushed to g1t is silently lost at the next sync. | |
| 23 | + | - **Push** ("Move to g1t") forwards each `git.push` to GitHub. A ref GitHub | |
| 24 | + | refuses ends up in `last_error`, and nothing reconciles it. | |
| 25 | + | - Mirrored and imported refs publish `git.push` with no actor. They start | |
| 26 | + | `.g1t/workflows` like any push. They are written straight to the store, so | |
| 27 | + | they skip `workflow_gate`. | |
| 28 | + | - `.github/workflows` is never read, except as a reusable `uses:` target. | |
| 29 | + | - No provider port. `Endpoint::github` and `https://github.com/{full_name}.git` | |
| 30 | + | are hard-coded. `ProjectSource { kind: "mirror", provider }` is declared but | |
| 31 | + | unused. | |
| 32 | + | ||
| 33 | + | The first three behaviours are the bugs this plan fixes. Pushes are lost | |
| 34 | + | silently, divergence goes undetected, and anyone who can push to GitHub can | |
| 35 | + | change a workflow that runs with g1t's secrets. | |
| 36 | + | ||
| 37 | + | ## The rule | |
| 38 | + | ||
| 39 | + | **Every linked repository has exactly one leader at any moment. A write | |
| 40 | + | reaches the leader first, or it doesn't land. The leader's word is final.** | |
| 41 | + | ||
| 42 | + | Everything else follows from that rule: | |
| 43 | + | ||
| 44 | + | - *Two-way sync* doesn't need its own mode. When g1t follows, a push to g1t is | |
| 45 | + | forwarded to the leader *before* g1t's ref moves (write-through). When g1t | |
| 46 | + | leads, a fast-forward push on the other side is adopted. Both sides accept | |
| 47 | + | pushes, and there is always a tie-breaker. | |
| 48 | + | - *Conflicts* can only happen when the leader was unreachable and someone | |
| 49 | + | wrote anyway (failover), or when a follower took a write it couldn't | |
| 50 | + | forward. A conflict is never resolved by overwriting silently. | |
| 51 | + | - *Echo loops* are impossible. When an update's new sha equals the tip | |
| 52 | + | already held, the update is a no-op. That covers a webhook announcing our | |
| 53 | + | own push and a chain of g1t instances alike. | |
| 54 | + | ||
| 55 | + | ## Words (UI and docs) | |
| 56 | + | ||
| 57 | + | | State | Badge on the repository | Meaning | | |
| 58 | + | | --- | --- | --- | | |
| 59 | + | | No link | (none) | An ordinary g1t repository. *Import* is a one-time copy that leaves no link. | | |
| 60 | + | | **Mirror** | `Mirror of github.com/acme/web` | The remote leads and g1t follows. Pushes to g1t are forwarded to the remote. | | |
| 61 | + | | **Mirrored** | `Mirrored to github.com/acme/web` (+ more) | g1t leads and the remotes follow. A repository can have any number of these. | | |
| 62 | + | | **Failover** | `Mirror of github.com/acme/web · Failover` (amber) | The leader is unreachable, so g1t is temporarily accepting writes. They are replayed when the leader returns. | | |
| 63 | + | | **Reconciling** | `… · 2 branches diverged` (red) | Some branches moved on both sides and need a decision. Other branches keep syncing. | | |
| 64 | + | ||
| 65 | + | A repository is a Mirror of **at most one** remote, and it can also be | |
| 66 | + | Mirrored to others. Self-hosted g1t can be a Mirror of GitHub and Mirrored | |
| 67 | + | to g1t.sh, for example. Creating a link that would make a cycle is refused. | |
| 68 | + | For g1t-to-g1t links we can ask the other side for its leader chain. | |
| 69 | + | ||
| 70 | + | The existing modes map as follows: `mirror` → Mirror, `push` → Mirrored, | |
| 71 | + | `import` → no link. | |
| 72 | + | ||
| 73 | + | ## Pushes, case by case | |
| 74 | + | ||
| 75 | + | ### When g1t is a Mirror (the remote leads) | |
| 76 | + | ||
| 77 | + | | Where the push happens | What happens | | |
| 78 | + | | --- | --- | | |
| 79 | + | | On the remote | The webhook arrives (or a poll runs when there is no webhook), g1t fetches, and the ref moves. A force-push by the leader is followed, because the leader may rewrite history. The old tip is kept at `refs/g1t/replaced/<branch>/<time>` for 30 days, and the change shows in the activity feed. Nothing is lost silently. | | |
| 80 | + | | On g1t (person, agent, merge button, web edit) | **Write-through.** g1t receives the pack, pushes it to the leader with a compare-and-swap (expected old = our tip), and moves its own ref only when the leader accepts. If the leader refuses (branch protection, non-fast-forward, a hook), the push fails with the leader's own message passed through as `remote:` lines. Branch protection on the remote is enforced for free. | | |
| 81 | + | | On g1t, branch only on g1t | There are no g1t-only branches on a Mirror. Every branch is forwarded. This removes the current `Prune::Yes` data loss: g1t never holds anything the leader doesn't. | | |
| 82 | + | | On g1t, leader unreachable | Depends on the failover setting (below). The default refuses with: `acme/web is a mirror of github.com/acme/web, which isn't answering. Pushes resume when it's back, or turn on failover in Settings → Mirroring.` | | |
| 83 | + | ||
| 84 | + | The cost is latency: a push to a Mirror takes as long as a push to the | |
| 85 | + | remote plus our own write. That trade buys g1t's commits never diverging | |
| 86 | + | while the leader is up. The 40 MB pack relay limit in `mirror.rs` applies to | |
| 87 | + | forwarded pushes as well, and the error message has to say so. | |
| 88 | + | ||
| 89 | + | ### When g1t is Mirrored (g1t leads) | |
| 90 | + | ||
| 91 | + | | Where the push happens | What happens | | |
| 92 | + | | --- | --- | | |
| 93 | + | | On g1t | It's g1t's normal push with all of g1t's rules. After it lands, a `git.push` event queues a forward to each follower (this exists today). A follower that refuses (its own protection, for example) marks that ref **stuck** on that remote and shows it on the repository. Retries back off and never force. | | |
| 94 | + | | On the remote, fast-forward | **Default: adopt.** The webhook triggers a fetch, and the update goes through g1t's ref rules as if the pusher had pushed to g1t, with the pusher mapped to a g1t account (see Actors). If g1t's rules allow it, the ref moves and the update is forwarded to the other followers. If they refuse (a protected branch, say), the ref is marked **diverged**. | | |
| 95 | + | | On the remote, not a fast-forward | Marked **diverged**. g1t never force-pushes over it without a person's or agent's decision. | | |
| 96 | + | ||
| 97 | + | A setting covers pushes made directly on the remote: | |
| 98 | + | ||
| 99 | + | - **Adopt fast-forwards** (default) | |
| 100 | + | - **Overwrite them**: g1t force-pushes its tip and keeps the replaced | |
| 101 | + | remote tip under `refs/g1t/replaced/…`. This suits teams that consider the | |
| 102 | + | remote read-only. | |
| 103 | + | - **Lock the remote**: g1t creates a ruleset on the GitHub repository that | |
| 104 | + | lets only the g1t App update refs. This needs `administration: write` on | |
| 105 | + | the App, which we request only when someone picks this option. | |
| 106 | + | ||
| 107 | + | ### Diverged branches | |
| 108 | + | ||
| 109 | + | A diverged branch stops syncing and every other branch keeps going. Both | |
| 110 | + | tips are kept: g1t's at the branch, and the other side's at | |
| 111 | + | `refs/remotes/<remote>/<branch>`, which is browsable and diffable. The | |
| 112 | + | repository shows `main diverged from github.com/acme/web: 3 commits here, 1 | |
| 113 | + | there` with three actions: | |
| 114 | + | ||
| 115 | + | 1. **Keep g1t's**, which force-pushes to the other side (the old tip is kept). | |
| 116 | + | 2. **Keep theirs**, which moves g1t's ref (the old tip is kept). | |
| 117 | + | 3. **Merge them**, which opens a pull request merging the other tip into the | |
| 118 | + | branch. One click hands it to @g1t, and it lands through the normal rules. | |
| 119 | + | ||
| 120 | + | ## Failover: GitHub is down, and I push on g1t | |
| 121 | + | ||
| 122 | + | Failover is a *state* of a Mirror, not a separate mode. The setting | |
| 123 | + | "When github.com/acme/web is unreachable" has three choices: | |
| 124 | + | ||
| 125 | + | - **Stop accepting pushes** (default for new links) | |
| 126 | + | - **Ask me**: a banner and an inbox item for repository admins offer | |
| 127 | + | **Start failover**. | |
| 128 | + | - **Fail over automatically** | |
| 129 | + | ||
| 130 | + | **Entering failover.** The leader is unreachable when forwarded writes fail | |
| 131 | + | with a timeout, a connection error or a 5xx, *and* a health probe of | |
| 132 | + | `info/refs` fails 3 times over 2 minutes. A provider status page alone isn't | |
| 133 | + | enough, because they lag and over-report. While in failover: | |
| 134 | + | ||
| 135 | + | - g1t accepts pushes, merges and agent work on every branch. Branch rules | |
| 136 | + | still apply: g1t imports a read-only copy of the remote's protection when | |
| 137 | + | the link is made, refreshes it on every sync, and enforces it during | |
| 138 | + | failover. "Main needs a reviewed pull request" still holds while GitHub is | |
| 139 | + | down. | |
| 140 | + | - Each ref records its **base**, the last sha both sides agreed on. | |
| 141 | + | - The repository wears the amber Failover badge with the duration and the | |
| 142 | + | count of commits waiting to go back. | |
| 143 | + | ||
| 144 | + | **Leaving failover.** Once the probe succeeds 3 times over 5 minutes, each | |
| 145 | + | branch changed during failover is replayed: | |
| 146 | + | ||
| 147 | + | | Remote since base | g1t since base | Result | | |
| 148 | + | | --- | --- | --- | | |
| 149 | + | | unchanged | moved | Fast-forward push to the remote. | | |
| 150 | + | | moved | unchanged | Fetch, as normal. | | |
| 151 | + | | moved | moved, one contains the other | Fast-forward whichever side is behind. | | |
| 152 | + | | moved | moved, split | **Diverged** (above). | | |
| 153 | + | | (remote refuses: protected branch) | moved | **Replayed as a pull request on GitHub** from `g1t/failover/<branch>`, titled *Changes made on g1t while GitHub was unreachable*. g1t can't push straight to a protected main, and shouldn't. | | |
| 154 | + | ||
| 155 | + | The repository goes back to a plain Mirror when nothing is diverged or | |
| 156 | + | stuck. Until then it shows Reconciling. An admin can end failover by hand at | |
| 157 | + | any time, and the replay runs the same way. | |
| 158 | + | ||
| 159 | + | ## Workflows | |
| 160 | + | ||
| 161 | + | ### Which files run on g1t | |
| 162 | + | ||
| 163 | + | | Files | Default | Setting | | |
| 164 | + | | --- | --- | --- | | |
| 165 | + | | `.g1t/workflows` | Run on every push, whatever its origin | None needed. These are g1t's files: having them means you want them run. Workflows can filter on `g1t.event.origin` (`local` / `remote`). | | |
| 166 | + | | `.github/workflows` | **Off** | **Run GitHub's workflows on g1t: Off · While GitHub Actions is down · Always.** | | |
| 167 | + | ||
| 168 | + | This deliberately narrows the rule in `crates/actions/src/workflow.rs` ("g1t | |
| 169 | + | never reads `.github`"). g1t reads `.github/workflows` only for repositories | |
| 170 | + | linked to GitHub, and only when this setting is on. When both folders define | |
| 171 | + | the same `name:`, `.g1t` wins. | |
| 172 | + | ||
| 173 | + | ### "While GitHub Actions is down" | |
| 174 | + | ||
| 175 | + | This is the scenario from the request. The window opens when either of these | |
| 176 | + | happens: | |
| 177 | + | ||
| 178 | + | 1. A pushed commit gets no check suite on GitHub within 10 minutes. This | |
| 179 | + | needs `checks: read` on the App and is the most reliable signal, because it | |
| 180 | + | means GitHub really didn't run it. | |
| 181 | + | 2. Someone with admin on the repository clicks **GitHub Actions is down: run | |
| 182 | + | here** on the repository or on a single commit. A per-commit **Run on g1t** | |
| 183 | + | button is always available, in any setting. | |
| 184 | + | ||
| 185 | + | GitHub's status page is shown next to the banner for context. It never opens | |
| 186 | + | or closes the window by itself. | |
| 187 | + | ||
| 188 | + | The window closes once GitHub starts check suites again for new commits. Runs | |
| 189 | + | already started on g1t finish. | |
| 190 | + | ||
| 191 | + | While the window is open: | |
| 192 | + | ||
| 193 | + | - Pushes and pull requests on g1t, **including failover pushes**, run the | |
| 194 | + | `.github` workflows that match. | |
| 195 | + | - Commits pushed during the window that GitHub never ran are **backfilled**: | |
| 196 | + | g1t offers to run them in one click. | |
| 197 | + | - Results go back to GitHub as check runs named `g1t / <workflow> / | |
| 198 | + | <job>` (needs `checks: write`). People on GitHub see them, and GitHub's | |
| 199 | + | required-check rules can name them if the team chooses. | |
| 200 | + | ||
| 201 | + | **Deploys and other side effects.** A workflow running in two places can | |
| 202 | + | deploy twice. In "While down" mode, jobs that declare an `environment:` or | |
| 203 | + | use a secret named `*DEPLOY*`/`*TOKEN*` that only GitHub has are **held for | |
| 204 | + | approval** by default. The setting is "Side-effect jobs in GitHub's | |
| 205 | + | workflows: hold for approval (default) · run". In "Always" mode they run. | |
| 206 | + | That mode is for teams that have moved CI to g1t and keep GitHub for code | |
| 207 | + | review. | |
| 208 | + | ||
| 209 | + | **Secrets.** GitHub won't give out secret values. When the setting is turned | |
| 210 | + | on, g1t lists every `secrets.X` the `.github` workflows reference, ticks off | |
| 211 | + | the ones already set on the project, and blocks "Always" until each is set | |
| 212 | + | or marked "not needed". A run that hits a missing secret fails with | |
| 213 | + | `NPM_TOKEN is set on GitHub but not on g1t: add it in Project → Secrets`. | |
| 214 | + | It does not fail with an empty string. | |
| 215 | + | ||
| 216 | + | **After GitHub returns.** Failover commits replayed to GitHub start GitHub's | |
| 217 | + | own workflows there. That is expected, because GitHub's checks are what | |
| 218 | + | GitHub's rules ask for. g1t's check runs for the same sha are already posted, | |
| 219 | + | so reviewers see both. Deploy jobs held on g1t can be discarded once GitHub | |
| 220 | + | has deployed. | |
| 221 | + | ||
| 222 | + | ### Workflow changes that arrive from a remote | |
| 223 | + | ||
| 224 | + | Today a commit fetched from GitHub that edits `.g1t/workflows` runs with | |
| 225 | + | g1t's secrets, and its author never needed `workflow_files: write` on g1t. | |
| 226 | + | That gets fixed in step 1, before any of the rest: | |
| 227 | + | ||
| 228 | + | - A fetched push that changes workflow files, or `.github/workflows` while | |
| 229 | + | that setting is on, runs only if the pusher maps to a g1t account with | |
| 230 | + | `workflow_files: write` on the repository. | |
| 231 | + | - Otherwise its runs wait for approval: `Workflow changed on GitHub by | |
| 232 | + | @octo, who has no write access to workflows here. Approve run?` This is | |
| 233 | + | the same mechanism as runs from forks. | |
| 234 | + | ||
| 235 | + | ## Actors | |
| 236 | + | ||
| 237 | + | GitHub's push webhook names the pusher, and `github_accounts` already links | |
| 238 | + | GitHub users to g1t users. A fetched push is attributed to the linked g1t | |
| 239 | + | account, or to `github:<login>` (shown greyed out, not a g1t user) when there | |
| 240 | + | is none. Today such pushes have no actor. Webhooks, audit and the workflow | |
| 241 | + | `actor` all get the attributed value. | |
| 242 | + | ||
| 243 | + | ## What works on a Mirror, and what's disabled | |
| 244 | + | ||
| 245 | + | This table is also the source for the UI's disabled states. Every disabled | |
| 246 | + | control uses the shadcn Tooltip with the reason and the setting that would | |
| 247 | + | enable it. | |
| 248 | + | ||
| 249 | + | | | Mirror (remote leads) | Mirror in Failover | Mirrored (g1t leads) | | |
| 250 | + | | --- | --- | --- | --- | | |
| 251 | + | | Browse, clone, fetch, blame, search | ✓ | ✓ | ✓ | | |
| 252 | + | | Push branches and tags | ✓ forwarded to the remote | ✓ held, replayed later | ✓ | | |
| 253 | + | | Branch protection | The remote's (read-only copy shown) | The remote's copy, enforced by g1t | g1t's | | |
| 254 | + | | Pull requests | **The remote's.** Listed on g1t; *New pull request* and agents open them on the remote; *Merge* merges there through the API under its rules | g1t pull requests allowed. Each is replayed as a remote pull request when the remote returns | g1t's | | |
| 255 | + | | Merge queue, required g1t checks | Disabled: "Merges happen on GitHub, which leads this repository." | ✓ on g1t's copy of the rules | ✓ | | |
| 256 | + | | Issues, agents, plans | ✓ g1t's own. Agents push branches that are forwarded, and open pull requests on the remote | ✓ | ✓ | | |
| 257 | + | | `.g1t/workflows` | ✓ | ✓ | ✓ | | |
| 258 | + | | `.github/workflows` | Per setting | Per setting | Per setting (only if a follower is GitHub) | | |
| 259 | + | | Projects, deployments, previews, secrets | ✓ (the on-ramp) | ✓ | ✓ | | |
| 260 | + | | Releases | Read from the remote (later) | Held | g1t's, copied to followers (later) | | |
| 261 | + | | Rename, transfer | g1t side only. The link stays | Same | Same | | |
| 262 | + | | Delete branch, set default branch | Forwarded / read from the remote | Held | g1t's, forwarded | | |
| 263 | + | | Archive | Stops syncing. The link is kept and paused | Not allowed | Stops forwarding | | |
| 264 | + | | Unlink | Becomes an ordinary repository (choice: keep g1t-side failover commits) | Must reconcile first | Followers stop receiving | | |
| 265 | + | ||
| 266 | + | Other places that must show the state, not only the badge: | |
| 267 | + | ||
| 268 | + | - **Clone box:** `This is a mirror of github.com/acme/web. Pushes here are | |
| 269 | + | forwarded there.` | |
| 270 | + | - **Push output:** `remote: forwarded to github.com/acme/web (412 ms)`, so | |
| 271 | + | people learn the model the first time they push. | |
| 272 | + | - **Repository header:** the sync dot (`in sync · 40s ago`, `syncing`, | |
| 273 | + | `failover · 23m`, `2 diverged`, `stuck on gitlab.com/…`). It opens | |
| 274 | + | **Settings → Mirroring**, which lists every remote, each branch's state, | |
| 275 | + | the last 50 sync events and **Sync now**. | |
| 276 | + | - **Workspace and project lists:** a small mirror glyph with the leader's | |
| 277 | + | host. | |
| 278 | + | ||
| 279 | + | ## The provider port | |
| 280 | + | ||
| 281 | + | All of this lives behind one port, so GitLab, Bitbucket, plain git and other | |
| 282 | + | g1t instances are each an adapter. The Rust trait lives in | |
| 283 | + | `services/integrations`, because the connections live there: | |
| 284 | + | ||
| 285 | + | ```rust | |
| 286 | + | trait Remote { | |
| 287 | + | fn kind(&self) -> RemoteKind; // github | g1t | gitlab | bitbucket | git | |
| 288 | + | fn capabilities(&self) -> Capabilities; // webhooks, checks, lock, pull_requests, protection | |
| 289 | + | async fn endpoint(&self) -> Result<mirror::Endpoint>;// URL + fresh credential for mirror.rs | |
| 290 | + | async fn parse_event(&self, req: &Request) -> Result<Vec<RemoteRefChange>>; // verify + normalise | |
| 291 | + | async fn probe(&self) -> Health; // info/refs reachability | |
| 292 | + | async fn protection(&self) -> Result<Vec<RuleCopy>>; // read the remote's branch rules | |
| 293 | + | async fn lock(&self, on: bool) -> Result<()>; // optional | |
| 294 | + | async fn post_check(&self, sha: &str, run: &CheckRun) -> Result<()>; // optional | |
| 295 | + | async fn pull_requests(&self) -> Result<PullRequestPort>; // optional, step 7 | |
| 296 | + | } | |
| 297 | + | ``` | |
| 298 | + | ||
| 299 | + | - Each `Capabilities` flag that's off greys out the matching UI and setting. | |
| 300 | + | The UI never offers a lever the provider can't honour. | |
| 301 | + | - Providers without webhooks (plain `git`, or a self-hosted g1t behind a | |
| 302 | + | firewall) get a **poll** every 1 to 10 minutes with backoff, using | |
| 303 | + | `ls-remote` and comparing tips. The poll uses the same code path as the | |
| 304 | + | webhook. | |
| 305 | + | - **`g1t` adapter (self-hosted ↔ g1t.sh).** It authenticates with a | |
| 306 | + | fine-grained token on the other instance (`contents: write`, | |
| 307 | + | `webhooks: write`) and registers its own webhook there through our API. | |
| 308 | + | Because we control both ends, write-through works in either direction, and | |
| 309 | + | the cycle check can ask the other side for its chain. Issues and pull | |
| 310 | + | requests between g1t instances come later, over the public API. | |
| 311 | + | ||
| 312 | + | ## Data | |
| 313 | + | ||
| 314 | + | **integrations, `remotes`** (replaces `github_repos`; existing rows are | |
| 315 | + | migrated): | |
| 316 | + | - `id`, `repo_id`, `workspace`, `kind`, `url`, `role` (`leader` | | |
| 317 | + | `follower`), and `connection_id` (installation or token row). | |
| 318 | + | - `provider_ref`, provider-specific JSON (GitHub: `installation_id`, | |
| 319 | + | `github_repo_id`). | |
| 320 | + | - `settings` JSON: on-unreachable, remote-push policy, `.github` workflows, | |
| 321 | + | side-effect jobs. | |
| 322 | + | - `state` (`ok` | `failover` | `reconciling` | `paused` | `error`), | |
| 323 | + | `state_since`, `last_error`, `synced_at`. | |
| 324 | + | - A unique index on `(repo_id) WHERE role = 'leader'` enforces at most one | |
| 325 | + | leader. | |
| 326 | + | ||
| 327 | + | **repos, `ref_sync`** (per repository, inside the repository's store so it | |
| 328 | + | moves atomically with refs): | |
| 329 | + | - `remote_id`, `ref`, `base_sha` (last agreed), `remote_sha` (last seen | |
| 330 | + | there). | |
| 331 | + | - `state` (`ok` | `ahead` | `behind` | `held` | `diverged` | `stuck`), | |
| 332 | + | `updated_at`. | |
| 333 | + | ||
| 334 | + | **repos, repository row:** `mirror_of` (remote id or null), cached from | |
| 335 | + | integrations through a `remote.updated` event. The git front door, commit | |
| 336 | + | API and merge paths can then decide forward-or-refuse without an RPC. | |
| 337 | + | `lifecycle::archived_refusal` is the template for a matching | |
| 338 | + | `mirror::route(repo, ref)` used at the same call sites. | |
| 339 | + | ||
| 340 | + | **contracts, `GitPush`** gains `origin: { kind: "local" | "remote", | |
| 341 | + | remote_id?, provider?, pusher? }`. Actions, webhooks, audit and the inbox | |
| 342 | + | read it. `ProjectSource`'s unused `mirror` variant is dropped: a Mirror is | |
| 343 | + | still a hosted repository with a full copy, so its project stays `hosted`, as | |
| 344 | + | PLAN.md already decided. | |
| 345 | + | ||
| 346 | + | **Events:** `remote.linked`, `remote.updated`, `remote.unlinked`, | |
| 347 | + | `remote.failover_started`, `remote.failover_ended`, `ref.diverged`, | |
| 348 | + | `ref.reconciled`. These are in the public webhook catalogue, so people can | |
| 349 | + | build alerts on them. | |
| 350 | + | ||
| 351 | + | ## Build order | |
| 352 | + | ||
| 353 | + | 1. **Safety and the port.** The `Remote` trait with the GitHub adapter. Move | |
| 354 | + | `github_repos` to `remotes`. Add `ref_sync` with base shas. Add `origin` | |
| 355 | + | and the attributed actor on `GitPush`. Echo suppression by sha. Keep | |
| 356 | + | replaced tips instead of overwriting silently. Approval for workflow | |
| 357 | + | changes arriving from a remote. Badge, sync dot, and **Settings → | |
| 358 | + | Mirroring** (read-only state). | |
| 359 | + | 2. **Write-through Mirror.** Forward pushes from the git front door, the | |
| 360 | + | commit API, merges and agents. Pass the leader's refusals through. Apply | |
| 361 | + | the disabled-feature table in the UI. Read-only copy of the remote's | |
| 362 | + | protection. | |
| 363 | + | 3. **Mirrored with remote pushes handled.** Adopt / overwrite / lock, stuck | |
| 364 | + | refs, diverged refs with the three resolutions. | |
| 365 | + | 4. **Failover.** Probe, ask/automatic, enforcement of the protection copy, | |
| 366 | + | replay with the table above, pull-request replay for protected branches. | |
| 367 | + | 5. **GitHub's workflows on g1t.** The three-way setting, the | |
| 368 | + | missing-check-suite signal, *Run on g1t*, backfill, check runs back to | |
| 369 | + | GitHub, held side-effect jobs, the secrets checklist. | |
| 370 | + | 6. **g1t ↔ g1t.** The `g1t` adapter, polling, the cycle check, docs for | |
| 371 | + | self-hosted ↔ g1t.sh. | |
| 372 | + | 7. **Remote pull requests on Mirrors.** List, open, merge through the API, | |
| 373 | + | agents opening theirs there (already "Not yet" in PLAN.md step 5). | |
| 374 | + | 8. **GitLab, Bitbucket, plain git** adapters. | |
| 375 | + | ||
| 376 | + | Each step updates `guides/github.md` and a new `guides/mirroring.md` in the | |
| 377 | + | same change (docs standard). `guides/github.md`'s "anything pushed to the | |
| 378 | + | g1t copy directly is overwritten" goes away with step 1. | |
| 379 | + | ||
| 380 | + | ## Decisions to confirm | |
| 381 | + | ||
| 382 | + | 1. **Failover defaults to "Stop accepting pushes"** for new links, with | |
| 383 | + | "Ask me" one click away. Automatic failover is opt-in, because it creates | |
| 384 | + | work that has to be replayed. | |
| 385 | + | 2. **On a Mirror, pull requests live on the leader**, and g1t has no pull | |
| 386 | + | requests of its own there (except during failover). One place to merge | |
| 387 | + | avoids two review histories for one branch. Making g1t the leader is the | |
| 388 | + | way to get g1t's review and queue. | |
| 389 | + | 3. **App permissions are asked for when a feature needs them**: | |
| 390 | + | `checks: read/write` for step 5 and `administration: write` only for | |
| 391 | + | *Lock the remote*. They are not requested up front. |
| 10 | 10 | import type { Release } from "./about"; | |
| 11 | 11 | import type { RepoRole } from "./access"; | |
| 12 | 12 | import type { CheckRunEventData, CheckSuiteEventData, StatusEventData } from "./checks"; | |
| 13 | + | import type { RepoMirror } from "./mirrors"; | |
| 13 | 14 | import type { DeploymentStatus, RepoDeployment } from "./deployments"; | |
| 14 | 15 | import type { TeamRole, TeamVisibility } from "./teams"; | |
| 15 | 16 | import type { Confidence, Verdict } from "./work"; | |
| ⋯ | |||
| 178 | 179 | * points to now, and `defaultBranch` whether it is the default branch. | |
| 179 | 180 | */ | |
| 180 | 181 | /** `before` is where the ref pointed before; absent for a new branch or tag. */ | |
| 181 | − | "git.push": { repoId: string; ref: string; before?: string; after: string; defaultBranch: boolean }; | |
| 182 | + | "git.push": { | |
| 183 | + | repoId: string; | |
| 184 | + | ref: string; | |
| 185 | + | before?: string; | |
| 186 | + | after: string; | |
| 187 | + | defaultBranch: boolean; | |
| 188 | + | /** Set when it was copied in from the remote a mirror follows, not made on g1t. */ | |
| 189 | + | mirrored?: boolean; | |
| 190 | + | /** The repository's mirror state when it landed; absent for one that leads. */ | |
| 191 | + | mirror?: RepoMirror; | |
| 192 | + | }; | |
| 182 | 193 | /** | |
| 183 | 194 | * `author` is who opened it: g1t, for one its agent filed while at work, | |
| 184 | 195 | * with `requestedBy` the person it was working for. Every issue and pull | |
| 25 | 25 | export * from "./integrations"; | |
| 26 | 26 | export * from "./members"; | |
| 27 | 27 | export * from "./mentions"; | |
| 28 | + | export * from "./mirrors"; | |
| 28 | 29 | export * from "./names"; | |
| 29 | 30 | export * from "./oauth"; | |
| 30 | 31 | export * from "./og"; |
| 1 | + | /** | |
| 2 | + | * Mirroring: a repository kept in step with copies of it on other hosts. | |
| 3 | + | * Mirrors `g1t_contracts::mirrors`. | |
| 4 | + | * | |
| 5 | + | * Every linked repository has exactly one leader, where work happens. A | |
| 6 | + | * **mirror** follows a remote that leads; while it stands by it is an exact, | |
| 7 | + | * read-only copy that runs nothing. Someone can **take over** (g1t leads for | |
| 8 | + | * a while, then **hands back**), or **move it to g1t** for good, after which | |
| 9 | + | * g1t no longer tracks the remote. A repository g1t leads can be **mirrored | |
| 10 | + | * to** any number of followers. | |
| 11 | + | */ | |
| 12 | + | import type { ServiceBinding } from "./clients"; | |
| 13 | + | import type { User } from "./identity"; | |
| 14 | + | import type { Result } from "./result"; | |
| 15 | + | ||
| 16 | + | /** Where a mirror stands with the remote it follows. */ | |
| 17 | + | export type MirrorState = "standby" | "ci" | "takeover" | "handing_back"; | |
| 18 | + | ||
| 19 | + | /** A repository's tie to the remote it mirrors, on `Repo.mirror`. */ | |
| 20 | + | export type RepoMirror = { | |
| 21 | + | state: MirrorState; | |
| 22 | + | /** For people: `github.com/acme/web`. */ | |
| 23 | + | remote: string; | |
| 24 | + | /** Its web address. */ | |
| 25 | + | url: string; | |
| 26 | + | /** RFC 3339: when it entered this state. */ | |
| 27 | + | since: string; | |
| 28 | + | warm?: boolean; | |
| 29 | + | githubWorkflows?: boolean; | |
| 30 | + | holdDeploys?: boolean; | |
| 31 | + | }; | |
| 32 | + | ||
| 33 | + | /** Whether the repository takes pushes, merges, issues and agents now. */ | |
| 34 | + | export function mirrorWritable(mirror: RepoMirror | null | undefined): boolean { | |
| 35 | + | return !mirror || mirror.state === "takeover"; | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | export type RemoteProvider = "github" | "g1t" | "git"; | |
| 39 | + | export type RemoteRole = "leader" | "follower"; | |
| 40 | + | export type RemoteState = MirrorState | "following" | "stuck"; | |
| 41 | + | ||
| 42 | + | export type MirrorSettings = { | |
| 43 | + | /** Who hears that the remote stopped answering: the banner only, or the inbox too. */ | |
| 44 | + | notify: "banner" | "inbox"; | |
| 45 | + | /** Take over on its own after this many minutes unreachable; null for only when someone does. */ | |
| 46 | + | takeOverAfter: number | null; | |
| 47 | + | /** When a takeover goes back once the remote answers: when someone says, or by itself when clean. */ | |
| 48 | + | handBack: "ask" | "when_clean"; | |
| 49 | + | keepCiWarm: boolean; | |
| 50 | + | githubWorkflows: boolean; | |
| 51 | + | holdDeploys: boolean; | |
| 52 | + | /** For followers: pushes made on the remote itself. */ | |
| 53 | + | remotePushes: "adopt" | "overwrite"; | |
| 54 | + | }; | |
| 55 | + | ||
| 56 | + | export const DEFAULT_MIRROR_SETTINGS: MirrorSettings = { | |
| 57 | + | notify: "banner", | |
| 58 | + | takeOverAfter: null, | |
| 59 | + | handBack: "when_clean", | |
| 60 | + | keepCiWarm: false, | |
| 61 | + | githubWorkflows: true, | |
| 62 | + | holdDeploys: true, | |
| 63 | + | remotePushes: "adopt", | |
| 64 | + | }; | |
| 65 | + | ||
| 66 | + | /** The shortest and longest wait before an automatic takeover, in minutes. */ | |
| 67 | + | export const TAKE_OVER_AFTER_MINUTES = [5, 24 * 60] as const; | |
| 68 | + | ||
| 69 | + | export type Remote = { | |
| 70 | + | id: string; | |
| 71 | + | repoId: string; | |
| 72 | + | repo: string; | |
| 73 | + | provider: RemoteProvider; | |
| 74 | + | role: RemoteRole; | |
| 75 | + | /** For people: `github.com/acme/web`. */ | |
| 76 | + | name: string; | |
| 77 | + | url: string; | |
| 78 | + | state: RemoteState; | |
| 79 | + | stateSince: string; | |
| 80 | + | /** A username, or `g1t` when it happened on its own. */ | |
| 81 | + | stateBy: string | null; | |
| 82 | + | reachable: boolean; | |
| 83 | + | unreachableSince: string | null; | |
| 84 | + | syncedAt: string | null; | |
| 85 | + | lastError: string | null; | |
| 86 | + | settings: MirrorSettings; | |
| 87 | + | createdAt: string; | |
| 88 | + | }; | |
| 89 | + | ||
| 90 | + | export type RefAction = "same" | "push" | "fetch" | "pull_request" | "diverged"; | |
| 91 | + | export type RefDecision = "keep_ours" | "keep_theirs" | "pull_request"; | |
| 92 | + | ||
| 93 | + | export type RefPlan = { | |
| 94 | + | ref: string; | |
| 95 | + | base: string | null; | |
| 96 | + | ours: string | null; | |
| 97 | + | theirs: string | null; | |
| 98 | + | action: RefAction; | |
| 99 | + | decision: RefDecision | null; | |
| 100 | + | }; | |
| 101 | + | ||
| 102 | + | export type HandbackPlan = { | |
| 103 | + | refs: RefPlan[]; | |
| 104 | + | reachable: boolean; | |
| 105 | + | ready: boolean; | |
| 106 | + | }; | |
| 107 | + | ||
| 108 | + | export type MirrorView = { | |
| 109 | + | remotes: Remote[]; | |
| 110 | + | plan: HandbackPlan | null; | |
| 111 | + | canManage: boolean; | |
| 112 | + | /** What the last action did that people should know: pull requests it opened, and so on. */ | |
| 113 | + | notes?: string[]; | |
| 114 | + | }; | |
| 115 | + | ||
| 116 | + | export type RemoteBrief = { | |
| 117 | + | repoId: string; | |
| 118 | + | role: RemoteRole; | |
| 119 | + | name: string; | |
| 120 | + | state: RemoteState; | |
| 121 | + | reachable: boolean; | |
| 122 | + | }; | |
| 123 | + | ||
| 124 | + | export type MirrorAddInput = { | |
| 125 | + | provider: Exclude<RemoteProvider, "github">; | |
| 126 | + | role: RemoteRole; | |
| 127 | + | url: string; | |
| 128 | + | username?: string | null; | |
| 129 | + | token?: string | null; | |
| 130 | + | }; | |
| 131 | + | ||
| 132 | + | async function rpc<T>(service: ServiceBinding, method: string, args: object): Promise<T> { | |
| 133 | + | const response = await service.fetch(`https://service/rpc/${method}`, { | |
| 134 | + | method: "POST", | |
| 135 | + | headers: { "content-type": "application/json" }, | |
| 136 | + | body: JSON.stringify(args), | |
| 137 | + | }); | |
| 138 | + | if (!response.ok) throw new Error(`${method} failed with status ${response.status}`); | |
| 139 | + | return (await response.json()) as T; | |
| 140 | + | } | |
| 141 | + | ||
| 142 | + | /** Mirroring: methods of the integrations service. */ | |
| 143 | + | export function mirrorsClient(integrations: ServiceBinding) { | |
| 144 | + | const call = <T>(method: string, args: object) => rpc<T>(integrations, method, args); | |
| 145 | + | return { | |
| 146 | + | view: (viewer: User | null, repoId: string) => call<Result<MirrorView>>("mirror_view", { viewer, repoId }), | |
| 147 | + | briefs: (repoIds: string[]) => call<RemoteBrief[]>("mirror_briefs", { repoIds }), | |
| 148 | + | takeOver: (actor: User, repoId: string) => call<Result<MirrorView>>("mirror_take_over", { actor, repoId }), | |
| 149 | + | ci: (actor: User, repoId: string, on: boolean) => call<Result<MirrorView>>("mirror_ci", { actor, repoId, on }), | |
| 150 | + | plan: (actor: User, repoId: string) => call<Result<MirrorView>>("mirror_hand_back_plan", { actor, repoId }), | |
| 151 | + | handBack: (actor: User, repoId: string, decisions: Record<string, RefDecision>) => | |
| 152 | + | call<Result<MirrorView>>("mirror_hand_back", { actor, repoId, decisions }), | |
| 153 | + | moveIn: (actor: User, repoId: string, keepRemoteUpdated: boolean) => | |
| 154 | + | call<Result<MirrorView>>("mirror_move_in", { actor, repoId, keepRemoteUpdated }), | |
| 155 | + | sync: (actor: User, repoId: string) => call<Result<MirrorView>>("mirror_sync", { actor, repoId }), | |
| 156 | + | settings: (actor: User, remoteId: string, settings: MirrorSettings) => | |
| 157 | + | call<Result<Remote>>("mirror_settings", { actor, remoteId, settings }), | |
| 158 | + | add: (actor: User, repoId: string, input: MirrorAddInput) => call<Result<Remote>>("mirror_add", { actor, repoId, ...input }), | |
| 159 | + | remove: (actor: User, remoteId: string) => call<Result<boolean>>("mirror_remove", { actor, remoteId }), | |
| 160 | + | }; | |
| 161 | + | } |
| 1 | 1 | import type { Contributors, Languages, License, NewRelease, Release, ReleaseChange, RepoAbout, Stargazer, StarredRepo, Stars } from "./about"; | |
| 2 | 2 | import type { User, Viewer } from "./identity"; | |
| 3 | + | import type { RepoMirror } from "./mirrors"; | |
| 3 | 4 | import type { Result } from "./result"; | |
| 4 | 5 | ||
| 5 | 6 | export type Repo = { | |
| ⋯ | |||
| 33 | 34 | * requests are locked. Null when it is not archived. | |
| 34 | 35 | */ | |
| 35 | 36 | archivedAt?: string | null; | |
| 37 | + | /** | |
| 38 | + | * Set when it mirrors a remote that leads. Unless g1t has taken over, | |
| 39 | + | * it is read-only: pushes, merges, issues, pull requests and agents are | |
| 40 | + | * refused, and nothing runs. See `./mirrors`. | |
| 41 | + | */ | |
| 42 | + | mirror?: RepoMirror | null; | |
| 36 | 43 | }; | |
| 37 | 44 | ||
| 38 | 45 | /** How long a deleted repository can be restored before it is purged. */ | |
| 23 | 23 | /** This month's open-source pool, or the repository's share of it, is spent. */ | |
| 24 | 24 | | "oss_pool_empty" | |
| 25 | 25 | /** A sensitive change needs a recent sign-in or the password again. */ | |
| 26 | − | | "reauth_required"; | |
| 26 | + | | "reauth_required" | |
| 27 | + | /** Another host g1t depends on for this did not answer. */ | |
| 28 | + | | "unavailable"; | |
| 27 | 29 | ||
| 28 | 30 | export type Failure = { code: FailureCode; message: string }; | |
| 29 | 31 | ||
| ⋯ | |||
| 50 | 52 | paused: 409, | |
| 51 | 53 | oss_pool_empty: 402, | |
| 52 | 54 | reauth_required: 403, | |
| 55 | + | unavailable: 503, | |
| 53 | 56 | }; | |
| 54 | 57 | ||
| 55 | 58 | export function httpStatus(failure: Failure): number { | |
| 1 | + | -- Mirroring: a repository's links to copies of it on other hosts, and the | |
| 2 | + | -- health of those hosts. See src/remotes.rs and crates/contracts | |
| 3 | + | -- src/mirrors.rs. Every timestamp is RFC 3339 UTC; *_ms are milliseconds | |
| 4 | + | -- since the epoch. | |
| 5 | + | ||
| 6 | + | -- One link. role leader: the remote leads and the repository is its | |
| 7 | + | -- mirror (at most one per repository). role follower: g1t leads and the | |
| 8 | + | -- remote is kept in step. | |
| 9 | + | -- | |
| 10 | + | -- provider: github | g1t | git. | |
| 11 | + | -- name: for people, `github.com/acme/web`. url: its web address. | |
| 12 | + | -- clone_url: its https git address. | |
| 13 | + | -- external_id: the provider's id for it (GitHub's numeric repository id, | |
| 14 | + | -- which survives renames). connection_id: GitHub's installation id. | |
| 15 | + | -- username, credential: for g1t and git, the user and token sent by basic | |
| 16 | + | -- authentication; the token sealed under INTEGRATIONS_KEY. | |
| 17 | + | -- state: standby | ci | takeover | handing_back (leaders), following | | |
| 18 | + | -- stuck (followers). state_by: a username, or g1t. | |
| 19 | + | -- settings: JSON `MirrorSettings`. | |
| 20 | + | -- recorded: 0 until the repos service has this state (set_mirror); the | |
| 21 | + | -- cron retries until it has. | |
| 22 | + | -- polled_ms: the last poll of a remote with no webhook. | |
| 23 | + | CREATE TABLE remotes ( | |
| 24 | + | id TEXT PRIMARY KEY, | |
| 25 | + | repo_id TEXT NOT NULL, | |
| 26 | + | workspace TEXT NOT NULL, | |
| 27 | + | repo TEXT NOT NULL, | |
| 28 | + | provider TEXT NOT NULL, | |
| 29 | + | role TEXT NOT NULL, | |
| 30 | + | name TEXT NOT NULL, | |
| 31 | + | url TEXT NOT NULL, | |
| 32 | + | clone_url TEXT NOT NULL, | |
| 33 | + | external_id TEXT, | |
| 34 | + | connection_id TEXT, | |
| 35 | + | username TEXT, | |
| 36 | + | credential TEXT, | |
| 37 | + | state TEXT NOT NULL, | |
| 38 | + | state_since TEXT NOT NULL, | |
| 39 | + | state_by TEXT, | |
| 40 | + | settings TEXT NOT NULL DEFAULT '{}', | |
| 41 | + | recorded INTEGER NOT NULL DEFAULT 0, | |
| 42 | + | polled_ms INTEGER NOT NULL DEFAULT 0, | |
| 43 | + | synced_at TEXT, | |
| 44 | + | last_error TEXT, | |
| 45 | + | created_by TEXT NOT NULL, | |
| 46 | + | created_at TEXT NOT NULL | |
| 47 | + | ); | |
| 48 | + | CREATE UNIQUE INDEX remotes_one_leader ON remotes (repo_id) WHERE role = 'leader'; | |
| 49 | + | CREATE INDEX remotes_by_repo ON remotes (repo_id); | |
| 50 | + | CREATE INDEX remotes_by_external ON remotes (provider, external_id); | |
| 51 | + | CREATE INDEX remotes_unrecorded ON remotes (recorded) WHERE recorded = 0; | |
| 52 | + | ||
| 53 | + | -- During a takeover: each ref's commit when it began (base), what both | |
| 54 | + | -- sides agreed on, and what someone decided for one that diverged. | |
| 55 | + | CREATE TABLE remote_refs ( | |
| 56 | + | remote_id TEXT NOT NULL, | |
| 57 | + | ref TEXT NOT NULL, | |
| 58 | + | base TEXT, | |
| 59 | + | decision TEXT, | |
| 60 | + | PRIMARY KEY (remote_id, ref) | |
| 61 | + | ); | |
| 62 | + | ||
| 63 | + | -- Whether a host answers. It is unreachable after three failed checks | |
| 64 | + | -- over at least two minutes, and reachable again after three good ones | |
| 65 | + | -- over at least five. | |
| 66 | + | -- | |
| 67 | + | -- failures, successes: in a row. streak_ms: when the current run began. | |
| 68 | + | -- unreachable_since: set while unreachable. | |
| 69 | + | CREATE TABLE remote_hosts ( | |
| 70 | + | host TEXT PRIMARY KEY, | |
| 71 | + | failures INTEGER NOT NULL DEFAULT 0, | |
| 72 | + | successes INTEGER NOT NULL DEFAULT 0, | |
| 73 | + | streak_ms INTEGER NOT NULL DEFAULT 0, | |
| 74 | + | unreachable_since TEXT, | |
| 75 | + | checked_ms INTEGER NOT NULL DEFAULT 0, | |
| 76 | + | last_problem TEXT | |
| 77 | + | ); | |
| 78 | + | ||
| 79 | + | -- The GitHub links that kept g1t in step (mirror) or GitHub in step | |
| 80 | + | -- (push) become remotes. A mirror becomes a standby mirror: read-only on | |
| 81 | + | -- g1t until someone takes over, which is what it always was in effect | |
| 82 | + | -- (anything pushed to it was overwritten at the next sync). | |
| 83 | + | INSERT INTO remotes | |
| 84 | + | (id, repo_id, workspace, repo, provider, role, name, url, clone_url, external_id, connection_id, | |
| 85 | + | state, state_since, settings, synced_at, last_error, created_by, created_at) | |
| 86 | + | SELECT | |
| 87 | + | 'rmt_' || lower(hex(randomblob(10))), repo_id, workspace, repo, 'github', | |
| 88 | + | CASE mode WHEN 'mirror' THEN 'leader' ELSE 'follower' END, | |
| 89 | + | 'github.com/' || full_name, 'https://github.com/' || full_name, 'https://github.com/' || full_name || '.git', | |
| 90 | + | CAST(github_repo_id AS TEXT), CAST(installation_id AS TEXT), | |
| 91 | + | CASE mode WHEN 'mirror' THEN 'standby' ELSE 'following' END, | |
| 92 | + | COALESCE(synced_at, created_at), '{}', synced_at, last_error, created_by, created_at | |
| 93 | + | FROM github_repos | |
| 94 | + | WHERE mode IN ('mirror', 'push'); |
| 15 | 15 | //! length or format is assumed. | |
| 16 | 16 | //! | |
| 17 | 17 | //! A repository comes across one of three ways (`GithubMode`): imported | |
| 18 | − | //! once; mirrored, so each push on GitHub is fetched into g1t; or pushed, | |
| 19 | − | //! so each push on g1t is sent to GitHub. Either way g1t holds a full copy, | |
| 20 | − | //! and the project built from it is hosted on g1t like any other. | |
| 18 | + | //! once; mirrored, so g1t keeps a standby copy that follows GitHub; or | |
| 19 | + | //! pushed, so GitHub follows g1t. Either way g1t holds a full copy, and the | |
| 20 | + | //! project built from it is hosted on g1t like any other. Mirrored and | |
| 21 | + | //! pushed repositories are links in `remotes` (see `remotes.rs`), which | |
| 22 | + | //! does everything after the import: following pushes, takeovers, | |
| 23 | + | //! hand-backs, moving in. | |
| 21 | 24 | //! | |
| 22 | 25 | //! The webhook, `https://api.g1t.sh/hooks/github`, is checked against | |
| 23 | 26 | //! GITHUB_APP_WEBHOOK_SECRET in constant time, de-duplicated by | |
| ⋯ | |||
| 27 | 30 | use std::collections::{HashMap, HashSet}; | |
| 28 | 31 | ||
| 29 | 32 | use g1t_contracts::access::{self, Capability}; | |
| 30 | − | use g1t_contracts::events::Event; | |
| 31 | 33 | use g1t_contracts::github::*; | |
| 32 | 34 | use g1t_contracts::integrations::Received; | |
| 33 | − | use g1t_contracts::repos::{CreateArgs, MirrorArgs, MirrorDirection, Mirrored, Repo, RepoPath}; | |
| 35 | + | use g1t_contracts::mirrors::{MirrorActArgs, MirrorState, RepoMirror}; | |
| 36 | + | use g1t_contracts::repos::{CreateArgs, Repo, RepoPath}; | |
| 34 | 37 | use g1t_contracts::time::rfc3339; | |
| 35 | 38 | use g1t_contracts::work::{Issue, IssueActionArgs, IssueReason, OpenIssueArgs}; | |
| 36 | 39 | use g1t_contracts::{FailureCode, Outcome, PrincipalKind, Role, User, is_valid_repo_name}; | |
| ⋯ | |||
| 288 | 291 | }) | |
| 289 | 292 | } | |
| 290 | 293 | ||
| 294 | + | /// GitHub's REST API, with an installation or user token. | |
| 295 | + | pub(crate) async fn api(&self, method: Method, path: &str, token: &str, body: Option<Value>) -> Result<Answer> { | |
| 296 | + | self.github(method, path, token, body).await | |
| 297 | + | } | |
| 298 | + | ||
| 291 | 299 | async fn github(&self, method: Method, path: &str, token: &str, body: Option<Value>) -> Result<Answer> { | |
| 292 | 300 | let authorization = format!("Bearer {token}"); | |
| 293 | 301 | let url = if path.starts_with("https://") { path.to_owned() } else { format!("{API}{path}") }; | |
| ⋯ | |||
| 310 | 318 | } | |
| 311 | 319 | ||
| 312 | 320 | /// An installation access token, from the cache or fresh from GitHub. | |
| 313 | − | async fn installation_token(&self, installation_id: u64) -> Result<std::result::Result<String, String>> { | |
| 321 | + | pub(crate) async fn installation_token(&self, installation_id: u64) -> Result<std::result::Result<String, String>> { | |
| 314 | 322 | #[derive(Deserialize)] | |
| 315 | 323 | struct Cached { | |
| 316 | 324 | token: String, | |
| ⋯ | |||
| 495 | 503 | ||
| 496 | 504 | /// Forgets an installation in a workspace; its mirrors stop. The app | |
| 497 | 505 | /// stays installed on GitHub until it is uninstalled there. | |
| 498 | − | pub async fn remove_installation(&self, a: GithubInstallationArgs) -> Result<Outcome<bool>> { | |
| 506 | + | pub async fn remove_installation(&self, a: GithubInstallationArgs, mirrors: Option<&crate::remotes::Mirrors>) -> Result<Outcome<bool>> { | |
| 499 | 507 | let workspace = a.workspace.to_lowercase(); | |
| 500 | 508 | if let Some(refused) = Self::owner_only(&a.actor, &workspace) { | |
| 501 | 509 | return Ok(refused); | |
| ⋯ | |||
| 510 | 518 | .bind(&[workspace.as_str().into(), number(a.installation_id)])? | |
| 511 | 519 | .run() | |
| 512 | 520 | .await?; | |
| 521 | + | if let Some(mirrors) = mirrors { | |
| 522 | + | let rows = self | |
| 523 | + | .db | |
| 524 | + | .prepare("SELECT id FROM remotes WHERE provider = 'github' AND workspace = ? AND connection_id = ?") | |
| 525 | + | .bind(&[workspace.as_str().into(), a.installation_id.to_string().into()])? | |
| 526 | + | .all() | |
| 527 | + | .await? | |
| 528 | + | .results::<Value>()?; | |
| 529 | + | for id in rows.iter().filter_map(|row| row["id"].as_str()) { | |
| 530 | + | mirrors.link_gone(id, "lost its GitHub account in this workspace").await?; | |
| 531 | + | } | |
| 532 | + | } | |
| 513 | 533 | Ok(Outcome::Ok(true)) | |
| 514 | 534 | } | |
| 515 | 535 | ||
| ⋯ | |||
| 635 | 655 | is_private: a.private.unwrap_or_else(|| github["private"].as_bool().unwrap_or(true)), | |
| 636 | 656 | import_url: Some(clone_url.to_owned()), | |
| 637 | 657 | import_token: Some(token), | |
| 658 | + | // A mirror is read-only from the start. | |
| 659 | + | mirror: (a.mode == GithubMode::Mirror).then(|| RepoMirror { | |
| 660 | + | state: MirrorState::Standby, | |
| 661 | + | remote: format!("github.com/{full_name}"), | |
| 662 | + | url: format!("https://github.com/{full_name}"), | |
| 663 | + | since: rfc3339(now_ms()), | |
| 664 | + | warm: false, | |
| 665 | + | github_workflows: true, | |
| 666 | + | hold_deploys: true, | |
| 667 | + | }), | |
| 638 | 668 | }, | |
| 639 | 669 | ) | |
| 640 | 670 | .await?; | |
| ⋯ | |||
| 663 | 693 | ])? | |
| 664 | 694 | .run() | |
| 665 | 695 | .await?; | |
| 696 | + | if a.mode != GithubMode::Import { | |
| 697 | + | let (role, state) = if a.mode == GithubMode::Mirror { ("leader", "standby") } else { ("follower", "following") }; | |
| 698 | + | self.db | |
| 699 | + | .prepare( | |
| 700 | + | "INSERT INTO remotes | |
| 701 | + | (id, repo_id, workspace, repo, provider, role, name, url, clone_url, external_id, connection_id, | |
| 702 | + | state, state_since, state_by, settings, recorded, synced_at, created_by, created_at) | |
| 703 | + | VALUES (?1, ?2, ?3, ?4, 'github', ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, '{}', 1, ?12, ?14, ?12)", | |
| 704 | + | ) | |
| 705 | + | .bind(&[ | |
| 706 | + | g1t_contracts::new_id("rmt", now_ms()).into(), | |
| 707 | + | repo.id.as_str().into(), | |
| 708 | + | workspace.as_str().into(), | |
| 709 | + | path.as_str().into(), | |
| 710 | + | role.into(), | |
| 711 | + | format!("github.com/{full_name}").into(), | |
| 712 | + | format!("https://github.com/{full_name}").into(), | |
| 713 | + | format!("https://github.com/{full_name}.git").into(), | |
| 714 | + | a.github_repo_id.to_string().into(), | |
| 715 | + | a.installation_id.to_string().into(), | |
| 716 | + | state.into(), | |
| 717 | + | now.as_str().into(), | |
| 718 | + | a.actor.username.as_str().into(), | |
| 719 | + | a.actor.id.as_str().into(), | |
| 720 | + | ])? | |
| 721 | + | .run() | |
| 722 | + | .await?; | |
| 723 | + | } | |
| 666 | 724 | let job = a.issues.then(|| IssueJob { | |
| 667 | 725 | actor: a.actor.clone(), | |
| 668 | 726 | repo: RepoPath { | |
| ⋯ | |||
| 777 | 835 | } | |
| 778 | 836 | ||
| 779 | 837 | pub async fn link_for(&self, a: GithubLinkArgs) -> Result<Option<GithubRepoLink>> { | |
| 780 | − | Ok(self.link(&a.repo_id).await?.map(Into::into)) | |
| 838 | + | let Some(row) = self.link(&a.repo_id).await? else { return Ok(None) }; | |
| 839 | + | // What the repository is now is the remote's to say. | |
| 840 | + | let role = self | |
| 841 | + | .db | |
| 842 | + | .prepare("SELECT role FROM remotes WHERE repo_id = ? AND provider = 'github' AND external_id = ?") | |
| 843 | + | .bind(&[a.repo_id.as_str().into(), row.github_repo_id.to_string().into()])? | |
| 844 | + | .first::<String>(Some("role")) | |
| 845 | + | .await?; | |
| 846 | + | let mut link: GithubRepoLink = row.into(); | |
| 847 | + | link.mode = match role.as_deref() { | |
| 848 | + | Some("leader") => GithubMode::Mirror, | |
| 849 | + | Some("follower") => GithubMode::Push, | |
| 850 | + | _ => GithubMode::Import, | |
| 851 | + | }; | |
| 852 | + | Ok(Some(link)) | |
| 781 | 853 | } | |
| 782 | 854 | ||
| 783 | − | /// Stops a mirror: the g1t repository keeps what it has and is its own. | |
| 784 | − | pub async fn unlink_repo(&self, a: GithubUnlinkRepoArgs) -> Result<Outcome<bool>> { | |
| 855 | + | /// Stops a link to GitHub: the g1t repository keeps what it has and is | |
| 856 | + | /// its own. Refused during a takeover (see `remotes.rs`). | |
| 857 | + | pub async fn unlink_repo(&self, a: GithubUnlinkRepoArgs, env: &Env) -> Result<Outcome<bool>> { | |
| 785 | 858 | let Some(row) = self.link(&a.repo_id).await? else { | |
| 786 | 859 | return Ok(Outcome::Ok(false)); | |
| 787 | 860 | }; | |
| 788 | 861 | if let Some(refused) = refused(&a.actor, &row, Capability::ManageIntegrations) { | |
| 789 | 862 | return Ok(refused); | |
| 790 | 863 | } | |
| 864 | + | let ids = self | |
| 865 | + | .db | |
| 866 | + | .prepare("SELECT id FROM remotes WHERE repo_id = ? AND provider = 'github'") | |
| 867 | + | .bind(&[a.repo_id.as_str().into()])? | |
| 868 | + | .all() | |
| 869 | + | .await? | |
| 870 | + | .results::<Value>()?; | |
| 871 | + | let mirrors = crate::remotes::Mirrors::new(env)?; | |
| 872 | + | for id in ids.iter().filter_map(|row| row["id"].as_str()) { | |
| 873 | + | let removed = mirrors | |
| 874 | + | .remove(g1t_contracts::mirrors::MirrorRemoveArgs { actor: a.actor.clone(), remote_id: id.to_owned() }) | |
| 875 | + | .await?; | |
| 876 | + | if let Outcome::Fail(failure) = removed { | |
| 877 | + | return Ok(Outcome::Fail(failure)); | |
| 878 | + | } | |
| 879 | + | } | |
| 791 | 880 | self.db | |
| 792 | 881 | .prepare("UPDATE github_repos SET mode = 'import' WHERE repo_id = ?") | |
| 793 | 882 | .bind(&[a.repo_id.as_str().into()])? | |
| ⋯ | |||
| 796 | 885 | Ok(Outcome::Ok(true)) | |
| 797 | 886 | } | |
| 798 | 887 | ||
| 799 | − | /// Copies refs now, in the link's direction. | |
| 800 | − | async fn sync(&self, row: &LinkRow) -> Result<std::result::Result<Mirrored, String>> { | |
| 801 | − | let direction = match GithubMode::parse(&row.mode) { | |
| 802 | − | GithubMode::Mirror => MirrorDirection::Pull, | |
| 803 | − | GithubMode::Push => MirrorDirection::Push, | |
| 804 | − | GithubMode::Import => return Ok(Err("This repository was imported once; it is not mirrored.".to_owned())), | |
| 805 | − | }; | |
| 806 | − | let token = match self.installation_token(row.installation_id).await? { | |
| 807 | − | Ok(token) => token, | |
| 808 | − | Err(reason) => { | |
| 809 | − | self.note(&row.repo_id, Some(&reason)).await?; | |
| 810 | − | return Ok(Err(reason)); | |
| 811 | − | } | |
| 812 | − | }; | |
| 813 | − | let done: Outcome<Mirrored> = g1t_kit::call( | |
| 814 | − | &self.repos, | |
| 815 | − | "mirror", | |
| 816 | − | &MirrorArgs { | |
| 817 | − | repo_id: row.repo_id.clone(), | |
| 818 | − | url: format!("https://github.com/{}.git", row.full_name), | |
| 819 | − | token, | |
| 820 | − | direction, | |
| 821 | − | }, | |
| 822 | − | ) | |
| 823 | − | .await?; | |
| 824 | − | Ok(match done { | |
| 825 | − | Outcome::Ok(mirrored) => { | |
| 826 | − | self.note(&row.repo_id, None).await?; | |
| 827 | − | Ok(mirrored) | |
| 828 | − | } | |
| 829 | − | Outcome::Fail(failure) => { | |
| 830 | − | self.note(&row.repo_id, Some(&failure.message)).await?; | |
| 831 | − | Err(failure.message) | |
| 832 | − | } | |
| 833 | − | }) | |
| 834 | − | } | |
| 835 | − | ||
| 836 | − | pub async fn sync_now(&self, a: GithubUnlinkRepoArgs) -> Result<Outcome<GithubRepoLink>> { | |
| 837 | − | let Some(row) = self.link(&a.repo_id).await? else { | |
| 888 | + | pub async fn sync_now(&self, a: GithubUnlinkRepoArgs, env: &Env) -> Result<Outcome<GithubRepoLink>> { | |
| 889 | + | if self.link(&a.repo_id).await?.is_none() { | |
| 838 | 890 | return Ok(fail(FailureCode::NotFound, "This repository is not linked to GitHub.")); | |
| 839 | − | }; | |
| 840 | − | // Syncing brings commits in, as pushing does. | |
| 841 | − | if let Some(refused) = refused(&a.actor, &row, Capability::Push) { | |
| 842 | − | return Ok(refused); | |
| 843 | 891 | } | |
| 844 | − | if let Err(reason) = self.sync(&row).await? { | |
| 845 | − | return Ok(fail(FailureCode::Conflict, reason)); | |
| 892 | + | let synced = crate::remotes::Mirrors::new(env)? | |
| 893 | + | .sync_now(MirrorActArgs { actor: a.actor.clone(), repo_id: a.repo_id.clone() }) | |
| 894 | + | .await?; | |
| 895 | + | if let Outcome::Fail(failure) = synced { | |
| 896 | + | return Ok(Outcome::Fail(failure)); | |
| 846 | 897 | } | |
| 847 | − | Ok(self.link(&a.repo_id).await?.map_or_else(|| fail(FailureCode::NotFound, "Gone."), |row| Outcome::Ok(row.into()))) | |
| 898 | + | Ok(self | |
| 899 | + | .link_for(GithubLinkArgs { repo_id: a.repo_id.clone() }) | |
| 900 | + | .await? | |
| 901 | + | .map_or_else(|| fail(FailureCode::NotFound, "Gone."), Outcome::Ok)) | |
| 848 | 902 | } | |
| 849 | 903 | ||
| 850 | 904 | // --- The webhook ----------------------------------------------------------- | |
| ⋯ | |||
| 882 | 936 | Ok((answer(202, "Received."), Some((event, payload)))) | |
| 883 | 937 | } | |
| 884 | 938 | ||
| 885 | − | pub async fn process(&self, event: &str, payload: &Value) -> Result<()> { | |
| 939 | + | pub async fn process(&self, event: &str, payload: &Value, mirrors: Option<&crate::remotes::Mirrors>) -> Result<()> { | |
| 886 | 940 | let action = payload["action"].as_str().unwrap_or_default(); | |
| 887 | 941 | let installation = payload["installation"]["id"].as_u64(); | |
| 888 | 942 | match (event, action) { | |
| 889 | 943 | ("installation", "deleted") | ("installation", "suspend") | ("installation", "unsuspend") => { | |
| 890 | 944 | let Some(id) = installation else { return Ok(()) }; | |
| 891 | 945 | match action { | |
| 892 | − | "deleted" => self.installation_gone(id, "The GitHub App was uninstalled from this account.").await?, | |
| 946 | + | "deleted" => self.installation_gone(id, "The GitHub App was uninstalled from this account.", mirrors).await?, | |
| 893 | 947 | "suspend" => { | |
| 894 | 948 | self.db | |
| 895 | 949 | .prepare("UPDATE github_installations SET suspended_at = ? WHERE id = ?") | |
| ⋯ | |||
| 923 | 977 | } | |
| 924 | 978 | } | |
| 925 | 979 | ("push", _) => { | |
| 926 | − | let Some(repo) = payload["repository"]["id"].as_u64() else { return Ok(()) }; | |
| 927 | − | let rows = self | |
| 928 | − | .db | |
| 929 | − | .prepare("SELECT * FROM github_repos WHERE github_repo_id = ? AND mode = 'mirror'") | |
| 930 | − | .bind(&[number(repo)])? | |
| 931 | − | .all() | |
| 932 | − | .await? | |
| 933 | − | .results::<LinkRow>()?; | |
| 934 | − | for row in rows { | |
| 935 | − | if let Err(reason) = self.sync(&row).await? { | |
| 936 | − | worker::console_log!("github mirror of {} not synced: {reason}", row.repo); | |
| 937 | − | } | |
| 980 | + | if let Some(mirrors) = mirrors { | |
| 981 | + | mirrors.on_github_push(payload).await?; | |
| 938 | 982 | } | |
| 939 | 983 | } | |
| 940 | 984 | ("repository", "renamed") | ("repository", "transferred") => { | |
| ⋯ | |||
| 946 | 990 | .bind(&[full_name.into(), number(repo)])? | |
| 947 | 991 | .run() | |
| 948 | 992 | .await?; | |
| 993 | + | self.db | |
| 994 | + | .prepare( | |
| 995 | + | "UPDATE remotes SET name = ?1, url = ?2, clone_url = ?3, recorded = 0 | |
| 996 | + | WHERE provider = 'github' AND external_id = ?4", | |
| 997 | + | ) | |
| 998 | + | .bind(&[ | |
| 999 | + | format!("github.com/{full_name}").into(), | |
| 1000 | + | format!("https://github.com/{full_name}").into(), | |
| 1001 | + | format!("https://github.com/{full_name}.git").into(), | |
| 1002 | + | repo.to_string().into(), | |
| 1003 | + | ])? | |
| 1004 | + | .run() | |
| 1005 | + | .await?; | |
| 949 | 1006 | } | |
| 950 | 1007 | ("repository", "deleted") => { | |
| 951 | 1008 | if let Some(repo) = payload["repository"]["id"].as_u64() { | |
| 952 | 1009 | self.mark_github_repo(repo, "The repository was deleted on GitHub. The copy on g1t is kept.").await?; | |
| 1010 | + | self.links_gone("external_id", &repo.to_string(), "was deleted on GitHub", mirrors).await?; | |
| 953 | 1011 | self.db | |
| 954 | 1012 | .prepare("UPDATE github_repos SET mode = 'import' WHERE github_repo_id = ?") | |
| 955 | 1013 | .bind(&[number(repo)])? | |
| ⋯ | |||
| 971 | 1029 | .results::<Value>()?; | |
| 972 | 1030 | for row in ids { | |
| 973 | 1031 | if let Some(id) = row["id"].as_u64() { | |
| 974 | − | self.installation_gone(id, "g1t's GitHub App was deleted.").await?; | |
| 1032 | + | self.installation_gone(id, "g1t's GitHub App was deleted.", mirrors).await?; | |
| 975 | 1033 | } | |
| 976 | 1034 | } | |
| 977 | 1035 | } | |
| ⋯ | |||
| 986 | 1044 | Ok(()) | |
| 987 | 1045 | } | |
| 988 | 1046 | ||
| 989 | − | async fn installation_gone(&self, id: u64, why: &str) -> Result<()> { | |
| 1047 | + | /// The remotes GitHub no longer lets g1t reach: a mirror standing by | |
| 1048 | + | /// becomes an ordinary repository with what it has (it can no longer | |
| 1049 | + | /// follow), a follower stops; a takeover keeps going and is told why. | |
| 1050 | + | async fn links_gone(&self, column: &str, value: &str, why: &str, mirrors: Option<&crate::remotes::Mirrors>) -> Result<()> { | |
| 1051 | + | let Some(mirrors) = mirrors else { return Ok(()) }; | |
| 1052 | + | let column = if column == "connection_id" { "connection_id" } else { "external_id" }; | |
| 1053 | + | let rows = self | |
| 1054 | + | .db | |
| 1055 | + | .prepare(format!("SELECT id FROM remotes WHERE provider = 'github' AND {column} = ?")) | |
| 1056 | + | .bind(&[value.into()])? | |
| 1057 | + | .all() | |
| 1058 | + | .await? | |
| 1059 | + | .results::<Value>()?; | |
| 1060 | + | for id in rows.iter().filter_map(|row| row["id"].as_str()) { | |
| 1061 | + | mirrors.link_gone(id, why).await?; | |
| 1062 | + | } | |
| 1063 | + | Ok(()) | |
| 1064 | + | } | |
| 1065 | + | ||
| 1066 | + | async fn installation_gone(&self, id: u64, why: &str, mirrors: Option<&crate::remotes::Mirrors>) -> Result<()> { | |
| 1067 | + | self.links_gone("connection_id", &id.to_string(), "lost its GitHub App installation", mirrors).await?; | |
| 990 | 1068 | self.db.prepare("DELETE FROM github_installations WHERE id = ?").bind(&[number(id)])?.run().await?; | |
| 991 | 1069 | self.db.prepare("DELETE FROM github_tokens WHERE installation_id = ?").bind(&[number(id)])?.run().await?; | |
| 992 | 1070 | self.db | |
| ⋯ | |||
| 1003 | 1081 | .bind(&[why.into(), number(github_repo_id)])? | |
| 1004 | 1082 | .run() | |
| 1005 | 1083 | .await?; | |
| 1006 | − | Ok(()) | |
| 1007 | − | } | |
| 1008 | − | ||
| 1009 | − | /// A push on g1t: a repository GitHub follows is pushed out. | |
| 1010 | − | pub async fn on_event(&self, event: &Event) -> Result<()> { | |
| 1011 | − | if event.kind != "git.push" { | |
| 1012 | − | return Ok(()); | |
| 1013 | − | } | |
| 1014 | − | let Some(repo_id) = event.repo_id.as_deref() else { | |
| 1015 | − | return Ok(()); | |
| 1016 | − | }; | |
| 1017 | − | if let Some(row) = self.link(repo_id).await?.filter(|row| row.mode == "push") | |
| 1018 | − | && let Err(reason) = self.sync(&row).await? | |
| 1019 | − | { | |
| 1020 | − | worker::console_log!("github push mirror of {} failed: {reason}", row.repo); | |
| 1021 | − | } | |
| 1022 | 1084 | Ok(()) | |
| 1023 | 1085 | } | |
| 1024 | 1086 | } | |
| ⋯ | |||
| 1036 | 1098 | match method { | |
| 1037 | 1099 | "github_status" => reply(&app.status(args(body)?).await?), | |
| 1038 | 1100 | "github_add_installation" => reply(&app.add_installation(args(body)?).await?), | |
| 1039 | − | "github_remove_installation" => reply(&app.remove_installation(args(body)?).await?), | |
| 1101 | + | "github_remove_installation" => { | |
| 1102 | + | let mirrors = crate::remotes::Mirrors::new(env).ok(); | |
| 1103 | + | reply(&app.remove_installation(args(body)?, mirrors.as_ref()).await?) | |
| 1104 | + | } | |
| 1040 | 1105 | "github_repositories" => reply(&app.repositories(args(body)?).await?), | |
| 1041 | 1106 | "github_import" => { | |
| 1042 | 1107 | let (outcome, job) = app.import(args(body)?).await?; | |
| ⋯ | |||
| 1052 | 1117 | reply(&outcome) | |
| 1053 | 1118 | } | |
| 1054 | 1119 | "github_link" => reply(&app.link_for(args(body)?).await?), | |
| 1055 | − | "github_unlink_repo" => reply(&app.unlink_repo(args(body)?).await?), | |
| 1056 | − | "github_sync" => reply(&app.sync_now(args(body)?).await?), | |
| 1120 | + | "github_unlink_repo" => reply(&app.unlink_repo(args(body)?, env).await?), | |
| 1121 | + | "github_sync" => reply(&app.sync_now(args(body)?, env).await?), | |
| 1057 | 1122 | "github_receive" => { | |
| 1058 | 1123 | let received: GithubReceiveArgs = args(body)?; | |
| 1059 | 1124 | let (answer, work) = app.receive(&received).await?; | |
| ⋯ | |||
| 1061 | 1126 | let env = env.clone(); | |
| 1062 | 1127 | ctx.wait_until(async move { | |
| 1063 | 1128 | let Ok(app) = GithubApp::new(&env) else { return }; | |
| 1064 | − | if let Err(error) = app.process(&event, &payload).await { | |
| 1129 | + | let mirrors = crate::remotes::Mirrors::new(&env).ok(); | |
| 1130 | + | if let Err(error) = app.process(&event, &payload, mirrors.as_ref()).await { | |
| 1065 | 1131 | worker::console_error!("github: acting on a {event} delivery failed: {error}"); | |
| 1066 | 1132 | } | |
| 1067 | 1133 | }); | |
| ⋯ | |||
| 1070 | 1136 | } | |
| 1071 | 1137 | _ => Response::error("Unknown method", 404), | |
| 1072 | 1138 | } | |
| 1073 | − | } | |
| 1074 | − | ||
| 1075 | − | /// For the queue: pushes on g1t that GitHub follows. | |
| 1076 | − | pub async fn on_event(env: &Env, event: &Event) -> Result<()> { | |
| 1077 | − | if event.kind != "git.push" || !AppConfig::from_env(env).configured() { | |
| 1078 | − | return Ok(()); | |
| 1079 | − | } | |
| 1080 | − | GithubApp::new(env)?.on_event(event).await | |
| 1081 | 1139 | } | |
| 1082 | 1140 | ||
| 1083 | 1141 | #[cfg(test)] | |
| 14 | 14 | mod http; | |
| 15 | 15 | mod models; | |
| 16 | 16 | mod refs; | |
| 17 | + | mod remotes; | |
| 17 | 18 | mod rename; | |
| 18 | 19 | mod sentry; | |
| 19 | 20 | mod trackers; | |
| ⋯ | |||
| 29 | 30 | use serde::{Deserialize, Serialize}; | |
| 30 | 31 | use serde_json::{Value, json}; | |
| 31 | 32 | use worker::wasm_bindgen::JsValue; | |
| 32 | − | use worker::{Context, D1Database, Env, Fetcher, MessageBatch, MessageExt, Request, Response, Result, event}; | |
| 33 | + | use worker::{Context, D1Database, Env, Fetcher, MessageBatch, MessageExt, Request, Response, Result, ScheduleContext, ScheduledEvent, event}; | |
| 33 | 34 | ||
| 34 | 35 | use alerts::{Action, Signal}; | |
| 35 | 36 | use g1t_secrets::{self as crypto, Sealer}; | |
| ⋯ | |||
| 1538 | 1539 | if let Some(answer) = github::route(&method, &body, &env, &ctx).await { | |
| 1539 | 1540 | return answer; | |
| 1540 | 1541 | } | |
| 1542 | + | // Mirroring: links to other hosts, takeovers and hand-backs; see remotes.rs. | |
| 1543 | + | if let Some(answer) = remotes::route(&method, &body, &env, &ctx).await { | |
| 1544 | + | return answer; | |
| 1545 | + | } | |
| 1541 | 1546 | let service = Integrations::new(&env)?; | |
| 1542 | 1547 | match method.as_str() { | |
| 1543 | 1548 | "list" => reply(&service.list(args(body)?).await?), | |
| ⋯ | |||
| 1598 | 1603 | } | |
| 1599 | 1604 | // A repository purged: what was kept for it goes. | |
| 1600 | 1605 | rename::on_purged(&env.d1("DB")?, message.body()).await?; | |
| 1601 | − | github::on_event(&env, message.body()).await?; | |
| 1606 | + | if message.body().kind == "git.push" { | |
| 1607 | + | remotes::Mirrors::new(&env)?.on_event(message.body()).await?; | |
| 1608 | + | } | |
| 1602 | 1609 | service.on_event(message.body()).await?; | |
| 1603 | 1610 | message.ack(); | |
| 1604 | 1611 | } | |
| 1605 | 1612 | Ok(()) | |
| 1606 | 1613 | } | |
| 1607 | 1614 | ||
| 1615 | + | /// Every minute: mirrors' hosts checked, links the repos service has not | |
| 1616 | + | /// heard of recorded, remotes with no webhook polled, and the takeovers and | |
| 1617 | + | /// hand-backs people asked to happen on their own. See remotes.rs. | |
| 1618 | + | #[event(scheduled)] | |
| 1619 | + | async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { | |
| 1620 | + | match remotes::Mirrors::new(&env) { | |
| 1621 | + | Ok(mirrors) => { | |
| 1622 | + | if let Err(error) = mirrors.on_minute().await { | |
| 1623 | + | worker::console_error!("integrations: the mirrors' minute failed: {error}"); | |
| 1624 | + | } | |
| 1625 | + | } | |
| 1626 | + | Err(error) => worker::console_error!("integrations: mirrors unavailable: {error}"), | |
| 1627 | + | } | |
| 1628 | + | } | |
| 1629 | + | ||
| 1608 | 1630 | #[cfg(test)] | |
| 1609 | 1631 | mod close_tests { | |
| 1610 | 1632 | use super::{closable_hashes, requester}; | |
| 1 | + | //! Mirroring: a repository's links to copies of it on other hosts (see | |
| 2 | + | //! `g1t_contracts::mirrors` for the model). | |
| 3 | + | //! | |
| 4 | + | //! Every linked repository has exactly one leader. A **mirror** follows a | |
| 5 | + | //! remote that leads: it stands by as an exact, read-only copy, and nothing | |
| 6 | + | //! runs on it. Someone can turn on **CI failover** (the remote keeps the | |
| 7 | + | //! code, g1t runs its workflows) or **take over** (g1t leads for a while). | |
| 8 | + | //! A takeover is **handed back** ref by ref: what only g1t changed is | |
| 9 | + | //! pushed, a protected branch goes as a pull request, and a branch both | |
| 10 | + | //! sides changed waits for a person's decision. A mirror can also be | |
| 11 | + | //! **moved to g1t** for good, after which g1t no longer tracks the remote. | |
| 12 | + | //! A repository g1t leads can be **mirrored to** any number of followers. | |
| 13 | + | //! | |
| 14 | + | //! This service keeps the links (`remotes`), each host's health | |
| 15 | + | //! (`remote_hosts`) and a takeover's starting point (`remote_refs`), and | |
| 16 | + | //! decides. The repos service keeps each repository's `RepoMirror`, set | |
| 17 | + | //! here with `set_mirror`, so pushes and merges are refused or allowed | |
| 18 | + | //! without asking. Git itself moves through the repos service (`mirror`, | |
| 19 | + | //! `mirror_refs`, `mirror_apply`). | |
| 20 | + | //! | |
| 21 | + | //! Hosts are adapters: [`RemoteProvider`] names them, and the few places a | |
| 22 | + | //! host's own API is used (credentials, whether a branch is protected, | |
| 23 | + | //! opening a pull request) match on it. GitHub goes through g1t's GitHub | |
| 24 | + | //! App; another g1t or any git host takes a username and token. A host | |
| 25 | + | //! that sends no webhook is polled. | |
| 26 | + | //! | |
| 27 | + | //! Nothing happens on its own unless someone asked for it in the link's | |
| 28 | + | //! settings: by default an unreachable remote is only shown on the | |
| 29 | + | //! repository, and a takeover starts when someone starts it. | |
| 30 | + | ||
| 31 | + | use std::collections::{BTreeMap, BTreeSet, HashMap}; | |
| 32 | + | ||
| 33 | + | use g1t_contracts::access::{self, Capability}; | |
| 34 | + | use g1t_contracts::events::{NewEvent, Publish}; | |
| 35 | + | use g1t_contracts::identity::{ListMembersArgs, Member}; | |
| 36 | + | use g1t_contracts::mirrors::*; | |
| 37 | + | use g1t_contracts::repos::{ | |
| 38 | + | GetByIdArgs, MirrorApplied, MirrorApplyArgs, MirrorArgs, MirrorDirection, MirrorRefs, MirrorRefsArgs, Mirrored, RefMove, | |
| 39 | + | Repo, SetMirrorArgs, | |
| 40 | + | }; | |
| 41 | + | use g1t_contracts::time::rfc3339; | |
| 42 | + | use g1t_contracts::{FailureCode, Outcome, Role, User, new_id}; | |
| 43 | + | use g1t_kit::{args, now_ms, reply}; | |
| 44 | + | use g1t_secrets::Sealer; | |
| 45 | + | use serde::Deserialize; | |
| 46 | + | use serde_json::Value; | |
| 47 | + | use worker::wasm_bindgen::JsValue; | |
| 48 | + | use worker::{Context, D1Database, Env, Fetcher, Method, Response, Result}; | |
| 49 | + | ||
| 50 | + | use crate::github::GithubApp; | |
| 51 | + | use crate::http; | |
| 52 | + | ||
| 53 | + | const SOURCE: &str = "integrations"; | |
| 54 | + | /// A host is unreachable after this many failed checks in a row, spanning | |
| 55 | + | /// at least [`DOWN_AFTER_MS`]. | |
| 56 | + | const DOWN_CHECKS: u32 = 3; | |
| 57 | + | const DOWN_AFTER_MS: u64 = 2 * 60 * 1000; | |
| 58 | + | /// And reachable again after this many good ones, spanning [`UP_AFTER_MS`]. | |
| 59 | + | const UP_CHECKS: u32 = 3; | |
| 60 | + | const UP_AFTER_MS: u64 = 5 * 60 * 1000; | |
| 61 | + | /// How often a remote with no webhook is asked for news. | |
| 62 | + | const POLL_MS: u64 = 5 * 60 * 1000; | |
| 63 | + | /// Where g1t's commits go when the remote will not take them directly. | |
| 64 | + | const HANDBACK_PREFIX: &str = "refs/heads/g1t/handback/"; | |
| 65 | + | /// The most branches asked about protection in one plan. | |
| 66 | + | const MAX_PROTECTION_CHECKS: usize = 20; | |
| 67 | + | ||
| 68 | + | fn fail<T>(code: FailureCode, message: impl Into<String>) -> Outcome<T> { | |
| 69 | + | Outcome::fail(code, message) | |
| 70 | + | } | |
| 71 | + | ||
| 72 | + | fn null_or(value: Option<&str>) -> JsValue { | |
| 73 | + | value.map_or(JsValue::NULL, Into::into) | |
| 74 | + | } | |
| 75 | + | ||
| 76 | + | // --- Pure parts, tested below ---------------------------------------------- | |
| 77 | + | ||
| 78 | + | /// The host of an https address, lowercased: `github.com`. | |
| 79 | + | pub fn host_of(url: &str) -> String { | |
| 80 | + | url.trim() | |
| 81 | + | .trim_start_matches("https://") | |
| 82 | + | .trim_start_matches("http://") | |
| 83 | + | .split(['/', '?', '#']) | |
| 84 | + | .next() | |
| 85 | + | .unwrap_or_default() | |
| 86 | + | .rsplit('@') | |
| 87 | + | .next() | |
| 88 | + | .unwrap_or_default() | |
| 89 | + | .to_lowercase() | |
| 90 | + | } | |
| 91 | + | ||
| 92 | + | /// A remote as people name it: `github.com/acme/web`. | |
| 93 | + | pub fn display_name(url: &str) -> String { | |
| 94 | + | let rest = url | |
| 95 | + | .trim() | |
| 96 | + | .trim_start_matches("https://") | |
| 97 | + | .trim_start_matches("http://") | |
| 98 | + | .trim_end_matches('/'); | |
| 99 | + | let rest = rest.strip_suffix(".git").unwrap_or(rest); | |
| 100 | + | let rest = rest.rsplit_once('@').map_or(rest, |(_, after)| after); | |
| 101 | + | let (host, path) = rest.split_once('/').unwrap_or((rest, "")); | |
| 102 | + | if path.is_empty() { host.to_lowercase() } else { format!("{}/{path}", host.to_lowercase()) } | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | /// A clone address: https, with `.git` added when the host leaves it off, | |
| 106 | + | /// and no credentials in it. | |
| 107 | + | pub fn clean_clone_url(url: &str) -> Option<String> { | |
| 108 | + | let url = url.trim().trim_end_matches('/'); | |
| 109 | + | let rest = url.strip_prefix("https://")?; | |
| 110 | + | if rest.contains('@') || rest.contains(' ') || !rest.contains('/') || rest.starts_with('/') { | |
| 111 | + | return None; | |
| 112 | + | } | |
| 113 | + | Some(if url.ends_with(".git") { url.to_owned() } else { format!("{url}.git") }) | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | /// The web address of a clone address. | |
| 117 | + | pub fn web_url(clone_url: &str) -> String { | |
| 118 | + | clone_url.strip_suffix(".git").unwrap_or(clone_url).to_owned() | |
| 119 | + | } | |
| 120 | + | ||
| 121 | + | /// How a host's health moves with one more check. `now` in milliseconds. | |
| 122 | + | #[derive(Clone, Debug, Default, PartialEq, Eq)] | |
| 123 | + | pub struct HostHealth { | |
| 124 | + | pub failures: u32, | |
| 125 | + | pub successes: u32, | |
| 126 | + | /// When the current run of failures or successes began. | |
| 127 | + | pub streak_ms: u64, | |
| 128 | + | pub unreachable_since: Option<u64>, | |
| 129 | + | } | |
| 130 | + | ||
| 131 | + | #[derive(Clone, Copy, Debug, PartialEq, Eq)] | |
| 132 | + | pub enum Change { | |
| 133 | + | None, | |
| 134 | + | WentDown, | |
| 135 | + | CameBack, | |
| 136 | + | } | |
| 137 | + | ||
| 138 | + | impl HostHealth { | |
| 139 | + | pub fn reachable(&self) -> bool { | |
| 140 | + | self.unreachable_since.is_none() | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | pub fn checked(&self, answered: bool, now: u64) -> (HostHealth, Change) { | |
| 144 | + | let mut next = self.clone(); | |
| 145 | + | if answered { | |
| 146 | + | if next.successes == 0 { | |
| 147 | + | next.streak_ms = now; | |
| 148 | + | } | |
| 149 | + | next.successes += 1; | |
| 150 | + | next.failures = 0; | |
| 151 | + | if next.unreachable_since.is_some() | |
| 152 | + | && next.successes >= UP_CHECKS | |
| 153 | + | && now.saturating_sub(next.streak_ms) >= UP_AFTER_MS | |
| 154 | + | { | |
| 155 | + | next.unreachable_since = None; | |
| 156 | + | return (next, Change::CameBack); | |
| 157 | + | } | |
| 158 | + | } else { | |
| 159 | + | if next.failures == 0 { | |
| 160 | + | next.streak_ms = now; | |
| 161 | + | } | |
| 162 | + | next.failures += 1; | |
| 163 | + | next.successes = 0; | |
| 164 | + | if next.unreachable_since.is_none() | |
| 165 | + | && next.failures >= DOWN_CHECKS | |
| 166 | + | && now.saturating_sub(next.streak_ms) >= DOWN_AFTER_MS | |
| 167 | + | { | |
| 168 | + | next.unreachable_since = Some(now); | |
| 169 | + | return (next, Change::WentDown); | |
| 170 | + | } | |
| 171 | + | } | |
| 172 | + | (next, Change::None) | |
| 173 | + | } | |
| 174 | + | } | |
| 175 | + | ||
| 176 | + | /// The branch name in a ref: `main` for `refs/heads/main`. | |
| 177 | + | fn branch_of(git_ref: &str) -> Option<&str> { | |
| 178 | + | git_ref.strip_prefix("refs/heads/") | |
| 179 | + | } | |
| 180 | + | ||
| 181 | + | /// The moves that carry out a hand-back plan, and the refs that go as pull | |
| 182 | + | /// requests. `theirs_all` is every ref the remote has, so a hand-back | |
| 183 | + | /// branch made before is moved from where it is. | |
| 184 | + | pub fn hand_back_moves(plan: &HandbackPlan, theirs_all: &BTreeMap<String, String>) -> (Vec<RefMove>, Vec<String>) { | |
| 185 | + | let mut moves = Vec::new(); | |
| 186 | + | let mut pull_requests = Vec::new(); | |
| 187 | + | for item in &plan.refs { | |
| 188 | + | let push = |moves: &mut Vec<RefMove>| { | |
| 189 | + | moves.push(RefMove { | |
| 190 | + | direction: MirrorDirection::Push, | |
| 191 | + | git_ref: item.git_ref.clone(), | |
| 192 | + | to: None, | |
| 193 | + | old: item.theirs.clone(), | |
| 194 | + | new: item.ours.clone(), | |
| 195 | + | }) | |
| 196 | + | }; | |
| 197 | + | let fetch = |moves: &mut Vec<RefMove>| { | |
| 198 | + | moves.push(RefMove { | |
| 199 | + | direction: MirrorDirection::Pull, | |
| 200 | + | git_ref: item.git_ref.clone(), | |
| 201 | + | to: None, | |
| 202 | + | old: item.ours.clone(), | |
| 203 | + | new: item.theirs.clone(), | |
| 204 | + | }) | |
| 205 | + | }; | |
| 206 | + | let decision = match item.action { | |
| 207 | + | RefAction::Diverged => item.decision, | |
| 208 | + | RefAction::PullRequest => Some(RefDecision::PullRequest), | |
| 209 | + | _ => None, | |
| 210 | + | }; | |
| 211 | + | match (item.action, decision) { | |
| 212 | + | (RefAction::Same, _) => {} | |
| 213 | + | (RefAction::Push, _) | (RefAction::Diverged, Some(RefDecision::KeepOurs)) => push(&mut moves), | |
| 214 | + | (RefAction::Fetch, _) | (RefAction::Diverged, Some(RefDecision::KeepTheirs)) => fetch(&mut moves), | |
| 215 | + | (_, Some(RefDecision::PullRequest)) => { | |
| 216 | + | let Some(branch) = branch_of(&item.git_ref) else { | |
| 217 | + | // Only branches can be pulled; a tag goes as it is. | |
| 218 | + | push(&mut moves); | |
| 219 | + | continue; | |
| 220 | + | }; | |
| 221 | + | if let Some(ours) = &item.ours { | |
| 222 | + | let target = format!("{HANDBACK_PREFIX}{branch}"); | |
| 223 | + | moves.push(RefMove { | |
| 224 | + | direction: MirrorDirection::Push, | |
| 225 | + | git_ref: item.git_ref.clone(), | |
| 226 | + | to: Some(target.clone()), | |
| 227 | + | old: theirs_all.get(&target).cloned(), | |
| 228 | + | new: Some(ours.clone()), | |
| 229 | + | }); | |
| 230 | + | pull_requests.push(item.git_ref.clone()); | |
| 231 | + | } | |
| 232 | + | // g1t follows the remote's branch; its commits are on the | |
| 233 | + | // hand-back branch and kept under refs/g1t/replaced/. | |
| 234 | + | fetch(&mut moves); | |
| 235 | + | } | |
| 236 | + | (RefAction::Diverged, None) | (RefAction::PullRequest, _) => {} | |
| 237 | + | } | |
| 238 | + | } | |
| 239 | + | (moves, pull_requests) | |
| 240 | + | } | |
| 241 | + | ||
| 242 | + | /// Builds the plan for a hand-back from both sides' refs and what each ref | |
| 243 | + | /// was when the takeover began. `protected` names branches the remote | |
| 244 | + | /// protects. | |
| 245 | + | pub fn plan_refs( | |
| 246 | + | bases: &BTreeMap<String, (Option<String>, Option<RefDecision>)>, | |
| 247 | + | ours: &BTreeMap<String, String>, | |
| 248 | + | theirs: &BTreeMap<String, String>, | |
| 249 | + | protected: &BTreeSet<String>, | |
| 250 | + | ) -> Vec<RefPlan> { | |
| 251 | + | let names: BTreeSet<&String> = bases.keys().chain(ours.keys()).chain(theirs.keys()).collect(); | |
| 252 | + | names | |
| 253 | + | .into_iter() | |
| 254 | + | .filter(|name| !name.starts_with(HANDBACK_PREFIX)) | |
| 255 | + | .filter(|name| name.starts_with("refs/heads/") || name.starts_with("refs/tags/")) | |
| 256 | + | .filter_map(|name| { | |
| 257 | + | let (base, decision) = bases.get(name).cloned().unwrap_or((None, None)); | |
| 258 | + | let ours = ours.get(name).cloned(); | |
| 259 | + | let theirs = theirs.get(name).cloned(); | |
| 260 | + | // A ref g1t never had and the remote made since is copied in; a | |
| 261 | + | // ref neither had at the start nor has now is nothing. | |
| 262 | + | let action = ref_action(base.as_deref(), ours.as_deref(), theirs.as_deref(), protected.contains(name)); | |
| 263 | + | (action != RefAction::Same).then(|| RefPlan { | |
| 264 | + | git_ref: name.clone(), | |
| 265 | + | base, | |
| 266 | + | ours, | |
| 267 | + | theirs, | |
| 268 | + | action, | |
| 269 | + | decision: if action == RefAction::Diverged { decision } else { None }, | |
| 270 | + | }) | |
| 271 | + | }) | |
| 272 | + | .collect() | |
| 273 | + | } | |
| 274 | + | ||
| 275 | + | // --- Rows ------------------------------------------------------------------- | |
| 276 | + | ||
| 277 | + | #[derive(Clone, Debug, Deserialize)] | |
| 278 | + | pub(crate) struct RemoteRow { | |
| 279 | + | pub id: String, | |
| 280 | + | pub repo_id: String, | |
| 281 | + | pub workspace: String, | |
| 282 | + | pub repo: String, | |
| 283 | + | pub provider: String, | |
| 284 | + | pub role: String, | |
| 285 | + | pub name: String, | |
| 286 | + | pub url: String, | |
| 287 | + | pub clone_url: String, | |
| 288 | + | pub connection_id: Option<String>, | |
| 289 | + | pub username: Option<String>, | |
| 290 | + | pub credential: Option<String>, | |
| 291 | + | pub state: String, | |
| 292 | + | pub state_since: String, | |
| 293 | + | pub state_by: Option<String>, | |
| 294 | + | pub settings: String, | |
| 295 | + | pub synced_at: Option<String>, | |
| 296 | + | pub last_error: Option<String>, | |
| 297 | + | pub created_at: String, | |
| 298 | + | } | |
| 299 | + | ||
| 300 | + | impl RemoteRow { | |
| 301 | + | fn provider(&self) -> RemoteProvider { | |
| 302 | + | RemoteProvider::parse(&self.provider).unwrap_or(RemoteProvider::Git) | |
| 303 | + | } | |
| 304 | + | ||
| 305 | + | fn leads(&self) -> bool { | |
| 306 | + | self.role == RemoteRole::Leader.as_str() | |
| 307 | + | } | |
| 308 | + | ||
| 309 | + | fn state(&self) -> RemoteState { | |
| 310 | + | RemoteState::parse(&self.state) | |
| 311 | + | } | |
| 312 | + | ||
| 313 | + | fn settings(&self) -> MirrorSettings { | |
| 314 | + | serde_json::from_str(&self.settings).unwrap_or_default() | |
| 315 | + | } | |
| 316 | + | ||
| 317 | + | fn host(&self) -> String { | |
| 318 | + | host_of(&self.clone_url) | |
| 319 | + | } | |
| 320 | + | ||
| 321 | + | /// `owner/name` on GitHub. | |
| 322 | + | fn full_name(&self) -> &str { | |
| 323 | + | self.url.trim_start_matches("https://github.com/") | |
| 324 | + | } | |
| 325 | + | ||
| 326 | + | /// What the repos service keeps for a mirror of this remote. | |
| 327 | + | fn repo_mirror(&self) -> Option<RepoMirror> { | |
| 328 | + | if !self.leads() { | |
| 329 | + | return None; | |
| 330 | + | } | |
| 331 | + | let settings = self.settings(); | |
| 332 | + | Some(RepoMirror { | |
| 333 | + | state: self.state().mirror().unwrap_or_default(), | |
| 334 | + | remote: self.name.clone(), | |
| 335 | + | url: self.url.clone(), | |
| 336 | + | since: self.state_since.clone(), | |
| 337 | + | warm: settings.keep_ci_warm, | |
| 338 | + | github_workflows: settings.github_workflows, | |
| 339 | + | hold_deploys: settings.hold_deploys, | |
| 340 | + | }) | |
| 341 | + | } | |
| 342 | + | ||
| 343 | + | fn contract(&self, health: Option<&HostRow>) -> Remote { | |
| 344 | + | Remote { | |
| 345 | + | id: self.id.clone(), | |
| 346 | + | repo_id: self.repo_id.clone(), | |
| 347 | + | repo: self.repo.clone(), | |
| 348 | + | provider: self.provider(), | |
| 349 | + | role: if self.leads() { RemoteRole::Leader } else { RemoteRole::Follower }, | |
| 350 | + | name: self.name.clone(), | |
| 351 | + | url: self.url.clone(), | |
| 352 | + | state: self.state(), | |
| 353 | + | state_since: self.state_since.clone(), | |
| 354 | + | state_by: self.state_by.clone(), | |
| 355 | + | reachable: health.is_none_or(|health| health.unreachable_since.is_none()), | |
| 356 | + | unreachable_since: health.and_then(|health| health.unreachable_since.clone()), | |
| 357 | + | synced_at: self.synced_at.clone(), | |
| 358 | + | last_error: self.last_error.clone(), | |
| 359 | + | settings: self.settings(), | |
| 360 | + | created_at: self.created_at.clone(), | |
| 361 | + | } | |
| 362 | + | } | |
| 363 | + | ||
| 364 | + | fn link(&self) -> String { | |
| 365 | + | format!("/{}/settings/mirroring", self.repo) | |
| 366 | + | } | |
| 367 | + | } | |
| 368 | + | ||
| 369 | + | #[derive(Clone, Debug, Default, Deserialize)] | |
| 370 | + | pub(crate) struct HostRow { | |
| 371 | + | pub host: String, | |
| 372 | + | pub failures: u32, | |
| 373 | + | pub successes: u32, | |
| 374 | + | pub streak_ms: f64, | |
| 375 | + | pub unreachable_since: Option<String>, | |
| 376 | + | } | |
| 377 | + | ||
| 378 | + | impl HostRow { | |
| 379 | + | fn health(&self) -> HostHealth { | |
| 380 | + | HostHealth { | |
| 381 | + | failures: self.failures, | |
| 382 | + | successes: self.successes, | |
| 383 | + | streak_ms: self.streak_ms as u64, | |
| 384 | + | unreachable_since: self.unreachable_since.as_deref().and_then(crate::github::parse_time), | |
| 385 | + | } | |
| 386 | + | } | |
| 387 | + | } | |
| 388 | + | ||
| 389 | + | /// Who is acting: a person, or g1t on its own (an automatic takeover or | |
| 390 | + | /// hand-back), which needs no role. | |
| 391 | + | enum By<'a> { | |
| 392 | + | Person(&'a User), | |
| 393 | + | G1t, | |
| 394 | + | } | |
| 395 | + | ||
| 396 | + | impl By<'_> { | |
| 397 | + | fn name(&self) -> String { | |
| 398 | + | match self { | |
| 399 | + | By::Person(user) => user.username.clone(), | |
| 400 | + | By::G1t => "g1t".to_owned(), | |
| 401 | + | } | |
| 402 | + | } | |
| 403 | + | } | |
| 404 | + | ||
| 405 | + | pub struct Mirrors { | |
| 406 | + | db: D1Database, | |
| 407 | + | sealer: Option<Sealer>, | |
| 408 | + | repos: Fetcher, | |
| 409 | + | identity: Fetcher, | |
| 410 | + | events: Option<Fetcher>, | |
| 411 | + | github: GithubApp, | |
| 412 | + | } | |
| 413 | + | ||
| 414 | + | impl Mirrors { | |
| 415 | + | pub fn new(env: &Env) -> Result<Self> { | |
| 416 | + | Ok(Mirrors { | |
| 417 | + | db: env.d1("DB")?, | |
| 418 | + | sealer: env.secret("INTEGRATIONS_KEY").ok().and_then(|key| Sealer::new(&key.to_string())), | |
| 419 | + | repos: env.service("REPOS")?, | |
| 420 | + | identity: env.service("IDENTITY")?, | |
| 421 | + | events: env.service("EVENTS").ok(), | |
| 422 | + | github: GithubApp::new(env)?, | |
| 423 | + | }) | |
| 424 | + | } | |
| 425 | + | ||
| 426 | + | // --- Reading -------------------------------------------------------------- | |
| 427 | + | ||
| 428 | + | async fn rows(&self, repo_id: &str) -> Result<Vec<RemoteRow>> { | |
| 429 | + | self.db | |
| 430 | + | .prepare("SELECT * FROM remotes WHERE repo_id = ? ORDER BY role, created_at") | |
| 431 | + | .bind(&[repo_id.into()])? | |
| 432 | + | .all() | |
| 433 | + | .await? | |
| 434 | + | .results::<RemoteRow>() | |
| 435 | + | } | |
| 436 | + | ||
| 437 | + | async fn row(&self, id: &str) -> Result<Option<RemoteRow>> { | |
| 438 | + | self.db.prepare("SELECT * FROM remotes WHERE id = ?").bind(&[id.into()])?.first::<RemoteRow>(None).await | |
| 439 | + | } | |
| 440 | + | ||
| 441 | + | async fn leader(&self, repo_id: &str) -> Result<Option<RemoteRow>> { | |
| 442 | + | self.db | |
| 443 | + | .prepare("SELECT * FROM remotes WHERE repo_id = ? AND role = 'leader'") | |
| 444 | + | .bind(&[repo_id.into()])? | |
| 445 | + | .first::<RemoteRow>(None) | |
| 446 | + | .await | |
| 447 | + | } | |
| 448 | + | ||
| 449 | + | async fn hosts(&self) -> Result<HashMap<String, HostRow>> { | |
| 450 | + | Ok(self | |
| 451 | + | .db | |
| 452 | + | .prepare("SELECT * FROM remote_hosts") | |
| 453 | + | .all() | |
| 454 | + | .await? | |
| 455 | + | .results::<HostRow>()? | |
| 456 | + | .into_iter() | |
| 457 | + | .map(|row| (row.host.clone(), row)) | |
| 458 | + | .collect()) | |
| 459 | + | } | |
| 460 | + | ||
| 461 | + | async fn repo(&self, repo_id: &str, viewer: Option<&User>) -> Result<Option<Repo>> { | |
| 462 | + | let viewer = viewer.cloned().or_else(|| Some(User::system("g1t"))); | |
| 463 | + | let found: Outcome<Repo> = | |
| 464 | + | g1t_kit::call(&self.repos, "get_by_id", &GetByIdArgs { id: repo_id.to_owned(), viewer }).await?; | |
| 465 | + | Ok(found.into_result().ok()) | |
| 466 | + | } | |
| 467 | + | ||
| 468 | + | /// Why `actor` may not change `repo`'s links, if they may not. | |
| 469 | + | fn refused<T>(actor: &User, repo: &Repo, capability: Capability) -> Option<Outcome<T>> { | |
| 470 | + | let target = access::RepoRef { id: &repo.id, namespace: &repo.namespace, private: repo.is_private }; | |
| 471 | + | match access::check(Some(actor), target, capability) { | |
| 472 | + | Ok(()) => None, | |
| 473 | + | Err(access::Denied::NotFound) => Some(fail(FailureCode::NotFound, "Repository not found.")), | |
| 474 | + | Err(access::Denied::Forbidden) => { | |
| 475 | + | Some(fail(FailureCode::Forbidden, access::needs(capability, &format!("{}/{}", repo.namespace, repo.name)))) | |
| 476 | + | } | |
| 477 | + | } | |
| 478 | + | } | |
| 479 | + | ||
| 480 | + | /// The repository and its leader, for someone changing them. | |
| 481 | + | async fn leader_for(&self, actor: &User, repo_id: &str) -> Result<std::result::Result<(Repo, RemoteRow), Outcome<MirrorView>>> { | |
| 482 | + | let Some(repo) = self.repo(repo_id, Some(actor)).await? else { | |
| 483 | + | return Ok(Err(fail(FailureCode::NotFound, "Repository not found."))); | |
| 484 | + | }; | |
| 485 | + | if let Some(refused) = Self::refused(actor, &repo, Capability::ManageIntegrations) { | |
| 486 | + | return Ok(Err(refused)); | |
| 487 | + | } | |
| 488 | + | let Some(row) = self.leader(repo_id).await? else { | |
| 489 | + | return Ok(Err(fail(FailureCode::NotFound, format!("{}/{} is not a mirror.", repo.namespace, repo.name)))); | |
| 490 | + | }; | |
| 491 | + | Ok(Ok((repo, row))) | |
| 492 | + | } | |
| 493 | + | ||
| 494 | + | async fn view_of(&self, repo_id: &str, can_manage: bool, plan: Option<HandbackPlan>) -> Result<MirrorView> { | |
| 495 | + | let hosts = self.hosts().await?; | |
| 496 | + | Ok(MirrorView { | |
| 497 | + | remotes: self.rows(repo_id).await?.iter().map(|row| row.contract(hosts.get(&row.host()))).collect(), | |
| 498 | + | plan, | |
| 499 | + | can_manage, | |
| 500 | + | notes: Vec::new(), | |
| 501 | + | }) | |
| 502 | + | } | |
| 503 | + | ||
| 504 | + | pub async fn view(&self, a: MirrorViewArgs) -> Result<Outcome<MirrorView>> { | |
| 505 | + | let Some(repo) = self.repo(&a.repo_id, a.viewer.as_ref()).await? else { | |
| 506 | + | return Ok(fail(FailureCode::NotFound, "Repository not found.")); | |
| 507 | + | }; | |
| 508 | + | let can_manage = a | |
| 509 | + | .viewer | |
| 510 | + | .as_ref() | |
| 511 | + | .is_some_and(|viewer| Self::refused::<()>(viewer, &repo, Capability::ManageIntegrations).is_none()); | |
| 512 | + | Ok(Outcome::Ok(self.view_of(&repo.id, can_manage, None).await?)) | |
| 513 | + | } | |
| 514 | + | ||
| 515 | + | pub async fn briefs(&self, a: MirrorBriefsArgs) -> Result<Vec<RemoteBrief>> { | |
| 516 | + | let ids: Vec<&String> = a.repo_ids.iter().take(200).collect(); | |
| 517 | + | if ids.is_empty() { | |
| 518 | + | return Ok(Vec::new()); | |
| 519 | + | } | |
| 520 | + | let marks = vec!["?"; ids.len()].join(", "); | |
| 521 | + | let bind: Vec<JsValue> = ids.iter().map(|id| id.as_str().into()).collect(); | |
| 522 | + | let rows = self | |
| 523 | + | .db | |
| 524 | + | .prepare(format!("SELECT * FROM remotes WHERE repo_id IN ({marks}) ORDER BY role, created_at")) | |
| 525 | + | .bind(&bind)? | |
| 526 | + | .all() | |
| 527 | + | .await? | |
| 528 | + | .results::<RemoteRow>()?; | |
| 529 | + | let hosts = self.hosts().await?; | |
| 530 | + | Ok(rows | |
| 531 | + | .iter() | |
| 532 | + | .map(|row| RemoteBrief { | |
| 533 | + | repo_id: row.repo_id.clone(), | |
| 534 | + | role: if row.leads() { RemoteRole::Leader } else { RemoteRole::Follower }, | |
| 535 | + | name: row.name.clone(), | |
| 536 | + | state: row.state(), | |
| 537 | + | reachable: hosts.get(&row.host()).is_none_or(|host| host.unreachable_since.is_none()), | |
| 538 | + | }) | |
| 539 | + | .collect()) | |
| 540 | + | } | |
| 541 | + | ||
| 542 | + | // --- Talking to hosts ----------------------------------------------------- | |
| 543 | + | ||
| 544 | + | /// The user and token a remote is opened with. | |
| 545 | + | async fn credential(&self, row: &RemoteRow) -> Result<std::result::Result<(Option<String>, String), String>> { | |
| 546 | + | match row.provider() { | |
| 547 | + | RemoteProvider::Github => { | |
| 548 | + | let Some(installation) = row.connection_id.as_deref().and_then(|id| id.parse::<u64>().ok()) else { | |
| 549 | + | return Ok(Err("This link has no GitHub installation.".to_owned())); | |
| 550 | + | }; | |
| 551 | + | Ok(self.github.installation_token(installation).await?.map(|token| (None, token))) | |
| 552 | + | } | |
| 553 | + | RemoteProvider::G1t | RemoteProvider::Git => { | |
| 554 | + | let (Some(sealer), Some(sealed)) = (&self.sealer, row.credential.as_deref()) else { | |
| 555 | + | return Ok(Err("This link has no token. Add one in its settings.".to_owned())); | |
| 556 | + | }; | |
| 557 | + | match sealer.open(sealed, &format!("rmt:{}", row.id)) { | |
| 558 | + | Some(token) => Ok(Ok((row.username.clone(), token))), | |
| 559 | + | None => Ok(Err("This link's token could not be read. Add it again.".to_owned())), | |
| 560 | + | } | |
| 561 | + | } | |
| 562 | + | } | |
| 563 | + | } | |
| 564 | + | ||
| 565 | + | /// Which of `branches` the remote protects. Hosts that cannot say | |
| 566 | + | /// protect none: a refused push still goes as a pull request. | |
| 567 | + | async fn protected(&self, row: &RemoteRow, token: &str, branches: &[String]) -> Result<BTreeSet<String>> { | |
| 568 | + | let mut protected = BTreeSet::new(); | |
| 569 | + | if row.provider() != RemoteProvider::Github { | |
| 570 | + | return Ok(protected); | |
| 571 | + | } | |
| 572 | + | for git_ref in branches.iter().take(MAX_PROTECTION_CHECKS) { | |
| 573 | + | let Some(branch) = branch_of(git_ref) else { continue }; | |
| 574 | + | let Ok(answer) = self | |
| 575 | + | .github | |
| 576 | + | .api(Method::Get, &format!("/repos/{}/branches/{}", row.full_name(), branch), token, None) | |
| 577 | + | .await | |
| 578 | + | else { | |
| 579 | + | continue; | |
| 580 | + | }; | |
| 581 | + | if answer.ok() && answer.json()["protected"].as_bool() == Some(true) { | |
| 582 | + | protected.insert(git_ref.clone()); | |
| 583 | + | } | |
| 584 | + | } | |
| 585 | + | Ok(protected) | |
| 586 | + | } | |
| 587 | + | ||
| 588 | + | /// Opens a pull request on the remote from a hand-back branch. Returns | |
| 589 | + | /// a line for people: where it is, or how to open it by hand. | |
| 590 | + | async fn open_pull_request(&self, row: &RemoteRow, token: &str, git_ref: &str, repo: &str) -> Result<String> { | |
| 591 | + | let branch = branch_of(git_ref).unwrap_or(git_ref); | |
| 592 | + | let head = format!("g1t/handback/{branch}"); | |
| 593 | + | let by_hand = format!("{branch}: g1t's commits are on {head} on {}. Open a pull request from it into {branch}.", row.name); | |
| 594 | + | if row.provider() != RemoteProvider::Github { | |
| 595 | + | return Ok(by_hand); | |
| 596 | + | } | |
| 597 | + | let body = serde_json::json!({ | |
| 598 | + | "title": format!("Changes made on g1t while {} was unreachable", row.name), | |
| 599 | + | "head": head, | |
| 600 | + | "base": branch, | |
| 601 | + | "body": format!( | |
| 602 | + | "g1t led [{repo}](https://g1t.sh/{repo}) while this repository could not be reached. These are the commits made there on `{branch}`.\n\nThe branch is protected here, so they come as a pull request rather than a push." | |
| 603 | + | ), | |
| 604 | + | "maintainer_can_modify": true, | |
| 605 | + | }); | |
| 606 | + | let Ok(answer) = self.github.api(Method::Post, &format!("/repos/{}/pulls", row.full_name()), token, Some(body)).await | |
| 607 | + | else { | |
| 608 | + | return Ok(by_hand); | |
| 609 | + | }; | |
| 610 | + | let said = answer.json(); | |
| 611 | + | Ok(if answer.ok() { | |
| 612 | + | format!("{branch}: opened {}", said["html_url"].as_str().unwrap_or("a pull request")) | |
| 613 | + | } else if answer.status == 422 && said.to_string().contains("already exists") { | |
| 614 | + | format!("{branch}: the pull request from {head} was already open and now has the new commits.") | |
| 615 | + | } else { | |
| 616 | + | format!("{by_hand} ({})", answer.problem("GitHub")) | |
| 617 | + | }) | |
| 618 | + | } | |
| 619 | + | ||
| 620 | + | async fn repos_refs(&self, row: &RemoteRow, username: Option<String>, token: String) -> Result<Outcome<MirrorRefs>> { | |
| 621 | + | g1t_kit::call( | |
| 622 | + | &self.repos, | |
| 623 | + | "mirror_refs", | |
| 624 | + | &MirrorRefsArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username }, | |
| 625 | + | ) | |
| 626 | + | .await | |
| 627 | + | } | |
| 628 | + | ||
| 629 | + | /// g1t's own refs, asking nothing of the remote: a takeover starts while | |
| 630 | + | /// it is away, when not even a token can be had from it. | |
| 631 | + | async fn our_refs(&self, row: &RemoteRow) -> Result<Outcome<MirrorRefs>> { | |
| 632 | + | g1t_kit::call( | |
| 633 | + | &self.repos, | |
| 634 | + | "mirror_refs", | |
| 635 | + | &MirrorRefsArgs { repo_id: row.repo_id.clone(), url: String::new(), token: String::new(), username: None }, | |
| 636 | + | ) | |
| 637 | + | .await | |
| 638 | + | } | |
| 639 | + | ||
| 640 | + | async fn apply(&self, row: &RemoteRow, username: Option<String>, token: String, moves: Vec<RefMove>) -> Result<Outcome<MirrorApplied>> { | |
| 641 | + | g1t_kit::call( | |
| 642 | + | &self.repos, | |
| 643 | + | "mirror_apply", | |
| 644 | + | &MirrorApplyArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username, moves }, | |
| 645 | + | ) | |
| 646 | + | .await | |
| 647 | + | } | |
| 648 | + | ||
| 649 | + | /// Copies refs in the link's direction: a mirror catches up, a follower | |
| 650 | + | /// is pushed to. `Err` is a reason, already recorded on the link. | |
| 651 | + | async fn sync(&self, row: &RemoteRow) -> Result<std::result::Result<Mirrored, String>> { | |
| 652 | + | let direction = if row.leads() { MirrorDirection::Pull } else { MirrorDirection::Push }; | |
| 653 | + | let (username, token) = match self.credential(row).await? { | |
| 654 | + | Ok(credential) => credential, | |
| 655 | + | Err(reason) => { | |
| 656 | + | self.note(row, Some(&reason)).await?; | |
| 657 | + | return Ok(Err(reason)); | |
| 658 | + | } | |
| 659 | + | }; | |
| 660 | + | let done: Outcome<Mirrored> = g1t_kit::call( | |
| 661 | + | &self.repos, | |
| 662 | + | "mirror", | |
| 663 | + | &MirrorArgs { repo_id: row.repo_id.clone(), url: row.clone_url.clone(), token, username, direction }, | |
| 664 | + | ) | |
| 665 | + | .await?; | |
| 666 | + | match done { | |
| 667 | + | Outcome::Ok(mirrored) => { | |
| 668 | + | self.note(row, None).await?; | |
| 669 | + | self.host_checked(&row.host(), true, None).await?; | |
| 670 | + | Ok(Ok(mirrored)) | |
| 671 | + | } | |
| 672 | + | Outcome::Fail(failure) => { | |
| 673 | + | let unreachable = failure.code == FailureCode::Unavailable; | |
| 674 | + | if unreachable { | |
| 675 | + | self.host_checked(&row.host(), false, Some(&failure.message)).await?; | |
| 676 | + | } else { | |
| 677 | + | self.note(row, Some(&failure.message)).await?; | |
| 678 | + | } | |
| 679 | + | if !row.leads() && !unreachable { | |
| 680 | + | self.set_state(row, RemoteState::Stuck, None).await?; | |
| 681 | + | } | |
| 682 | + | Ok(Err(failure.message)) | |
| 683 | + | } | |
| 684 | + | } | |
| 685 | + | } | |
| 686 | + | ||
| 687 | + | async fn note(&self, row: &RemoteRow, problem: Option<&str>) -> Result<()> { | |
| 688 | + | let statement = match problem { | |
| 689 | + | Some(problem) => self | |
| 690 | + | .db | |
| 691 | + | .prepare("UPDATE remotes SET last_error = ?2 WHERE id = ?1") | |
| 692 | + | .bind(&[row.id.as_str().into(), problem.into()])?, | |
| 693 | + | None => self | |
| 694 | + | .db | |
| 695 | + | .prepare("UPDATE remotes SET last_error = NULL, synced_at = ?2 WHERE id = ?1") | |
| 696 | + | .bind(&[row.id.as_str().into(), rfc3339(now_ms()).into()])?, | |
| 697 | + | }; | |
| 698 | + | statement.run().await?; | |
| 699 | + | if problem.is_none() && row.state() == RemoteState::Stuck { | |
| 700 | + | self.set_state(row, RemoteState::Following, None).await?; | |
| 701 | + | } | |
| 702 | + | Ok(()) | |
| 703 | + | } | |
| 704 | + | ||
| 705 | + | // --- State ------------------------------------------------------------------ | |
| 706 | + | ||
| 707 | + | /// Records a link's state, and tells the repos service what a mirror | |
| 708 | + | /// is now. A link the repos service has not heard of yet is retried by | |
| 709 | + | /// the cron (`recorded`). | |
| 710 | + | async fn set_state(&self, row: &RemoteRow, state: RemoteState, by: Option<&str>) -> Result<RemoteRow> { | |
| 711 | + | let now = rfc3339(now_ms()); | |
| 712 | + | self.db | |
| 713 | + | .prepare("UPDATE remotes SET state = ?2, state_since = ?3, state_by = ?4, recorded = 0 WHERE id = ?1") | |
| 714 | + | .bind(&[row.id.as_str().into(), state.as_str().into(), now.as_str().into(), null_or(by)])? | |
| 715 | + | .run() | |
| 716 | + | .await?; | |
| 717 | + | let row = RemoteRow { | |
| 718 | + | state: state.as_str().to_owned(), | |
| 719 | + | state_since: now, | |
| 720 | + | state_by: by.map(str::to_owned), | |
| 721 | + | ..row.clone() | |
| 722 | + | }; | |
| 723 | + | self.record(&row).await?; | |
| 724 | + | Ok(row) | |
| 725 | + | } | |
| 726 | + | ||
| 727 | + | /// Tells the repos service a repository's mirror, as this link has it. | |
| 728 | + | async fn record(&self, row: &RemoteRow) -> Result<()> { | |
| 729 | + | if !row.leads() { | |
| 730 | + | return Ok(()); | |
| 731 | + | } | |
| 732 | + | let done: Outcome<Repo> = g1t_kit::call( | |
| 733 | + | &self.repos, | |
| 734 | + | "set_mirror", | |
| 735 | + | &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: row.repo_mirror() }, | |
| 736 | + | ) | |
| 737 | + | .await?; | |
| 738 | + | if done.into_result().is_ok() { | |
| 739 | + | self.db.prepare("UPDATE remotes SET recorded = 1 WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 740 | + | } | |
| 741 | + | Ok(()) | |
| 742 | + | } | |
| 743 | + | ||
| 744 | + | async fn publish(&self, kind: &'static str, row: &RemoteRow, by: Option<&str>, title: String, detail: Option<String>, notify: Vec<String>) { | |
| 745 | + | let Some(events) = &self.events else { return }; | |
| 746 | + | let event = NewEvent { | |
| 747 | + | kind, | |
| 748 | + | source: SOURCE, | |
| 749 | + | repo_id: Some(row.repo_id.clone()), | |
| 750 | + | actor: None, | |
| 751 | + | data: MirrorEvent { | |
| 752 | + | repo_id: row.repo_id.clone(), | |
| 753 | + | repo: row.repo.clone(), | |
| 754 | + | remote_id: row.id.clone(), | |
| 755 | + | remote: row.name.clone(), | |
| 756 | + | state: (kind != "mirror.moved_in").then(|| row.state.clone()), | |
| 757 | + | from: None, | |
| 758 | + | by: by.map(str::to_owned), | |
| 759 | + | title, | |
| 760 | + | detail, | |
| 761 | + | notify, | |
| 762 | + | link: row.link(), | |
| 763 | + | }, | |
| 764 | + | }; | |
| 765 | + | let sent: Result<Value> = g1t_kit::call(events, "publish", &Publish { events: vec![event] }).await; | |
| 766 | + | if let Err(error) = sent { | |
| 767 | + | worker::console_error!("mirrors: publishing {kind} for {} failed: {error}", row.repo); | |
| 768 | + | } | |
| 769 | + | } | |
| 770 | + | ||
| 771 | + | /// The workspace's owners, when the link asks for the inbox. | |
| 772 | + | async fn told(&self, row: &RemoteRow, except: Option<&str>) -> Vec<String> { | |
| 773 | + | if row.settings().notify != Notify::Inbox { | |
| 774 | + | return Vec::new(); | |
| 775 | + | } | |
| 776 | + | let members: Result<Outcome<Vec<Member>>> = g1t_kit::call( | |
| 777 | + | &self.identity, | |
| 778 | + | "list_members", | |
| 779 | + | &ListMembersArgs { slug: row.workspace.clone(), viewer: Some(User::system(&row.workspace)) }, | |
| 780 | + | ) | |
| 781 | + | .await; | |
| 782 | + | match members { | |
| 783 | + | Ok(Outcome::Ok(members)) => members | |
| 784 | + | .into_iter() | |
| 785 | + | .filter(|member| member.role == Role::Owner) | |
| 786 | + | .map(|member| member.username) | |
| 787 | + | .filter(|name| except.is_none_or(|except| !name.eq_ignore_ascii_case(except))) | |
| 788 | + | .collect(), | |
| 789 | + | _ => Vec::new(), | |
| 790 | + | } | |
| 791 | + | } | |
| 792 | + | ||
| 793 | + | // --- Takeover, CI failover, hand-back, moving in ------------------------------ | |
| 794 | + | ||
| 795 | + | pub async fn take_over(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { | |
| 796 | + | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { | |
| 797 | + | Ok(found) => found, | |
| 798 | + | Err(refused) => return Ok(refused), | |
| 799 | + | }; | |
| 800 | + | self.start_takeover(row, By::Person(&a.actor)).await | |
| 801 | + | } | |
| 802 | + | ||
| 803 | + | async fn start_takeover(&self, row: RemoteRow, by: By<'_>) -> Result<Outcome<MirrorView>> { | |
| 804 | + | match row.state() { | |
| 805 | + | RemoteState::Takeover => return Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)), | |
| 806 | + | RemoteState::HandingBack => { | |
| 807 | + | return Ok(fail(FailureCode::Conflict, "It is being handed back. Wait for that to finish, or for it to stop.")); | |
| 808 | + | } | |
| 809 | + | _ => {} | |
| 810 | + | } | |
| 811 | + | // The starting point: what g1t holds now, which is what it last | |
| 812 | + | // copied from the remote. | |
| 813 | + | let refs = match self.our_refs(&row).await? { | |
| 814 | + | Outcome::Ok(refs) => refs, | |
| 815 | + | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 816 | + | }; | |
| 817 | + | let mut statements = vec![self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?]; | |
| 818 | + | for (name, hash) in &refs.ours { | |
| 819 | + | statements.push( | |
| 820 | + | self.db | |
| 821 | + | .prepare("INSERT INTO remote_refs (remote_id, ref, base) VALUES (?, ?, ?)") | |
| 822 | + | .bind(&[row.id.as_str().into(), name.as_str().into(), hash.as_str().into()])?, | |
| 823 | + | ); | |
| 824 | + | } | |
| 825 | + | self.db.batch(statements).await?; | |
| 826 | + | let name = by.name(); | |
| 827 | + | let row = self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; | |
| 828 | + | let notify = self.told(&row, Some(&name)).await; | |
| 829 | + | let title = match by { | |
| 830 | + | By::G1t => format!("g1t took over {} while {} is unreachable", row.repo, row.name), | |
| 831 | + | By::Person(_) => format!("{name} took over {} from {}", row.repo, row.name), | |
| 832 | + | }; | |
| 833 | + | self.publish("mirror.state_changed", &row, Some(&name), title, None, notify).await; | |
| 834 | + | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)) | |
| 835 | + | } | |
| 836 | + | ||
| 837 | + | pub async fn ci(&self, a: MirrorCiArgs) -> Result<Outcome<MirrorView>> { | |
| 838 | + | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { | |
| 839 | + | Ok(found) => found, | |
| 840 | + | Err(refused) => return Ok(refused), | |
| 841 | + | }; | |
| 842 | + | let next = match (row.state(), a.on) { | |
| 843 | + | (RemoteState::Standby, true) => RemoteState::Ci, | |
| 844 | + | (RemoteState::Ci, false) => RemoteState::Standby, | |
| 845 | + | (RemoteState::Ci, true) | (RemoteState::Standby, false) => { | |
| 846 | + | return Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)); | |
| 847 | + | } | |
| 848 | + | _ => return Ok(fail(FailureCode::Conflict, "g1t leads this repository now: its workflows already run here.")), | |
| 849 | + | }; | |
| 850 | + | let row = self.set_state(&row, next, Some(&a.actor.username)).await?; | |
| 851 | + | let title = if a.on { | |
| 852 | + | format!("{} started running {}'s workflows on g1t", a.actor.username, row.name) | |
| 853 | + | } else { | |
| 854 | + | format!("{} ended CI failover for {}", a.actor.username, row.repo) | |
| 855 | + | }; | |
| 856 | + | self.publish("mirror.state_changed", &row, Some(&a.actor.username), title, None, Vec::new()).await; | |
| 857 | + | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, None).await?)) | |
| 858 | + | } | |
| 859 | + | ||
| 860 | + | /// What handing back would do now. | |
| 861 | + | async fn plan(&self, row: &RemoteRow) -> Result<Outcome<HandbackPlan>> { | |
| 862 | + | let (username, token) = match self.credential(row).await? { | |
| 863 | + | Ok(credential) => credential, | |
| 864 | + | Err(reason) => return Ok(fail(FailureCode::Conflict, reason)), | |
| 865 | + | }; | |
| 866 | + | let refs = match self.repos_refs(row, username, token.clone()).await? { | |
| 867 | + | Outcome::Ok(refs) => refs, | |
| 868 | + | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 869 | + | }; | |
| 870 | + | let bases = self.bases(row).await?; | |
| 871 | + | let Some(theirs) = refs.theirs else { | |
| 872 | + | // The remote is away: what g1t changed, as it would go. | |
| 873 | + | let refs = plan_refs(&bases, &refs.ours, &bases_as_theirs(&bases), &BTreeSet::new()); | |
| 874 | + | return Ok(Outcome::Ok(HandbackPlan::new(refs, false))); | |
| 875 | + | }; | |
| 876 | + | let candidates: Vec<String> = plan_refs(&bases, &refs.ours, &theirs, &BTreeSet::new()) | |
| 877 | + | .into_iter() | |
| 878 | + | .filter(|plan| plan.action == RefAction::Push && plan.ours.is_some() && plan.theirs.is_some()) | |
| 879 | + | .map(|plan| plan.git_ref) | |
| 880 | + | .collect(); | |
| 881 | + | let protected = self.protected(row, &token, &candidates).await?; | |
| 882 | + | Ok(Outcome::Ok(HandbackPlan::new(plan_refs(&bases, &refs.ours, &theirs, &protected), true))) | |
| 883 | + | } | |
| 884 | + | ||
| 885 | + | async fn bases(&self, row: &RemoteRow) -> Result<BTreeMap<String, (Option<String>, Option<RefDecision>)>> { | |
| 886 | + | #[derive(Deserialize)] | |
| 887 | + | struct Base { | |
| 888 | + | #[serde(rename = "ref")] | |
| 889 | + | git_ref: String, | |
| 890 | + | base: Option<String>, | |
| 891 | + | decision: Option<String>, | |
| 892 | + | } | |
| 893 | + | Ok(self | |
| 894 | + | .db | |
| 895 | + | .prepare("SELECT ref, base, decision FROM remote_refs WHERE remote_id = ?") | |
| 896 | + | .bind(&[row.id.as_str().into()])? | |
| 897 | + | .all() | |
| 898 | + | .await? | |
| 899 | + | .results::<Base>()? | |
| 900 | + | .into_iter() | |
| 901 | + | .map(|base| { | |
| 902 | + | let decision = base.decision.and_then(|text| serde_json::from_value(Value::String(text)).ok()); | |
| 903 | + | (base.git_ref, (base.base, decision)) | |
| 904 | + | }) | |
| 905 | + | .collect()) | |
| 906 | + | } | |
| 907 | + | ||
| 908 | + | pub async fn hand_back_plan(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { | |
| 909 | + | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { | |
| 910 | + | Ok(found) => found, | |
| 911 | + | Err(refused) => return Ok(refused), | |
| 912 | + | }; | |
| 913 | + | if !matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { | |
| 914 | + | return Ok(fail(FailureCode::Conflict, "g1t has not taken over, so there is nothing to hand back.")); | |
| 915 | + | } | |
| 916 | + | let plan = match self.plan(&row).await? { | |
| 917 | + | Outcome::Ok(plan) => plan, | |
| 918 | + | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 919 | + | }; | |
| 920 | + | Ok(Outcome::Ok(self.view_of(&row.repo_id, true, Some(plan)).await?)) | |
| 921 | + | } | |
| 922 | + | ||
| 923 | + | pub async fn hand_back(&self, a: MirrorHandBackArgs) -> Result<Outcome<MirrorView>> { | |
| 924 | + | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { | |
| 925 | + | Ok(found) => found, | |
| 926 | + | Err(refused) => return Ok(refused), | |
| 927 | + | }; | |
| 928 | + | let mut statements = Vec::new(); | |
| 929 | + | for (git_ref, decision) in &a.decisions { | |
| 930 | + | let decision = serde_json::to_value(decision).ok().and_then(|value| value.as_str().map(str::to_owned)); | |
| 931 | + | statements.push( | |
| 932 | + | self.db | |
| 933 | + | .prepare( | |
| 934 | + | "INSERT INTO remote_refs (remote_id, ref, decision) VALUES (?1, ?2, ?3) | |
| 935 | + | ON CONFLICT (remote_id, ref) DO UPDATE SET decision = excluded.decision", | |
| 936 | + | ) | |
| 937 | + | .bind(&[row.id.as_str().into(), git_ref.as_str().into(), null_or(decision.as_deref())])?, | |
| 938 | + | ); | |
| 939 | + | } | |
| 940 | + | if !statements.is_empty() { | |
| 941 | + | self.db.batch(statements).await?; | |
| 942 | + | } | |
| 943 | + | self.finish_takeover(row, By::Person(&a.actor)).await | |
| 944 | + | } | |
| 945 | + | ||
| 946 | + | /// Hands a takeover back: freezes the repository, moves each ref as the | |
| 947 | + | /// plan says, and returns to standing by. If anything is refused, g1t | |
| 948 | + | /// keeps the lead and says why, so nobody is stuck. | |
| 949 | + | async fn finish_takeover(&self, row: RemoteRow, by: By<'_>) -> Result<Outcome<MirrorView>> { | |
| 950 | + | if !matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { | |
| 951 | + | return Ok(fail(FailureCode::Conflict, "g1t has not taken over, so there is nothing to hand back.")); | |
| 952 | + | } | |
| 953 | + | let plan = match self.plan(&row).await? { | |
| 954 | + | Outcome::Ok(plan) => plan, | |
| 955 | + | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 956 | + | }; | |
| 957 | + | if !plan.reachable { | |
| 958 | + | return Ok(fail(FailureCode::Unavailable, format!("{} is not answering yet. Hand back once it is.", row.name))); | |
| 959 | + | } | |
| 960 | + | if !plan.ready { | |
| 961 | + | return Ok(Outcome::Fail(undecided(&plan))); | |
| 962 | + | } | |
| 963 | + | let name = by.name(); | |
| 964 | + | // Nothing moves on g1t while it goes back. | |
| 965 | + | let row = self.set_state(&row, RemoteState::HandingBack, Some(&name)).await?; | |
| 966 | + | let plan = match self.plan(&row).await? { | |
| 967 | + | Outcome::Ok(plan) if plan.ready => plan, | |
| 968 | + | Outcome::Ok(plan) => { | |
| 969 | + | let row = self.set_state(&row, RemoteState::Takeover, row.state_by.as_deref()).await?; | |
| 970 | + | let _ = row; | |
| 971 | + | return Ok(Outcome::Fail(undecided(&plan))); | |
| 972 | + | } | |
| 973 | + | Outcome::Fail(failure) => { | |
| 974 | + | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; | |
| 975 | + | return Ok(Outcome::Fail(failure)); | |
| 976 | + | } | |
| 977 | + | }; | |
| 978 | + | let (username, token) = match self.credential(&row).await? { | |
| 979 | + | Ok(credential) => credential, | |
| 980 | + | Err(reason) => { | |
| 981 | + | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; | |
| 982 | + | return Ok(fail(FailureCode::Conflict, reason)); | |
| 983 | + | } | |
| 984 | + | }; | |
| 985 | + | let theirs_all = match self.repos_refs(&row, username.clone(), token.clone()).await? { | |
| 986 | + | Outcome::Ok(refs) => refs.theirs.unwrap_or_default(), | |
| 987 | + | Outcome::Fail(_) => BTreeMap::new(), | |
| 988 | + | }; | |
| 989 | + | let (moves, mut pull_requests) = hand_back_moves(&plan, &theirs_all); | |
| 990 | + | let mut problems = Vec::new(); | |
| 991 | + | if !moves.is_empty() { | |
| 992 | + | let applied = match self.apply(&row, username.clone(), token.clone(), moves).await? { | |
| 993 | + | Outcome::Ok(applied) => applied, | |
| 994 | + | Outcome::Fail(failure) => { | |
| 995 | + | self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; | |
| 996 | + | self.note(&row, Some(&failure.message)).await?; | |
| 997 | + | return Ok(Outcome::Fail(failure)); | |
| 998 | + | } | |
| 999 | + | }; | |
| 1000 | + | // A push the remote refused (a protected branch it did not say | |
| 1001 | + | // was protected) goes as a pull request instead. | |
| 1002 | + | let refused: Vec<RefPlan> = applied | |
| 1003 | + | .moved | |
| 1004 | + | .iter() | |
| 1005 | + | .filter(|moved| moved.direction == MirrorDirection::Push && moved.problem.is_some()) | |
| 1006 | + | .filter_map(|moved| { | |
| 1007 | + | plan.refs | |
| 1008 | + | .iter() | |
| 1009 | + | .find(|item| item.git_ref == moved.git_ref && branch_of(&item.git_ref).is_some() && item.ours.is_some()) | |
| 1010 | + | .cloned() | |
| 1011 | + | }) | |
| 1012 | + | .collect(); | |
| 1013 | + | for moved in applied.moved.iter().filter(|moved| moved.problem.is_some()) { | |
| 1014 | + | if !refused.iter().any(|item| item.git_ref == moved.git_ref) { | |
| 1015 | + | problems.push(format!("{}: {}", moved.git_ref, moved.problem.as_deref().unwrap_or_default())); | |
| 1016 | + | } | |
| 1017 | + | } | |
| 1018 | + | if !refused.is_empty() { | |
| 1019 | + | let retry = HandbackPlan::new( | |
| 1020 | + | refused | |
| 1021 | + | .into_iter() | |
| 1022 | + | .map(|item| RefPlan { action: RefAction::PullRequest, ..item }) | |
| 1023 | + | .collect(), | |
| 1024 | + | true, | |
| 1025 | + | ); | |
| 1026 | + | let (moves, more) = hand_back_moves(&retry, &theirs_all); | |
| 1027 | + | match self.apply(&row, username.clone(), token.clone(), moves).await? { | |
| 1028 | + | Outcome::Ok(applied) => { | |
| 1029 | + | for moved in applied.moved.iter().filter(|moved| moved.problem.is_some()) { | |
| 1030 | + | problems.push(format!("{}: {}", moved.git_ref, moved.problem.as_deref().unwrap_or_default())); | |
| 1031 | + | } | |
| 1032 | + | pull_requests.extend(more); | |
| 1033 | + | } | |
| 1034 | + | Outcome::Fail(failure) => problems.push(failure.message), | |
| 1035 | + | } | |
| 1036 | + | } | |
| 1037 | + | } | |
| 1038 | + | if !problems.is_empty() { | |
| 1039 | + | let reason = format!("Some branches did not go back: {}", problems.join("; ")); | |
| 1040 | + | let row = self.set_state(&row, RemoteState::Takeover, Some(&name)).await?; | |
| 1041 | + | self.note(&row, Some(&reason)).await?; | |
| 1042 | + | return Ok(fail(FailureCode::Conflict, format!("{reason}. g1t still leads; try again or decide differently."))); | |
| 1043 | + | } | |
| 1044 | + | let mut notes = Vec::new(); | |
| 1045 | + | for git_ref in &pull_requests { | |
| 1046 | + | notes.push(self.open_pull_request(&row, &token, git_ref, &row.repo).await?); | |
| 1047 | + | } | |
| 1048 | + | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1049 | + | let row = self.set_state(&row, RemoteState::Standby, Some(&name)).await?; | |
| 1050 | + | // Anything else the remote has, g1t now follows. | |
| 1051 | + | if let Err(reason) = self.sync(&row).await? { | |
| 1052 | + | notes.push(format!("Catching up after handing back: {reason}")); | |
| 1053 | + | } | |
| 1054 | + | let notify = self.told(&row, Some(&name)).await; | |
| 1055 | + | let detail = (!notes.is_empty()).then(|| notes.join("\n")); | |
| 1056 | + | self.publish( | |
| 1057 | + | "mirror.state_changed", | |
| 1058 | + | &row, | |
| 1059 | + | Some(&name), | |
| 1060 | + | format!("{} was handed back to {}", row.repo, row.name), | |
| 1061 | + | detail, | |
| 1062 | + | notify, | |
| 1063 | + | ) | |
| 1064 | + | .await; | |
| 1065 | + | let mut view = self.view_of(&row.repo_id, true, None).await?; | |
| 1066 | + | view.notes = notes; | |
| 1067 | + | Ok(Outcome::Ok(view)) | |
| 1068 | + | } | |
| 1069 | + | ||
| 1070 | + | pub async fn move_in(&self, a: MirrorMoveInArgs) -> Result<Outcome<MirrorView>> { | |
| 1071 | + | let (_, row) = match self.leader_for(&a.actor, &a.repo_id).await? { | |
| 1072 | + | Ok(found) => found, | |
| 1073 | + | Err(refused) => return Ok(refused), | |
| 1074 | + | }; | |
| 1075 | + | if row.state() == RemoteState::HandingBack { | |
| 1076 | + | return Ok(fail(FailureCode::Conflict, "It is being handed back. Move it to g1t once that is done.")); | |
| 1077 | + | } | |
| 1078 | + | let done: Outcome<Repo> = | |
| 1079 | + | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; | |
| 1080 | + | if let Outcome::Fail(failure) = done { | |
| 1081 | + | return Ok(Outcome::Fail(failure)); | |
| 1082 | + | } | |
| 1083 | + | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1084 | + | let mut notes = Vec::new(); | |
| 1085 | + | if a.keep_remote_updated { | |
| 1086 | + | let now = rfc3339(now_ms()); | |
| 1087 | + | self.db | |
| 1088 | + | .prepare( | |
| 1089 | + | "UPDATE remotes SET role = 'follower', state = 'following', state_since = ?2, state_by = ?3, recorded = 1 | |
| 1090 | + | WHERE id = ?1", | |
| 1091 | + | ) | |
| 1092 | + | .bind(&[row.id.as_str().into(), now.as_str().into(), a.actor.username.as_str().into()])? | |
| 1093 | + | .run() | |
| 1094 | + | .await?; | |
| 1095 | + | if let Some(follower) = self.row(&row.id).await? | |
| 1096 | + | && let Err(reason) = self.sync(&follower).await? | |
| 1097 | + | { | |
| 1098 | + | notes.push(format!("{} could not be brought up to date yet: {reason}", row.name)); | |
| 1099 | + | } | |
| 1100 | + | } else { | |
| 1101 | + | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1102 | + | } | |
| 1103 | + | if row.provider() == RemoteProvider::Github { | |
| 1104 | + | let mode = if a.keep_remote_updated { "push" } else { "import" }; | |
| 1105 | + | self.db | |
| 1106 | + | .prepare("UPDATE github_repos SET mode = ? WHERE repo_id = ?") | |
| 1107 | + | .bind(&[mode.into(), row.repo_id.as_str().into()])? | |
| 1108 | + | .run() | |
| 1109 | + | .await?; | |
| 1110 | + | } | |
| 1111 | + | let title = if a.keep_remote_updated { | |
| 1112 | + | format!("{} moved {} to g1t; {} now follows it", a.actor.username, row.repo, row.name) | |
| 1113 | + | } else { | |
| 1114 | + | format!("{} moved {} to g1t and stopped tracking {}", a.actor.username, row.repo, row.name) | |
| 1115 | + | }; | |
| 1116 | + | self.publish("mirror.moved_in", &row, Some(&a.actor.username), title, None, Vec::new()).await; | |
| 1117 | + | let mut view = self.view_of(&row.repo_id, true, None).await?; | |
| 1118 | + | view.notes = notes; | |
| 1119 | + | Ok(Outcome::Ok(view)) | |
| 1120 | + | } | |
| 1121 | + | ||
| 1122 | + | pub async fn sync_now(&self, a: MirrorActArgs) -> Result<Outcome<MirrorView>> { | |
| 1123 | + | let Some(repo) = self.repo(&a.repo_id, Some(&a.actor)).await? else { | |
| 1124 | + | return Ok(fail(FailureCode::NotFound, "Repository not found.")); | |
| 1125 | + | }; | |
| 1126 | + | // Syncing brings commits in, as pushing does. | |
| 1127 | + | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::Push) { | |
| 1128 | + | return Ok(refused); | |
| 1129 | + | } | |
| 1130 | + | let rows = self.rows(&repo.id).await?; | |
| 1131 | + | if rows.is_empty() { | |
| 1132 | + | return Ok(fail(FailureCode::NotFound, "This repository is not linked to another host.")); | |
| 1133 | + | } | |
| 1134 | + | let mut problems = Vec::new(); | |
| 1135 | + | for row in &rows { | |
| 1136 | + | if row.leads() && !row.state().mirror().is_some_and(MirrorState::follows) { | |
| 1137 | + | continue; | |
| 1138 | + | } | |
| 1139 | + | if let Err(reason) = self.sync(row).await? { | |
| 1140 | + | problems.push(format!("{}: {reason}", row.name)); | |
| 1141 | + | } | |
| 1142 | + | } | |
| 1143 | + | if !problems.is_empty() { | |
| 1144 | + | return Ok(fail(FailureCode::Conflict, problems.join("; "))); | |
| 1145 | + | } | |
| 1146 | + | Ok(Outcome::Ok(self.view_of(&repo.id, true, None).await?)) | |
| 1147 | + | } | |
| 1148 | + | ||
| 1149 | + | pub async fn settings(&self, a: MirrorSettingsArgs) -> Result<Outcome<Remote>> { | |
| 1150 | + | let Some(row) = self.row(&a.remote_id).await? else { | |
| 1151 | + | return Ok(fail(FailureCode::NotFound, "That link was not found.")); | |
| 1152 | + | }; | |
| 1153 | + | let Some(repo) = self.repo(&row.repo_id, Some(&a.actor)).await? else { | |
| 1154 | + | return Ok(fail(FailureCode::NotFound, "Repository not found.")); | |
| 1155 | + | }; | |
| 1156 | + | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { | |
| 1157 | + | return Ok(refused); | |
| 1158 | + | } | |
| 1159 | + | let (least, most) = TAKE_OVER_AFTER_MINUTES; | |
| 1160 | + | if a.settings.take_over_after.is_some_and(|minutes| minutes < least || minutes > most) { | |
| 1161 | + | return Ok(fail(FailureCode::Invalid, format!("Take over after between {least} minutes and {} hours.", most / 60))); | |
| 1162 | + | } | |
| 1163 | + | let settings = serde_json::to_string(&a.settings).unwrap_or_else(|_| "{}".to_owned()); | |
| 1164 | + | self.db | |
| 1165 | + | .prepare("UPDATE remotes SET settings = ?2, recorded = 0 WHERE id = ?1") | |
| 1166 | + | .bind(&[row.id.as_str().into(), settings.as_str().into()])? | |
| 1167 | + | .run() | |
| 1168 | + | .await?; | |
| 1169 | + | let row = RemoteRow { settings, ..row }; | |
| 1170 | + | self.record(&row).await?; | |
| 1171 | + | let hosts = self.hosts().await?; | |
| 1172 | + | Ok(Outcome::Ok(row.contract(hosts.get(&row.host())))) | |
| 1173 | + | } | |
| 1174 | + | ||
| 1175 | + | pub async fn add(&self, a: MirrorAddArgs) -> Result<Outcome<Remote>> { | |
| 1176 | + | if a.provider == RemoteProvider::Github { | |
| 1177 | + | return Ok(fail(FailureCode::Invalid, "Link GitHub repositories through the GitHub App, from New → Import from GitHub.")); | |
| 1178 | + | } | |
| 1179 | + | let Some(repo) = self.repo(&a.repo_id, Some(&a.actor)).await? else { | |
| 1180 | + | return Ok(fail(FailureCode::NotFound, "Repository not found.")); | |
| 1181 | + | }; | |
| 1182 | + | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { | |
| 1183 | + | return Ok(refused); | |
| 1184 | + | } | |
| 1185 | + | let Some(clone_url) = clean_clone_url(&a.url) else { | |
| 1186 | + | return Ok(fail(FailureCode::Invalid, "Give the remote's https address, such as https://g1t.sh/acme/web.git.")); | |
| 1187 | + | }; | |
| 1188 | + | let Some(token) = a.token.as_deref().map(str::trim).filter(|token| !token.is_empty()) else { | |
| 1189 | + | return Ok(fail(FailureCode::Invalid, "Give a token that can read and push to the remote.")); | |
| 1190 | + | }; | |
| 1191 | + | let Some(sealer) = &self.sealer else { | |
| 1192 | + | return Ok(fail(FailureCode::Conflict, "Tokens cannot be kept on this g1t: INTEGRATIONS_KEY is not set.")); | |
| 1193 | + | }; | |
| 1194 | + | if a.role == RemoteRole::Leader && self.leader(&repo.id).await?.is_some() { | |
| 1195 | + | return Ok(fail(FailureCode::Conflict, "This repository already mirrors a remote. A repository follows one leader.")); | |
| 1196 | + | } | |
| 1197 | + | let now = now_ms(); | |
| 1198 | + | let id = new_id("rmt", now); | |
| 1199 | + | let row = RemoteRow { | |
| 1200 | + | id: id.clone(), | |
| 1201 | + | repo_id: repo.id.clone(), | |
| 1202 | + | workspace: repo.namespace.clone(), | |
| 1203 | + | repo: format!("{}/{}", repo.namespace, repo.name), | |
| 1204 | + | provider: a.provider.as_str().to_owned(), | |
| 1205 | + | role: a.role.as_str().to_owned(), | |
| 1206 | + | name: display_name(&clone_url), | |
| 1207 | + | url: web_url(&clone_url), | |
| 1208 | + | clone_url: clone_url.clone(), | |
| 1209 | + | connection_id: None, | |
| 1210 | + | username: a.username.map(|name| name.trim().to_owned()).filter(|name| !name.is_empty()), | |
| 1211 | + | credential: Some(sealer.seal(token, &format!("rmt:{id}"))), | |
| 1212 | + | state: if a.role == RemoteRole::Leader { "standby" } else { "following" }.to_owned(), | |
| 1213 | + | state_since: rfc3339(now), | |
| 1214 | + | state_by: Some(a.actor.username.clone()), | |
| 1215 | + | settings: "{}".to_owned(), | |
| 1216 | + | synced_at: None, | |
| 1217 | + | last_error: None, | |
| 1218 | + | created_at: rfc3339(now), | |
| 1219 | + | }; | |
| 1220 | + | if row.name.eq_ignore_ascii_case(&format!("g1t.sh/{}", row.repo)) { | |
| 1221 | + | return Ok(fail(FailureCode::Invalid, "That is this repository.")); | |
| 1222 | + | } | |
| 1223 | + | // A leader fills an empty repository; one with history of its own | |
| 1224 | + | // would lose it. | |
| 1225 | + | if a.role == RemoteRole::Leader { | |
| 1226 | + | let refs = match self.repos_refs(&row, row.username.clone(), token.to_owned()).await? { | |
| 1227 | + | Outcome::Ok(refs) => refs, | |
| 1228 | + | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 1229 | + | }; | |
| 1230 | + | if !refs.ours.is_empty() { | |
| 1231 | + | return Ok(fail( | |
| 1232 | + | FailureCode::Conflict, | |
| 1233 | + | "Only an empty repository can become a mirror: it is filled from the remote. Make a new repository for it.", | |
| 1234 | + | )); | |
| 1235 | + | } | |
| 1236 | + | if let Some(reason) = refs.unreachable { | |
| 1237 | + | return Ok(fail(FailureCode::Unavailable, reason)); | |
| 1238 | + | } | |
| 1239 | + | } | |
| 1240 | + | self.db | |
| 1241 | + | .prepare( | |
| 1242 | + | "INSERT INTO remotes | |
| 1243 | + | (id, repo_id, workspace, repo, provider, role, name, url, clone_url, username, credential, | |
| 1244 | + | state, state_since, state_by, settings, created_by, created_at) | |
| 1245 | + | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, '{}', ?15, ?13)", | |
| 1246 | + | ) | |
| 1247 | + | .bind(&[ | |
| 1248 | + | row.id.as_str().into(), | |
| 1249 | + | row.repo_id.as_str().into(), | |
| 1250 | + | row.workspace.as_str().into(), | |
| 1251 | + | row.repo.as_str().into(), | |
| 1252 | + | row.provider.as_str().into(), | |
| 1253 | + | row.role.as_str().into(), | |
| 1254 | + | row.name.as_str().into(), | |
| 1255 | + | row.url.as_str().into(), | |
| 1256 | + | row.clone_url.as_str().into(), | |
| 1257 | + | null_or(row.username.as_deref()), | |
| 1258 | + | null_or(row.credential.as_deref()), | |
| 1259 | + | row.state.as_str().into(), | |
| 1260 | + | row.state_since.as_str().into(), | |
| 1261 | + | a.actor.username.as_str().into(), | |
| 1262 | + | a.actor.id.as_str().into(), | |
| 1263 | + | ])? | |
| 1264 | + | .run() | |
| 1265 | + | .await?; | |
| 1266 | + | self.record(&row).await?; | |
| 1267 | + | let synced = self.sync(&row).await?; | |
| 1268 | + | let row = self.row(&id).await?.unwrap_or(row); | |
| 1269 | + | if let Err(reason) = synced { | |
| 1270 | + | let hosts = self.hosts().await?; | |
| 1271 | + | let mut remote = row.contract(hosts.get(&row.host())); | |
| 1272 | + | remote.last_error = Some(reason); | |
| 1273 | + | return Ok(Outcome::Ok(remote)); | |
| 1274 | + | } | |
| 1275 | + | let hosts = self.hosts().await?; | |
| 1276 | + | Ok(Outcome::Ok(row.contract(hosts.get(&row.host())))) | |
| 1277 | + | } | |
| 1278 | + | ||
| 1279 | + | pub async fn remove(&self, a: MirrorRemoveArgs) -> Result<Outcome<bool>> { | |
| 1280 | + | let Some(row) = self.row(&a.remote_id).await? else { | |
| 1281 | + | return Ok(Outcome::Ok(false)); | |
| 1282 | + | }; | |
| 1283 | + | let Some(repo) = self.repo(&row.repo_id, Some(&a.actor)).await? else { | |
| 1284 | + | return Ok(fail(FailureCode::NotFound, "Repository not found.")); | |
| 1285 | + | }; | |
| 1286 | + | if let Some(refused) = Self::refused(&a.actor, &repo, Capability::ManageIntegrations) { | |
| 1287 | + | return Ok(refused); | |
| 1288 | + | } | |
| 1289 | + | if row.leads() && matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { | |
| 1290 | + | return Ok(fail( | |
| 1291 | + | FailureCode::Conflict, | |
| 1292 | + | "g1t leads this repository for now. Hand it back, or move it to g1t, before unlinking.", | |
| 1293 | + | )); | |
| 1294 | + | } | |
| 1295 | + | if row.leads() { | |
| 1296 | + | let done: Outcome<Repo> = | |
| 1297 | + | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; | |
| 1298 | + | if let Outcome::Fail(failure) = done { | |
| 1299 | + | return Ok(Outcome::Fail(failure)); | |
| 1300 | + | } | |
| 1301 | + | } | |
| 1302 | + | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1303 | + | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1304 | + | if row.provider() == RemoteProvider::Github { | |
| 1305 | + | self.db | |
| 1306 | + | .prepare("UPDATE github_repos SET mode = 'import' WHERE repo_id = ?") | |
| 1307 | + | .bind(&[row.repo_id.as_str().into()])? | |
| 1308 | + | .run() | |
| 1309 | + | .await?; | |
| 1310 | + | } | |
| 1311 | + | Ok(Outcome::Ok(true)) | |
| 1312 | + | } | |
| 1313 | + | ||
| 1314 | + | /// A remote g1t can no longer reach for good (uninstalled, deleted). A | |
| 1315 | + | /// mirror standing by becomes an ordinary repository with what it has, | |
| 1316 | + | /// since it can no longer follow; a follower is let go; a takeover | |
| 1317 | + | /// keeps going and is told why, so nothing is lost. | |
| 1318 | + | pub async fn link_gone(&self, id: &str, why: &str) -> Result<()> { | |
| 1319 | + | let Some(row) = self.row(id).await? else { return Ok(()) }; | |
| 1320 | + | if row.leads() && matches!(row.state(), RemoteState::Takeover | RemoteState::HandingBack) { | |
| 1321 | + | let reason = format!("{} {why}. g1t keeps leading; move it to g1t to keep it here.", row.name); | |
| 1322 | + | return self.note(&row, Some(&reason)).await; | |
| 1323 | + | } | |
| 1324 | + | if row.leads() { | |
| 1325 | + | let _: Outcome<Repo> = | |
| 1326 | + | g1t_kit::call(&self.repos, "set_mirror", &SetMirrorArgs { repo_id: row.repo_id.clone(), mirror: None }).await?; | |
| 1327 | + | } | |
| 1328 | + | self.db.prepare("DELETE FROM remote_refs WHERE remote_id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1329 | + | self.db.prepare("DELETE FROM remotes WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1330 | + | let title = if row.leads() { | |
| 1331 | + | format!("{} stopped mirroring {}: it {why}. g1t keeps what it has.", row.repo, row.name) | |
| 1332 | + | } else { | |
| 1333 | + | format!("{} stopped pushing to {}: it {why}.", row.repo, row.name) | |
| 1334 | + | }; | |
| 1335 | + | let notify = self.told(&row, None).await; | |
| 1336 | + | self.publish("mirror.moved_in", &row, Some("g1t"), title, None, notify).await; | |
| 1337 | + | Ok(()) | |
| 1338 | + | } | |
| 1339 | + | ||
| 1340 | + | // --- What hosts and g1t say ----------------------------------------------- | |
| 1341 | + | ||
| 1342 | + | /// A push on GitHub, from the App's webhook. | |
| 1343 | + | pub async fn on_github_push(&self, payload: &Value) -> Result<()> { | |
| 1344 | + | let Some(id) = payload["repository"]["id"].as_u64() else { return Ok(()) }; | |
| 1345 | + | let rows = self | |
| 1346 | + | .db | |
| 1347 | + | .prepare("SELECT * FROM remotes WHERE provider = 'github' AND external_id = ?") | |
| 1348 | + | .bind(&[id.to_string().into()])? | |
| 1349 | + | .all() | |
| 1350 | + | .await? | |
| 1351 | + | .results::<RemoteRow>()?; | |
| 1352 | + | for row in rows { | |
| 1353 | + | if let Err(error) = self.on_remote_push(&row, payload).await { | |
| 1354 | + | worker::console_error!("mirrors: a push on {} was not followed: {error}", row.name); | |
| 1355 | + | } | |
| 1356 | + | } | |
| 1357 | + | Ok(()) | |
| 1358 | + | } | |
| 1359 | + | ||
| 1360 | + | /// A push on a remote: a mirror standing by catches up; during a | |
| 1361 | + | /// takeover nothing moves (the hand-back will see it); a follower takes | |
| 1362 | + | /// in fast-forwards, or overwrites, as its settings say. | |
| 1363 | + | async fn on_remote_push(&self, row: &RemoteRow, payload: &Value) -> Result<()> { | |
| 1364 | + | if row.leads() { | |
| 1365 | + | if row.state().mirror().is_some_and(MirrorState::follows) | |
| 1366 | + | && let Err(reason) = self.sync(row).await? | |
| 1367 | + | { | |
| 1368 | + | worker::console_log!("mirrors: {} did not catch up with {}: {reason}", row.repo, row.name); | |
| 1369 | + | } | |
| 1370 | + | return Ok(()); | |
| 1371 | + | } | |
| 1372 | + | let (Some(git_ref), Some(after)) = (payload["ref"].as_str(), payload["after"].as_str()) else { | |
| 1373 | + | return Ok(()); | |
| 1374 | + | }; | |
| 1375 | + | let before = payload["before"].as_str().filter(|hash| !hash.chars().all(|c| c == '0')); | |
| 1376 | + | let deleted = payload["deleted"].as_bool() == Some(true) || after.chars().all(|c| c == '0'); | |
| 1377 | + | let (username, token) = match self.credential(row).await? { | |
| 1378 | + | Ok(credential) => credential, | |
| 1379 | + | Err(reason) => return self.note(row, Some(&reason)).await, | |
| 1380 | + | }; | |
| 1381 | + | let refs = match self.repos_refs(row, username.clone(), token.clone()).await? { | |
| 1382 | + | Outcome::Ok(refs) => refs, | |
| 1383 | + | Outcome::Fail(failure) => return self.note(row, Some(&failure.message)).await, | |
| 1384 | + | }; | |
| 1385 | + | let ours = refs.ours.get(git_ref).map(String::as_str); | |
| 1386 | + | // g1t's own push coming back: nothing to do. | |
| 1387 | + | if ours == Some(after) || (deleted && ours.is_none()) { | |
| 1388 | + | return Ok(()); | |
| 1389 | + | } | |
| 1390 | + | if row.settings().remote_pushes == RemotePushes::Overwrite { | |
| 1391 | + | let _ = self.sync(row).await?; | |
| 1392 | + | return Ok(()); | |
| 1393 | + | } | |
| 1394 | + | let forced = payload["forced"].as_bool() == Some(true); | |
| 1395 | + | if !deleted && !forced && before == ours { | |
| 1396 | + | let moves = vec![RefMove { | |
| 1397 | + | direction: MirrorDirection::Pull, | |
| 1398 | + | git_ref: git_ref.to_owned(), | |
| 1399 | + | to: None, | |
| 1400 | + | old: ours.map(str::to_owned), | |
| 1401 | + | new: Some(after.to_owned()), | |
| 1402 | + | }]; | |
| 1403 | + | if let Outcome::Ok(applied) = self.apply(row, username, token, moves).await? | |
| 1404 | + | && applied.moved.iter().all(|moved| moved.problem.is_none()) | |
| 1405 | + | { | |
| 1406 | + | return self.note(row, None).await; | |
| 1407 | + | } | |
| 1408 | + | } | |
| 1409 | + | let branch = branch_of(git_ref).unwrap_or(git_ref); | |
| 1410 | + | let reason = format!( | |
| 1411 | + | "{branch} changed on {} in a way g1t cannot follow. Sync to push g1t's {branch} over it, or bring that change into g1t first.", | |
| 1412 | + | row.name | |
| 1413 | + | ); | |
| 1414 | + | self.note(row, Some(&reason)).await?; | |
| 1415 | + | self.set_state(row, RemoteState::Stuck, None).await?; | |
| 1416 | + | Ok(()) | |
| 1417 | + | } | |
| 1418 | + | ||
| 1419 | + | /// A push on g1t: each follower is sent it. A stuck follower waits for | |
| 1420 | + | /// someone to sync it, so a change made on it is not pushed over. | |
| 1421 | + | pub async fn on_event(&self, event: &g1t_contracts::events::Event) -> Result<()> { | |
| 1422 | + | if event.kind != "git.push" { | |
| 1423 | + | return Ok(()); | |
| 1424 | + | } | |
| 1425 | + | let Some(repo_id) = event.repo_id.as_deref() else { return Ok(()) }; | |
| 1426 | + | for row in self.rows(repo_id).await? { | |
| 1427 | + | if row.leads() || row.state() != RemoteState::Following { | |
| 1428 | + | continue; | |
| 1429 | + | } | |
| 1430 | + | if let Err(reason) = self.sync(&row).await? { | |
| 1431 | + | worker::console_log!("mirrors: {} not pushed to {}: {reason}", row.repo, row.name); | |
| 1432 | + | } | |
| 1433 | + | } | |
| 1434 | + | Ok(()) | |
| 1435 | + | } | |
| 1436 | + | ||
| 1437 | + | /// Records one check of a host, and acts when it goes down or comes back. | |
| 1438 | + | async fn host_checked(&self, host: &str, answered: bool, problem: Option<&str>) -> Result<()> { | |
| 1439 | + | let now = now_ms(); | |
| 1440 | + | let row = self | |
| 1441 | + | .db | |
| 1442 | + | .prepare("SELECT * FROM remote_hosts WHERE host = ?") | |
| 1443 | + | .bind(&[host.into()])? | |
| 1444 | + | .first::<HostRow>(None) | |
| 1445 | + | .await? | |
| 1446 | + | .unwrap_or_else(|| HostRow { host: host.to_owned(), ..HostRow::default() }); | |
| 1447 | + | let (next, change) = row.health().checked(answered, now); | |
| 1448 | + | self.db | |
| 1449 | + | .prepare( | |
| 1450 | + | "INSERT INTO remote_hosts (host, failures, successes, streak_ms, unreachable_since, checked_ms, last_problem) | |
| 1451 | + | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) | |
| 1452 | + | ON CONFLICT (host) DO UPDATE SET failures = excluded.failures, successes = excluded.successes, | |
| 1453 | + | streak_ms = excluded.streak_ms, unreachable_since = excluded.unreachable_since, | |
| 1454 | + | checked_ms = excluded.checked_ms, last_problem = COALESCE(excluded.last_problem, remote_hosts.last_problem)", | |
| 1455 | + | ) | |
| 1456 | + | .bind(&[ | |
| 1457 | + | host.into(), | |
| 1458 | + | next.failures.into(), | |
| 1459 | + | next.successes.into(), | |
| 1460 | + | (next.streak_ms as f64).into(), | |
| 1461 | + | null_or(next.unreachable_since.map(rfc3339).as_deref()), | |
| 1462 | + | (now as f64).into(), | |
| 1463 | + | null_or(problem), | |
| 1464 | + | ])? | |
| 1465 | + | .run() | |
| 1466 | + | .await?; | |
| 1467 | + | if change == Change::None { | |
| 1468 | + | return Ok(()); | |
| 1469 | + | } | |
| 1470 | + | let rows = self | |
| 1471 | + | .db | |
| 1472 | + | .prepare("SELECT * FROM remotes WHERE role = 'leader'") | |
| 1473 | + | .all() | |
| 1474 | + | .await? | |
| 1475 | + | .results::<RemoteRow>()?; | |
| 1476 | + | for row in rows.into_iter().filter(|row| row.host() == host) { | |
| 1477 | + | let (kind, title) = match change { | |
| 1478 | + | Change::WentDown => ("mirror.unreachable", format!("{} is not answering. {} keeps its copy.", row.name, row.repo)), | |
| 1479 | + | _ => ("mirror.reachable", format!("{} answers again", row.name)), | |
| 1480 | + | }; | |
| 1481 | + | let notify = self.told(&row, None).await; | |
| 1482 | + | let detail = (change == Change::WentDown && row.state().mirror().is_some_and(MirrorState::follows)) | |
| 1483 | + | .then(|| format!("Take over {} to keep working on g1t until it is back.", row.repo)); | |
| 1484 | + | self.publish(kind, &row, None, title, detail, notify).await; | |
| 1485 | + | } | |
| 1486 | + | Ok(()) | |
| 1487 | + | } | |
| 1488 | + | ||
| 1489 | + | // --- Every minute ----------------------------------------------------------- | |
| 1490 | + | ||
| 1491 | + | pub async fn on_minute(&self) -> Result<()> { | |
| 1492 | + | let now = now_ms(); | |
| 1493 | + | // Mirrors the repos service has not heard of yet. | |
| 1494 | + | let unrecorded = self | |
| 1495 | + | .db | |
| 1496 | + | .prepare("SELECT * FROM remotes WHERE recorded = 0 LIMIT 50") | |
| 1497 | + | .all() | |
| 1498 | + | .await? | |
| 1499 | + | .results::<RemoteRow>()?; | |
| 1500 | + | for row in &unrecorded { | |
| 1501 | + | if row.leads() { | |
| 1502 | + | self.record(row).await?; | |
| 1503 | + | } else { | |
| 1504 | + | self.db.prepare("UPDATE remotes SET recorded = 1 WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; | |
| 1505 | + | } | |
| 1506 | + | } | |
| 1507 | + | let leaders = self | |
| 1508 | + | .db | |
| 1509 | + | .prepare("SELECT * FROM remotes WHERE role = 'leader'") | |
| 1510 | + | .all() | |
| 1511 | + | .await? | |
| 1512 | + | .results::<RemoteRow>()?; | |
| 1513 | + | // One check per host a mirror follows. | |
| 1514 | + | let mut by_host: BTreeMap<String, &RemoteRow> = BTreeMap::new(); | |
| 1515 | + | for row in &leaders { | |
| 1516 | + | by_host.entry(row.host()).or_insert(row); | |
| 1517 | + | } | |
| 1518 | + | for (host, row) in &by_host { | |
| 1519 | + | let answered = self.probe(row).await?; | |
| 1520 | + | self.host_checked(host, answered.is_ok(), answered.err().as_deref()).await?; | |
| 1521 | + | } | |
| 1522 | + | let hosts = self.hosts().await?; | |
| 1523 | + | for row in &leaders { | |
| 1524 | + | let health = hosts.get(&row.host()).map(HostRow::health).unwrap_or_default(); | |
| 1525 | + | let settings = row.settings(); | |
| 1526 | + | match row.state() { | |
| 1527 | + | // Taking over on its own, only when asked to. | |
| 1528 | + | RemoteState::Standby | RemoteState::Ci => { | |
| 1529 | + | if let (Some(minutes), Some(since)) = (settings.take_over_after, health.unreachable_since) | |
| 1530 | + | && now.saturating_sub(since) >= u64::from(minutes) * 60_000 | |
| 1531 | + | { | |
| 1532 | + | if let Outcome::Fail(failure) = self.start_takeover(row.clone(), By::G1t).await? { | |
| 1533 | + | worker::console_log!("mirrors: {} not taken over: {}", row.repo, failure.message); | |
| 1534 | + | } | |
| 1535 | + | continue; | |
| 1536 | + | } | |
| 1537 | + | // Hosts with no webhook are polled. | |
| 1538 | + | if row.provider() != RemoteProvider::Github && health.reachable() { | |
| 1539 | + | let polled = self | |
| 1540 | + | .db | |
| 1541 | + | .prepare("UPDATE remotes SET polled_ms = ?2 WHERE id = ?1 AND polled_ms < ?3 RETURNING id") | |
| 1542 | + | .bind(&[row.id.as_str().into(), (now as f64).into(), (now.saturating_sub(POLL_MS) as f64).into()])? | |
| 1543 | + | .first::<Value>(None) | |
| 1544 | + | .await?; | |
| 1545 | + | if polled.is_some() { | |
| 1546 | + | let _ = self.sync(row).await?; | |
| 1547 | + | } | |
| 1548 | + | } | |
| 1549 | + | } | |
| 1550 | + | // Handing back on its own once the remote is back and it is | |
| 1551 | + | // clean, when asked to. | |
| 1552 | + | RemoteState::Takeover if health.reachable() && settings.hand_back == HandBack::WhenClean && row.state_by.as_deref() == Some("g1t") => { | |
| 1553 | + | if let Outcome::Ok(plan) = self.plan(row).await? | |
| 1554 | + | && plan.clean() | |
| 1555 | + | && let Outcome::Fail(failure) = self.finish_takeover(row.clone(), By::G1t).await? | |
| 1556 | + | { | |
| 1557 | + | worker::console_log!("mirrors: {} not handed back: {}", row.repo, failure.message); | |
| 1558 | + | } | |
| 1559 | + | } | |
| 1560 | + | _ => {} | |
| 1561 | + | } | |
| 1562 | + | } | |
| 1563 | + | Ok(()) | |
| 1564 | + | } | |
| 1565 | + | ||
| 1566 | + | /// Whether a remote's host answers `info/refs`: any answer but a server | |
| 1567 | + | /// error counts, since a refused credential is the link's problem, not | |
| 1568 | + | /// the host's. | |
| 1569 | + | async fn probe(&self, row: &RemoteRow) -> Result<std::result::Result<(), String>> { | |
| 1570 | + | let (username, token) = match self.credential(row).await? { | |
| 1571 | + | Ok(credential) => credential, | |
| 1572 | + | // No credential: ask without one; the host still answers. | |
| 1573 | + | Err(_) => (None, String::new()), | |
| 1574 | + | }; | |
| 1575 | + | let authorization = if token.is_empty() { | |
| 1576 | + | String::new() | |
| 1577 | + | } else { | |
| 1578 | + | let user = username.unwrap_or_else(|| "x-access-token".to_owned()); | |
| 1579 | + | format!("Basic {}", base64(&format!("{user}:{token}"))) | |
| 1580 | + | }; | |
| 1581 | + | let url = format!("{}/info/refs?service=git-upload-pack", row.clone_url); | |
| 1582 | + | let mut headers = vec![("accept", "*/*")]; | |
| 1583 | + | if !authorization.is_empty() { | |
| 1584 | + | headers.push(("authorization", authorization.as_str())); | |
| 1585 | + | } | |
| 1586 | + | match http::send(Method::Get, &url, &headers, None).await { | |
| 1587 | + | Ok(answer) if answer.status < 500 && answer.status != 429 && answer.status != 408 => Ok(Ok(())), | |
| 1588 | + | Ok(answer) => Ok(Err(format!("{} answered {}.", row.host(), answer.status))), | |
| 1589 | + | Err(_) => Ok(Err(format!("{} could not be reached.", row.host()))), | |
| 1590 | + | } | |
| 1591 | + | } | |
| 1592 | + | } | |
| 1593 | + | ||
| 1594 | + | fn base64(text: &str) -> String { | |
| 1595 | + | const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; | |
| 1596 | + | let bytes = text.as_bytes(); | |
| 1597 | + | let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); | |
| 1598 | + | for chunk in bytes.chunks(3) { | |
| 1599 | + | let n = (chunk[0] as u32) << 16 | (*chunk.get(1).unwrap_or(&0) as u32) << 8 | *chunk.get(2).unwrap_or(&0) as u32; | |
| 1600 | + | for (i, shift) in [18, 12, 6, 0].into_iter().enumerate() { | |
| 1601 | + | out.push(if i <= chunk.len() { ALPHABET[(n >> shift) as usize & 63] as char } else { '=' }); | |
| 1602 | + | } | |
| 1603 | + | } | |
| 1604 | + | out | |
| 1605 | + | } | |
| 1606 | + | ||
| 1607 | + | /// When the remote is away, the plan assumes it is where the takeover | |
| 1608 | + | /// started. | |
| 1609 | + | fn bases_as_theirs(bases: &BTreeMap<String, (Option<String>, Option<RefDecision>)>) -> BTreeMap<String, String> { | |
| 1610 | + | bases.iter().filter_map(|(name, (base, _))| base.clone().map(|base| (name.clone(), base))).collect() | |
| 1611 | + | } | |
| 1612 | + | ||
| 1613 | + | fn undecided(plan: &HandbackPlan) -> g1t_contracts::Failure { | |
| 1614 | + | let waiting: Vec<&str> = plan | |
| 1615 | + | .refs | |
| 1616 | + | .iter() | |
| 1617 | + | .filter(|item| item.action == RefAction::Diverged && item.decision.is_none()) | |
| 1618 | + | .map(|item| branch_of(&item.git_ref).unwrap_or(&item.git_ref)) | |
| 1619 | + | .collect(); | |
| 1620 | + | g1t_contracts::Failure { | |
| 1621 | + | code: FailureCode::Conflict, | |
| 1622 | + | message: format!( | |
| 1623 | + | "Changed on both sides: {}. Decide for each whether to keep g1t's, keep the remote's, or send g1t's as a pull request.", | |
| 1624 | + | waiting.join(", ") | |
| 1625 | + | ), | |
| 1626 | + | } | |
| 1627 | + | } | |
| 1628 | + | ||
| 1629 | + | /// Answers the `mirror_*` methods; `None` for any other. | |
| 1630 | + | pub async fn route(method: &str, body: &Value, env: &Env, _ctx: &Context) -> Option<Result<Response>> { | |
| 1631 | + | if !method.starts_with("mirror_") { | |
| 1632 | + | return None; | |
| 1633 | + | } | |
| 1634 | + | Some(handle(method, body.clone(), env).await) | |
| 1635 | + | } | |
| 1636 | + | ||
| 1637 | + | async fn handle(method: &str, body: Value, env: &Env) -> Result<Response> { | |
| 1638 | + | let mirrors = Mirrors::new(env)?; | |
| 1639 | + | match method { | |
| 1640 | + | "mirror_view" => reply(&mirrors.view(args(body)?).await?), | |
| 1641 | + | "mirror_briefs" => reply(&mirrors.briefs(args(body)?).await?), | |
| 1642 | + | "mirror_take_over" => reply(&mirrors.take_over(args(body)?).await?), | |
| 1643 | + | "mirror_ci" => reply(&mirrors.ci(args(body)?).await?), | |
| 1644 | + | "mirror_hand_back_plan" => reply(&mirrors.hand_back_plan(args(body)?).await?), | |
| 1645 | + | "mirror_hand_back" => reply(&mirrors.hand_back(args(body)?).await?), | |
| 1646 | + | "mirror_move_in" => reply(&mirrors.move_in(args(body)?).await?), | |
| 1647 | + | "mirror_sync" => reply(&mirrors.sync_now(args(body)?).await?), | |
| 1648 | + | "mirror_settings" => reply(&mirrors.settings(args(body)?).await?), | |
| 1649 | + | "mirror_add" => reply(&mirrors.add(args(body)?).await?), | |
| 1650 | + | "mirror_remove" => reply(&mirrors.remove(args(body)?).await?), | |
| 1651 | + | _ => Response::error("Unknown method", 404), | |
| 1652 | + | } | |
| 1653 | + | } | |
| 1654 | + | ||
| 1655 | + | #[cfg(test)] | |
| 1656 | + | mod tests { | |
| 1657 | + | use super::*; | |
| 1658 | + | ||
| 1659 | + | #[test] | |
| 1660 | + | fn remotes_are_named_as_people_say_them() { | |
| 1661 | + | assert_eq!(host_of("https://GitHub.com/acme/web.git"), "github.com"); | |
| 1662 | + | assert_eq!(host_of("https://user@git.acme.internal/x"), "git.acme.internal"); | |
| 1663 | + | assert_eq!(display_name("https://github.com/acme/web.git"), "github.com/acme/web"); | |
| 1664 | + | assert_eq!(display_name("https://g1t.sh/acme/web/"), "g1t.sh/acme/web"); | |
| 1665 | + | assert_eq!(clean_clone_url("https://g1t.sh/acme/web").as_deref(), Some("https://g1t.sh/acme/web.git")); | |
| 1666 | + | assert_eq!(clean_clone_url("https://git.example/a.git/").as_deref(), Some("https://git.example/a.git")); | |
| 1667 | + | assert_eq!(clean_clone_url("http://git.example/a"), None, "https only"); | |
| 1668 | + | assert_eq!(clean_clone_url("https://me:secret@git.example/a"), None, "no credentials in the address"); | |
| 1669 | + | assert_eq!(clean_clone_url("https://git.example"), None); | |
| 1670 | + | assert_eq!(web_url("https://github.com/acme/web.git"), "https://github.com/acme/web"); | |
| 1671 | + | } | |
| 1672 | + | ||
| 1673 | + | #[test] | |
| 1674 | + | fn a_host_goes_down_slowly_and_comes_back_slower() { | |
| 1675 | + | let minute = 60_000; | |
| 1676 | + | let mut health = HostHealth::default(); | |
| 1677 | + | let mut changes = Vec::new(); | |
| 1678 | + | for at in [0, minute, 2 * minute] { | |
| 1679 | + | let (next, change) = health.checked(false, at); | |
| 1680 | + | health = next; | |
| 1681 | + | changes.push(change); | |
| 1682 | + | } | |
| 1683 | + | assert_eq!(changes, [Change::None, Change::None, Change::WentDown], "three failures over two minutes"); | |
| 1684 | + | assert!(!health.reachable()); | |
| 1685 | + | // One blip does not bring it back, nor do three quick answers. | |
| 1686 | + | let (next, change) = health.checked(true, 3 * minute); | |
| 1687 | + | assert_eq!(change, Change::None); | |
| 1688 | + | let (next, _) = next.checked(true, 3 * minute + 1); | |
| 1689 | + | let (next, change) = next.checked(true, 3 * minute + 2); | |
| 1690 | + | assert_eq!(change, Change::None, "three answers in a moment are not five minutes"); | |
| 1691 | + | let (back, change) = next.checked(true, 8 * minute); | |
| 1692 | + | assert_eq!(change, Change::CameBack); | |
| 1693 | + | assert!(back.reachable()); | |
| 1694 | + | // A failure in between starts the count again. | |
| 1695 | + | let (flaky, _) = HostHealth::default().checked(false, 0); | |
| 1696 | + | let (flaky, _) = flaky.checked(true, minute); | |
| 1697 | + | let (flaky, _) = flaky.checked(false, 2 * minute); | |
| 1698 | + | let (_, change) = flaky.checked(false, 3 * minute); | |
| 1699 | + | assert_eq!(change, Change::None); | |
| 1700 | + | } | |
| 1701 | + | ||
| 1702 | + | fn refs(pairs: &[(&str, &str)]) -> BTreeMap<String, String> { | |
| 1703 | + | pairs.iter().map(|(name, hash)| (name.to_string(), hash.to_string())).collect() | |
| 1704 | + | } | |
| 1705 | + | ||
| 1706 | + | #[test] | |
| 1707 | + | fn a_hand_back_plan_reads_each_ref_against_where_the_takeover_began() { | |
| 1708 | + | let bases: BTreeMap<_, _> = [ | |
| 1709 | + | ("refs/heads/main", "a"), | |
| 1710 | + | ("refs/heads/fix", "f"), | |
| 1711 | + | ("refs/heads/docs", "d"), | |
| 1712 | + | ("refs/heads/quiet", "q"), | |
| 1713 | + | ] | |
| 1714 | + | .into_iter() | |
| 1715 | + | .map(|(name, base)| (name.to_owned(), (Some(base.to_owned()), None))) | |
| 1716 | + | .collect(); | |
| 1717 | + | let ours = refs(&[ | |
| 1718 | + | ("refs/heads/main", "a2"), | |
| 1719 | + | ("refs/heads/fix", "f2"), | |
| 1720 | + | ("refs/heads/docs", "d2"), | |
| 1721 | + | ("refs/heads/quiet", "q"), | |
| 1722 | + | ("refs/heads/agent", "n"), | |
| 1723 | + | ]); | |
| 1724 | + | let theirs = refs(&[ | |
| 1725 | + | ("refs/heads/main", "a"), | |
| 1726 | + | ("refs/heads/fix", "f"), | |
| 1727 | + | ("refs/heads/docs", "d3"), | |
| 1728 | + | ("refs/heads/quiet", "q"), | |
| 1729 | + | ("refs/heads/g1t/handback/old", "x"), | |
| 1730 | + | ]); | |
| 1731 | + | let protected = BTreeSet::from(["refs/heads/main".to_owned()]); | |
| 1732 | + | let plan = plan_refs(&bases, &ours, &theirs, &protected); | |
| 1733 | + | let actions: Vec<(&str, RefAction)> = plan.iter().map(|item| (item.git_ref.as_str(), item.action)).collect(); | |
| 1734 | + | assert_eq!(actions, [ | |
| 1735 | + | ("refs/heads/agent", RefAction::Push), | |
| 1736 | + | ("refs/heads/docs", RefAction::Diverged), | |
| 1737 | + | ("refs/heads/fix", RefAction::Push), | |
| 1738 | + | ("refs/heads/main", RefAction::PullRequest), | |
| 1739 | + | ]); | |
| 1740 | + | let plan = HandbackPlan::new(plan, true); | |
| 1741 | + | assert!(!plan.ready, "docs needs a decision"); | |
| 1742 | + | ||
| 1743 | + | let (moves, pull_requests) = hand_back_moves(&plan, &theirs); | |
| 1744 | + | assert_eq!(pull_requests, ["refs/heads/main"]); | |
| 1745 | + | let summary: Vec<(MirrorDirection, &str, Option<&str>)> = | |
| 1746 | + | moves.iter().map(|m| (m.direction, m.git_ref.as_str(), m.to.as_deref())).collect(); | |
| 1747 | + | assert_eq!(summary, [ | |
| 1748 | + | (MirrorDirection::Push, "refs/heads/agent", None), | |
| 1749 | + | (MirrorDirection::Push, "refs/heads/fix", None), | |
| 1750 | + | (MirrorDirection::Push, "refs/heads/main", Some("refs/heads/g1t/handback/main")), | |
| 1751 | + | (MirrorDirection::Pull, "refs/heads/main", None), | |
| 1752 | + | ], "an undecided ref does not move"); | |
| 1753 | + | let main_back = &moves[3]; | |
| 1754 | + | assert_eq!((main_back.old.as_deref(), main_back.new.as_deref()), (Some("a2"), Some("a")), "g1t follows the remote's main"); | |
| 1755 | + | let push_fix = &moves[1]; | |
| 1756 | + | assert_eq!((push_fix.old.as_deref(), push_fix.new.as_deref()), (Some("f"), Some("f2")), "only from where the remote is"); | |
| 1757 | + | } | |
| 1758 | + | ||
| 1759 | + | #[test] | |
| 1760 | + | fn decisions_settle_a_diverged_ref() { | |
| 1761 | + | let plan = |decision| { | |
| 1762 | + | HandbackPlan::new( | |
| 1763 | + | vec![RefPlan { | |
| 1764 | + | git_ref: "refs/heads/docs".into(), | |
| 1765 | + | base: Some("d".into()), | |
| 1766 | + | ours: Some("d2".into()), | |
| 1767 | + | theirs: Some("d3".into()), | |
| 1768 | + | action: RefAction::Diverged, | |
| 1769 | + | decision: Some(decision), | |
| 1770 | + | }], | |
| 1771 | + | true, | |
| 1772 | + | ) | |
| 1773 | + | }; | |
| 1774 | + | let theirs = refs(&[("refs/heads/g1t/handback/docs", "old")]); | |
| 1775 | + | let (moves, _) = hand_back_moves(&plan(RefDecision::KeepOurs), &theirs); | |
| 1776 | + | assert_eq!((moves[0].direction, moves[0].old.as_deref(), moves[0].new.as_deref()), (MirrorDirection::Push, Some("d3"), Some("d2"))); | |
| 1777 | + | let (moves, _) = hand_back_moves(&plan(RefDecision::KeepTheirs), &theirs); | |
| 1778 | + | assert_eq!((moves[0].direction, moves[0].old.as_deref(), moves[0].new.as_deref()), (MirrorDirection::Pull, Some("d2"), Some("d3"))); | |
| 1779 | + | let (moves, pull_requests) = hand_back_moves(&plan(RefDecision::PullRequest), &theirs); | |
| 1780 | + | assert_eq!(pull_requests, ["refs/heads/docs"]); | |
| 1781 | + | assert_eq!(moves[0].old.as_deref(), Some("old"), "an old hand-back branch is moved from where it is"); | |
| 1782 | + | assert_eq!(moves.len(), 2); | |
| 1783 | + | } | |
| 1784 | + | } |
| 24 | 24 | { "binding": "WORK", "service": "g1t-work" }, | |
| 25 | 25 | { "binding": "RUNNER", "service": "g1t-runner" }, | |
| 26 | 26 | // Where a transferred repository is now, for moving its links. | |
| 27 | − | { "binding": "REPOS", "service": "g1t-repos" } | |
| 27 | + | { "binding": "REPOS", "service": "g1t-repos" }, | |
| 28 | + | { "binding": "EVENTS", "service": "g1t-events" } | |
| 28 | 29 | ], | |
| 29 | 30 | // Work starting on, and finishing, what an outside system is waiting on. | |
| 31 | + | // Mirrors' hosts are checked every minute (src/remotes.rs). | |
| 32 | + | "triggers": { "crons": ["* * * * *"] }, | |
| 30 | 33 | "queues": { | |
| 31 | 34 | "consumers": [{ "queue": "g1t-events-integrations", "max_batch_size": 20, "max_batch_timeout": 1 }] | |
| 32 | 35 | }, |
| 1 | + | -- A repository that mirrors a remote which leads (see crates/contracts | |
| 2 | + | -- src/mirrors.rs). The integrations service owns the link and sets this | |
| 3 | + | -- through `set_mirror`; it is kept here so pushes, merges and agents are | |
| 4 | + | -- refused or allowed without asking anyone. | |
| 5 | + | -- | |
| 6 | + | -- mirror: JSON `RepoMirror` (state standby | ci | takeover | handing_back, | |
| 7 | + | -- the remote's name and address, since, and the workflow levers). Null | |
| 8 | + | -- for a repository that leads. | |
| 9 | + | ALTER TABLE repos ADD COLUMN mirror TEXT; |
| 441 | 441 | if !actor.verified { | |
| 442 | 442 | return Ok(Err(Outcome::fail(FailureCode::Forbidden, UNVERIFIED))); | |
| 443 | 443 | } | |
| 444 | − | if let Some((code, message)) = crate::lifecycle::archived_refusal(&repo) { | |
| 444 | + | if let Some((code, message)) = crate::lifecycle::read_only_refusal(&repo) { | |
| 445 | 445 | return Ok(Err(Outcome::fail(code, message))); | |
| 446 | 446 | } | |
| 447 | 447 | Ok(Ok(repo)) |
| 400 | 400 | if !a.actor.verified { | |
| 401 | 401 | return Ok(Outcome::fail(FailureCode::Forbidden, UNVERIFIED)); | |
| 402 | 402 | } | |
| 403 | − | if let Some((code, message)) = crate::lifecycle::archived_refusal(&target) { | |
| 403 | + | if let Some((code, message)) = crate::lifecycle::read_only_refusal(&target) { | |
| 404 | 404 | return Ok(Outcome::fail(code, message)); | |
| 405 | 405 | } | |
| 406 | 406 | // Moving between namespaces: wait for it (moves.rs). Both are read |
| 52 | 52 | if !a.actor.verified { | |
| 53 | 53 | return Ok(Outcome::fail(FailureCode::Forbidden, UNVERIFIED)); | |
| 54 | 54 | } | |
| 55 | − | if let Some((code, message)) = crate::lifecycle::archived_refusal(&repo) { | |
| 55 | + | if let Some((code, message)) = crate::lifecycle::read_only_refusal(&repo) { | |
| 56 | 56 | return Ok(Outcome::fail(code, message)); | |
| 57 | 57 | } | |
| 58 | 58 | // Moving between namespaces: wait for it (moves.rs). |
| 599 | 599 | topics: Vec::new(), | |
| 600 | 600 | website: None, | |
| 601 | 601 | archived_at: None, | |
| 602 | + | mirror: a.mirror.clone(), | |
| 602 | 603 | }; | |
| 603 | 604 | let namespace = match self.place(&repo).await? { | |
| 604 | 605 | Ok(namespace) => namespace, | |
| ⋯ | |||
| 703 | 704 | }) | |
| 704 | 705 | .await?; | |
| 705 | 706 | for (git_ref, head) in &pushed { | |
| 706 | − | self.publish_push(&repo, git_ref, None, head, None).await?; | |
| 707 | + | if repo.mirror.is_some() { | |
| 708 | + | self.publish_mirrored_push(&repo, git_ref, None, head).await?; | |
| 709 | + | } else { | |
| 710 | + | self.publish_push(&repo, git_ref, None, head, None).await?; | |
| 711 | + | } | |
| 707 | 712 | } | |
| 708 | 713 | Ok(Outcome::Ok(repo)) | |
| 709 | 714 | } | |
| ⋯ | |||
| 1258 | 1263 | else { | |
| 1259 | 1264 | return Ok(not_found()); | |
| 1260 | 1265 | }; | |
| 1261 | − | if let Some((code, message)) = lifecycle::archived_refusal(&source) { | |
| 1266 | + | if let Some((code, message)) = lifecycle::read_only_refusal(&source) { | |
| 1262 | 1267 | return Ok(Outcome::fail(code, message)); | |
| 1263 | 1268 | } | |
| 1264 | 1269 | // Its working copy is made in its namespace: not while it moves. | |
| ⋯ | |||
| 1282 | 1287 | topics: Vec::new(), | |
| 1283 | 1288 | website: None, | |
| 1284 | 1289 | archived_at: None, | |
| 1290 | + | mirror: None, | |
| 1285 | 1291 | }; | |
| 1286 | 1292 | // Artifacts forks within a namespace: the copy goes where its | |
| 1287 | 1293 | // repository is. | |
| ⋯ | |||
| 1407 | 1413 | if !allowed { | |
| 1408 | 1414 | return Ok(denied()); | |
| 1409 | 1415 | } | |
| 1410 | − | // An archived repository, or a pull request's copy of one, | |
| 1411 | − | // is read-only. | |
| 1416 | + | // An archived repository, a mirror standing by, or a pull | |
| 1417 | + | // request's copy of either, is read-only. | |
| 1412 | 1418 | if write { | |
| 1413 | 1419 | let archived = match &repo.fork_of { | |
| 1414 | 1420 | Some(source) => self.registry.by_id(source).await?, | |
| ⋯ | |||
| 1416 | 1422 | }; | |
| 1417 | 1423 | match archived { | |
| 1418 | 1424 | Some(source) => { | |
| 1419 | − | if let Some((code, message)) = lifecycle::archived_refusal(&source) { | |
| 1425 | + | if let Some((code, message)) = lifecycle::read_only_refusal(&source) { | |
| 1420 | 1426 | return Ok(Outcome::fail(code, format!("{message}\n"))); | |
| 1421 | 1427 | } | |
| 1422 | 1428 | } | |
| ⋯ | |||
| 1475 | 1481 | if !a.actor.verified { | |
| 1476 | 1482 | return Ok(Outcome::fail(FailureCode::Forbidden, UNVERIFIED)); | |
| 1477 | 1483 | } | |
| 1478 | − | if let Some((code, message)) = lifecycle::archived_refusal(&target) { | |
| 1484 | + | if let Some((code, message)) = lifecycle::read_only_refusal(&target) { | |
| 1479 | 1485 | return Ok(Outcome::fail(code, message)); | |
| 1480 | 1486 | } | |
| 1481 | 1487 | // Moving between namespaces: wait for it (moves.rs). Both are read | |
| ⋯ | |||
| 1654 | 1660 | actor: Option<&User>, | |
| 1655 | 1661 | ) -> Result<()> { | |
| 1656 | 1662 | let caused_by_job = actor.and_then(g1t_contracts::events::job_run_of).map(str::to_owned); | |
| 1657 | − | self.publish_git_push(repo, git_ref, before, after, actor.map(|user| user.id.clone()), false, caused_by_job).await | |
| 1663 | + | self.publish_git_push(repo, git_ref, before, after, actor.map(|user| user.id.clone()), false, caused_by_job, false) | |
| 1664 | + | .await | |
| 1665 | + | } | |
| 1666 | + | ||
| 1667 | + | /// A push copied in from the remote a mirror follows (mirror.rs), or | |
| 1668 | + | /// made by filling a new mirror from it. | |
| 1669 | + | async fn publish_mirrored_push(&self, repo: &Repo, git_ref: &str, before: Option<&str>, after: &str) -> Result<()> { | |
| 1670 | + | self.publish_git_push(repo, git_ref, before, after, None, false, None, true).await | |
| 1658 | 1671 | } | |
| 1659 | 1672 | ||
| 1660 | 1673 | /// `publish_push`, saying whether the push reached the store without | |
| ⋯ | |||
| 1669 | 1682 | actor: Option<String>, | |
| 1670 | 1683 | unscanned: bool, | |
| 1671 | 1684 | caused_by_job: Option<String>, | |
| 1685 | + | mirrored: bool, | |
| 1672 | 1686 | ) -> Result<()> { | |
| 1673 | 1687 | self.publish(NewEvent { | |
| 1674 | 1688 | kind: "git.push", | |
| ⋯ | |||
| 1684 | 1698 | == Some(repo.default_branch.as_str()), | |
| 1685 | 1699 | unscanned, | |
| 1686 | 1700 | caused_by_job, | |
| 1701 | + | mirrored, | |
| 1702 | + | mirror: repo.mirror.clone(), | |
| 1687 | 1703 | }, | |
| 1688 | 1704 | }) | |
| 1689 | 1705 | .await | |
| ⋯ | |||
| 2189 | 2205 | actor.clone(), | |
| 2190 | 2206 | unscanned, | |
| 2191 | 2207 | caused_by_job.clone(), | |
| 2208 | + | false, | |
| 2192 | 2209 | ) | |
| 2193 | 2210 | .await?; | |
| 2194 | 2211 | } | |
| ⋯ | |||
| 2394 | 2411 | ) | |
| 2395 | 2412 | } | |
| 2396 | 2413 | "create" => reply(&repos.create(args(body)?).await?), | |
| 2397 | − | // Services only: a GitHub mirror catching up, or pushing out. | |
| 2414 | + | // Services only: a mirror catching up, or pushing out; refs moved | |
| 2415 | + | // either way when a takeover is handed back; and the mirror state | |
| 2416 | + | // integrations decided (mirror.rs). | |
| 2398 | 2417 | "mirror" => reply(&repos.mirror(args(body)?).await?), | |
| 2418 | + | "mirror_refs" => reply(&repos.mirror_refs(args(body)?).await?), | |
| 2419 | + | "mirror_apply" => reply(&repos.mirror_apply(args(body)?).await?), | |
| 2420 | + | "set_mirror" => reply(&repos.set_mirror(args(body)?).await?), | |
| 2399 | 2421 | "transfer" => reply(&repos.transfer(args(body)?).await?), | |
| 2400 | 2422 | // A repository's lifecycle: see lifecycle.rs. | |
| 2401 | 2423 | "delete" => reply(&repos.delete(args(body)?).await?), | |
| ⋯ | |||
| 2771 | 2793 | is_private: true, | |
| 2772 | 2794 | import_url: None, | |
| 2773 | 2795 | import_token: None, | |
| 2796 | + | mirror: None, | |
| 2774 | 2797 | } | |
| 2775 | 2798 | } | |
| 2776 | 2799 | ||
| 273 | 273 | .then(|| (FailureCode::Forbidden, archived_message(&repo.namespace, &repo.name))) | |
| 274 | 274 | } | |
| 275 | 275 | ||
| 276 | + | /// Why a repository takes no pushes, merges or new work now: archived, or | |
| 277 | + | /// a mirror that is not taken over (see `g1t_contracts::mirrors`). | |
| 278 | + | /// Settings stay changeable on a mirror; only an archived repository | |
| 279 | + | /// refuses those (`archived_refusal`). | |
| 280 | + | pub fn read_only_refusal(repo: &Repo) -> Option<Refusal> { | |
| 281 | + | repo.read_only_reason().map(|message| (FailureCode::Forbidden, message)) | |
| 282 | + | } | |
| 283 | + | ||
| 276 | 284 | /// Everything that decides whether a repository may go private or public. | |
| 277 | 285 | #[derive(Debug, Default)] | |
| 278 | 286 | pub struct VisibilityFacts { | |
| ⋯ | |||
| 583 | 591 | Ok(()) | |
| 584 | 592 | } | |
| 585 | 593 | ||
| 594 | + | /// A repository's mirror (see `g1t_contracts::mirrors`), or none. | |
| 595 | + | pub async fn set_mirror(&self, id: &str, mirror: Option<&g1t_contracts::mirrors::RepoMirror>) -> Result<()> { | |
| 596 | + | let mirror = mirror.and_then(|mirror| serde_json::to_string(mirror).ok()); | |
| 597 | + | self.db | |
| 598 | + | .prepare("UPDATE repos SET mirror = ? WHERE id = ?") | |
| 599 | + | .bind(&[mirror.map_or(JsValue::NULL, JsValue::from), id.into()])? | |
| 600 | + | .run() | |
| 601 | + | .await?; | |
| 602 | + | Ok(()) | |
| 603 | + | } | |
| 604 | + | ||
| 586 | 605 | /// Makes a repository and its working copies public or private. | |
| 587 | 606 | pub async fn set_private(&self, id: &str, private: bool) -> Result<()> { | |
| 588 | 607 | self.db | |
| ⋯ | |||
| 1697 | 1716 | topics: Vec::new(), | |
| 1698 | 1717 | website: None, | |
| 1699 | 1718 | archived_at: archived.then(|| "2026-10-05T00:00:00.000Z".to_owned()), | |
| 1719 | + | mirror: None, | |
| 1700 | 1720 | } | |
| 1701 | 1721 | } | |
| 1702 | 1722 | ||
| 1703 | 1723 | #[test] | |
| 1724 | + | fn a_mirror_refuses_writes_until_g1t_takes_over() { | |
| 1725 | + | use g1t_contracts::mirrors::{MirrorState, RepoMirror}; | |
| 1726 | + | let mirror = |state| Repo { | |
| 1727 | + | mirror: Some(RepoMirror { state, remote: "github.com/acme/rocket".into(), ..RepoMirror::default() }), | |
| 1728 | + | ..repo(false) | |
| 1729 | + | }; | |
| 1730 | + | let (code, message) = read_only_refusal(&mirror(MirrorState::Standby)).unwrap(); | |
| 1731 | + | assert_eq!(code, FailureCode::Forbidden); | |
| 1732 | + | assert!(message.contains("acme/rocket is a mirror of github.com/acme/rocket")); | |
| 1733 | + | assert!(message.contains("take over")); | |
| 1734 | + | assert!(read_only_refusal(&mirror(MirrorState::Ci)).is_some(), "CI failover runs workflows, not pushes"); | |
| 1735 | + | assert!(read_only_refusal(&mirror(MirrorState::Takeover)).is_none()); | |
| 1736 | + | assert!(read_only_refusal(&mirror(MirrorState::HandingBack)).unwrap().1.contains("handing back")); | |
| 1737 | + | assert!(archived_refusal(&mirror(MirrorState::Standby)).is_none(), "a mirror's settings stay changeable"); | |
| 1738 | + | assert!(read_only_refusal(&repo(true)).unwrap().1.contains("archived")); | |
| 1739 | + | assert!(read_only_refusal(&repo(false)).is_none()); | |
| 1740 | + | } | |
| 1741 | + | ||
| 1742 | + | #[test] | |
| 1704 | 1743 | fn an_archived_repository_refuses_writes_with_the_reason() { | |
| 1705 | 1744 | assert!(archived_refusal(&repo(false)).is_none()); | |
| 1706 | 1745 | let (code, message) = archived_refusal(&repo(true)).unwrap(); | |
| 1 | 1 | //! Copying every branch and tag from one git server to another: importing a | |
| 2 | 2 | //! repository with a credential, keeping a mirror in step with the host it | |
| 3 | − | //! mirrors, and pushing a repository's refs out to a host it is mirrored to. | |
| 3 | + | //! mirrors, pushing a repository's refs out to a host it is mirrored to, | |
| 4 | + | //! and moving chosen refs either way when a takeover is handed back (see | |
| 5 | + | //! `g1t_contracts::mirrors`). | |
| 4 | 6 | //! | |
| 5 | 7 | //! Like landing (see `land.rs`), this speaks git's smart HTTP protocol and | |
| 6 | 8 | //! relays the pack it receives unchanged. Both sides are asked for their | |
| 7 | 9 | //! refs; the source is asked for one pack holding what the target lacks; | |
| 8 | 10 | //! the target is sent one push that moves every ref that differs. | |
| 11 | + | //! | |
| 12 | + | //! Nothing a mirror held is lost silently: when catching up moves a branch | |
| 13 | + | //! somewhere its old commit is not part of (a force-push or deletion on the | |
| 14 | + | //! remote), the old commit is kept under `refs/g1t/replaced/`, for at | |
| 15 | + | //! least [`REPLACED_KEPT_DAYS`] days. | |
| 9 | 16 | ||
| 10 | 17 | use std::collections::{BTreeMap, HashSet}; | |
| 18 | + | use std::fmt; | |
| 11 | 19 | ||
| 12 | 20 | use futures_util::StreamExt; | |
| 13 | 21 | use worker::js_sys::Uint8Array; | |
| 14 | 22 | use worker::{Fetch, Headers, Method, Request, RequestInit, Result}; | |
| 15 | 23 | ||
| 16 | − | use g1t_contracts::repos::{MirrorArgs, MirrorDirection, Mirrored}; | |
| 24 | + | use g1t_contracts::repos::{ | |
| 25 | + | GitAccess, MirrorApplied, MirrorApplyArgs, MirrorArgs, MirrorDirection, MirrorRefs, MirrorRefsArgs, Mirrored, RefMoved, | |
| 26 | + | Repo, SetMirrorArgs, | |
| 27 | + | }; | |
| 17 | 28 | use g1t_contracts::{FailureCode, Outcome}; | |
| 29 | + | use g1t_kit::now_ms; | |
| 18 | 30 | ||
| 19 | − | use crate::land::{read_pkt_lines, unpack_sideband}; | |
| 31 | + | use crate::land::{push_ref, read_pkt_lines, unpack_sideband}; | |
| 20 | 32 | use crate::registry::store_key; | |
| 21 | 33 | use crate::store::{GitRepo, GitStore, Scope}; | |
| 22 | − | use crate::{Repos, import, not_found}; | |
| 34 | + | use crate::{Repos, descends_from, import, not_found}; | |
| 23 | 35 | ||
| 24 | 36 | const ZERO_ID: &str = "0000000000000000000000000000000000000000"; | |
| 25 | 37 | /// The most a pack may hold. A Worker holds it in memory while relaying it. | |
| ⋯ | |||
| 28 | 40 | /// had, so a sync only carries what is new. | |
| 29 | 41 | const MAX_HAVES: usize = 256; | |
| 30 | 42 | const USER_AGENT: &str = "git/2.45.0 (g1t mirror)"; | |
| 43 | + | /// Where a mirror keeps a commit the remote stopped pointing at. | |
| 44 | + | pub const REPLACED_PREFIX: &str = "refs/g1t/replaced/"; | |
| 45 | + | /// How long a kept commit stays, at least. | |
| 46 | + | pub const REPLACED_KEPT_DAYS: u64 = 30; | |
| 47 | + | /// How far back a moved branch is searched for its old commit. | |
| 48 | + | const REPLACED_HISTORY: u32 = 200; | |
| 31 | 49 | /// A pack with no objects: for a push whose refs all name objects the | |
| 32 | 50 | /// target already holds. The last 20 bytes are the SHA-1 of the first 12. | |
| 33 | 51 | const EMPTY_PACK: [u8; 32] = [ | |
| ⋯ | |||
| 64 | 82 | /// one as the password of the user `x-access-token`. The token is used | |
| 65 | 83 | /// as given: its length and shape are GitHub's to change. | |
| 66 | 84 | pub fn github(url: &str, token: &str) -> Self { | |
| 85 | + | Self::basic(url, GITHUB_USER, token) | |
| 86 | + | } | |
| 87 | + | ||
| 88 | + | /// Any https git host, with a token sent as `username`'s password. | |
| 89 | + | pub fn basic(url: &str, username: &str, token: &str) -> Self { | |
| 67 | 90 | Endpoint { | |
| 68 | 91 | url: url.trim_end_matches('/').to_owned(), | |
| 69 | − | authorization: format!("Basic {}", base64(&format!("x-access-token:{token}"))), | |
| 92 | + | authorization: format!("Basic {}", base64(&format!("{username}:{token}"))), | |
| 70 | 93 | } | |
| 71 | 94 | } | |
| 95 | + | ||
| 96 | + | /// A remote named in a service's arguments: GitHub's user unless | |
| 97 | + | /// another is given. | |
| 98 | + | pub fn remote(url: &str, username: Option<&str>, token: &str) -> Self { | |
| 99 | + | Self::basic(url, username.filter(|name| !name.trim().is_empty()).unwrap_or(GITHUB_USER), token) | |
| 100 | + | } | |
| 101 | + | } | |
| 102 | + | ||
| 103 | + | const GITHUB_USER: &str = "x-access-token"; | |
| 104 | + | ||
| 105 | + | /// Why a copy did not happen. | |
| 106 | + | #[derive(Clone, Debug, PartialEq, Eq)] | |
| 107 | + | pub enum Problem { | |
| 108 | + | /// The other host did not answer, or answered with a server error: it | |
| 109 | + | /// may be down. | |
| 110 | + | Unreachable(String), | |
| 111 | + | /// It answered, and refused or could not be used. | |
| 112 | + | Refused(String), | |
| 72 | 113 | } | |
| 73 | 114 | ||
| 115 | + | impl Problem { | |
| 116 | + | pub fn code(&self) -> FailureCode { | |
| 117 | + | match self { | |
| 118 | + | Problem::Unreachable(_) => FailureCode::Unavailable, | |
| 119 | + | Problem::Refused(_) => FailureCode::Conflict, | |
| 120 | + | } | |
| 121 | + | } | |
| 122 | + | ||
| 123 | + | pub fn message(&self) -> &str { | |
| 124 | + | match self { | |
| 125 | + | Problem::Unreachable(message) | Problem::Refused(message) => message, | |
| 126 | + | } | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | fn map(self, wrap: impl Fn(&str) -> String) -> Self { | |
| 130 | + | match self { | |
| 131 | + | Problem::Unreachable(message) => Problem::Unreachable(wrap(&message)), | |
| 132 | + | Problem::Refused(message) => Problem::Refused(wrap(&message)), | |
| 133 | + | } | |
| 134 | + | } | |
| 135 | + | } | |
| 136 | + | ||
| 137 | + | impl fmt::Display for Problem { | |
| 138 | + | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
| 139 | + | f.write_str(self.message()) | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | /// What an answer's status says about the host: a server error, a timeout | |
| 144 | + | /// or a rate limit means it may be down; anything else, that it answered. | |
| 145 | + | pub fn problem_for_status(status: u16, what: impl Into<String>) -> Problem { | |
| 146 | + | if status >= 500 || status == 408 || status == 429 { | |
| 147 | + | Problem::Unreachable(what.into()) | |
| 148 | + | } else { | |
| 149 | + | Problem::Refused(what.into()) | |
| 150 | + | } | |
| 151 | + | } | |
| 152 | + | ||
| 74 | 153 | /// What a copy changed. | |
| 75 | 154 | #[derive(Debug, Default, PartialEq, Eq)] | |
| 76 | 155 | pub struct Copied { | |
| 77 | 156 | /// Refs created or moved, each with where it was and where it is now. | |
| 78 | 157 | pub updated: Vec<(String, Option<String>, String)>, | |
| 79 | 158 | pub deleted: Vec<String>, | |
| 159 | + | /// What each deleted ref pointed at. | |
| 160 | + | pub deleted_from: BTreeMap<String, String>, | |
| 80 | 161 | /// The branch the source's HEAD names, when it said. | |
| 81 | 162 | pub head: Option<String>, | |
| 82 | 163 | } | |
| ⋯ | |||
| 139 | 220 | /// Asks a source for its refs before anything is made from it, so an | |
| 140 | 221 | /// address or credential that does not work leaves nothing behind. | |
| 141 | 222 | pub async fn probe(source: &Endpoint) -> Result<std::result::Result<Advertised, String>> { | |
| 142 | − | advertise(source, "git-upload-pack").await | |
| 223 | + | Ok(advertise(source, "git-upload-pack").await?.map_err(|problem| problem.to_string())) | |
| 143 | 224 | } | |
| 144 | 225 | ||
| 145 | 226 | /// Reads `info/refs`. Peeled tags (`^{}`) and the placeholder an empty | |
| ⋯ | |||
| 282 | 363 | } | |
| 283 | 364 | ||
| 284 | 365 | /// Asks a server for its refs, as the given service would see them. | |
| 285 | − | async fn advertise(endpoint: &Endpoint, service: &str) -> Result<std::result::Result<Advertised, String>> { | |
| 366 | + | async fn advertise(endpoint: &Endpoint, service: &str) -> Result<std::result::Result<Advertised, Problem>> { | |
| 367 | + | Ok(advertise_raw(endpoint, service).await?.map(|bytes| parse_advertisement(&bytes))) | |
| 368 | + | } | |
| 369 | + | ||
| 370 | + | /// `info/refs` as sent. | |
| 371 | + | async fn advertise_raw(endpoint: &Endpoint, service: &str) -> Result<std::result::Result<Vec<u8>, Problem>> { | |
| 286 | 372 | let url = format!("{}/info/refs?service={service}", endpoint.url); | |
| 287 | 373 | let mut response = match Fetch::Request(request(Method::Get, &url, endpoint, None)?).send().await { | |
| 288 | 374 | Ok(response) => response, | |
| 289 | − | Err(_) => return Ok(Err("The repository could not be reached.".to_owned())), | |
| 375 | + | Err(_) => return Ok(Err(Problem::Unreachable("The repository could not be reached.".to_owned()))), | |
| 290 | 376 | }; | |
| 291 | 377 | match response.status_code() { | |
| 292 | − | 200 => Ok(Ok(parse_advertisement(&response.bytes().await?))), | |
| 293 | − | 401 | 403 => Ok(Err("The repository refused g1t's credential.".to_owned())), | |
| 294 | − | 404 => Ok(Err("The repository was not found, or g1t may not read it.".to_owned())), | |
| 295 | − | status => Ok(Err(format!("The repository answered {status}."))), | |
| 378 | + | 200 => Ok(Ok(response.bytes().await?)), | |
| 379 | + | 401 | 403 => Ok(Err(Problem::Refused("The repository refused g1t's credential.".to_owned()))), | |
| 380 | + | 404 => Ok(Err(Problem::Refused("The repository was not found, or g1t may not read it.".to_owned()))), | |
| 381 | + | status => Ok(Err(problem_for_status(status, format!("The repository answered {status}.")))), | |
| 296 | 382 | } | |
| 297 | 383 | } | |
| 298 | 384 | ||
| 385 | + | /// The kept commits in an advertisement: `refs/g1t/replaced/...` names | |
| 386 | + | /// with what they point at. | |
| 387 | + | pub fn replaced_refs(bytes: &[u8]) -> Vec<(String, String)> { | |
| 388 | + | let (lines, _) = read_pkt_lines(bytes); | |
| 389 | + | lines | |
| 390 | + | .into_iter() | |
| 391 | + | .filter_map(|line| { | |
| 392 | + | let line = std::str::from_utf8(line.split(|byte| *byte == 0).next()?).ok()?; | |
| 393 | + | let (hash, name) = line.trim_end().split_once(' ')?; | |
| 394 | + | (name.starts_with(REPLACED_PREFIX) && hash.len() == 40).then(|| (name.to_owned(), hash.to_owned())) | |
| 395 | + | }) | |
| 396 | + | .collect() | |
| 397 | + | } | |
| 398 | + | ||
| 399 | + | /// The ref that keeps `old` when `name` stops pointing at it, stamped with | |
| 400 | + | /// the time in milliseconds so expired ones are found from the name alone. | |
| 401 | + | pub fn replaced_name(name: &str, now_ms: u64) -> String { | |
| 402 | + | format!("{REPLACED_PREFIX}{}/{now_ms}", name.trim_start_matches("refs/")) | |
| 403 | + | } | |
| 404 | + | ||
| 405 | + | /// The ref a kept commit was replaced on: `refs/heads/main` for | |
| 406 | + | /// `refs/g1t/replaced/heads/main/<ms>`. | |
| 407 | + | pub fn replaced_from(kept: &str) -> Option<String> { | |
| 408 | + | let rest = kept.strip_prefix(REPLACED_PREFIX)?; | |
| 409 | + | let (name, _) = rest.rsplit_once('/')?; | |
| 410 | + | Some(format!("refs/{name}")) | |
| 411 | + | } | |
| 412 | + | ||
| 413 | + | /// Whether a kept commit's ref is older than [`REPLACED_KEPT_DAYS`]. | |
| 414 | + | pub fn replaced_expired(name: &str, now_ms: u64) -> bool { | |
| 415 | + | name.rsplit('/') | |
| 416 | + | .next() | |
| 417 | + | .and_then(|stamp| stamp.parse::<u64>().ok()) | |
| 418 | + | .is_some_and(|stamp| now_ms.saturating_sub(stamp) > REPLACED_KEPT_DAYS * 86_400_000) | |
| 419 | + | } | |
| 420 | + | ||
| 299 | 421 | /// Fetches one pack holding `wants`, reading in pieces so that one too | |
| 300 | 422 | /// large is noticed before it is all held. | |
| 301 | − | async fn fetch_pack(source: &Endpoint, wants: &[String], haves: &[String]) -> Result<std::result::Result<Vec<u8>, String>> { | |
| 423 | + | async fn fetch_pack(source: &Endpoint, wants: &[String], haves: &[String]) -> Result<std::result::Result<Vec<u8>, Problem>> { | |
| 302 | 424 | let body = upload_request(wants, haves); | |
| 303 | 425 | let url = format!("{}/git-upload-pack", source.url); | |
| 304 | − | let mut response = Fetch::Request(request(Method::Post, &url, source, Some(("git-upload-pack", body)))?) | |
| 426 | + | let Ok(mut response) = Fetch::Request(request(Method::Post, &url, source, Some(("git-upload-pack", body)))?) | |
| 305 | 427 | .send() | |
| 306 | − | .await?; | |
| 428 | + | .await | |
| 429 | + | else { | |
| 430 | + | return Ok(Err(Problem::Unreachable("The repository stopped answering while sending.".to_owned()))); | |
| 431 | + | }; | |
| 307 | 432 | if response.status_code() != 200 { | |
| 308 | − | return Ok(Err(format!("The source refused to send the repository ({}).", response.status_code()))); | |
| 433 | + | let status = response.status_code(); | |
| 434 | + | return Ok(Err(problem_for_status(status, format!("The source refused to send the repository ({status}).")))); | |
| 309 | 435 | } | |
| 310 | 436 | let mut received = Vec::new(); | |
| 311 | 437 | let mut stream = response.stream()?; | |
| 312 | 438 | while let Some(chunk) = stream.next().await { | |
| 313 | 439 | received.extend_from_slice(&chunk?); | |
| 314 | 440 | if received.len() > MAX_PACK_BYTES { | |
| 315 | − | return Ok(Err(format!( | |
| 316 | − | "The repository is larger than {} MB, the most g1t copies at once. Push it with git instead.", | |
| 441 | + | return Ok(Err(Problem::Refused(format!( | |
| 442 | + | "The change is larger than {} MB, the most g1t copies at once. Push it with git instead.", | |
| 317 | 443 | MAX_PACK_BYTES / 1024 / 1024 | |
| 318 | − | ))); | |
| 444 | + | )))); | |
| 319 | 445 | } | |
| 320 | 446 | } | |
| 321 | − | Ok(unpack_sideband(&received).map_err(|_| "The source did not send a usable pack.".to_owned())) | |
| 447 | + | Ok(unpack_sideband(&received).map_err(|_| Problem::Refused("The source did not send a usable pack.".to_owned()))) | |
| 322 | 448 | } | |
| 323 | 449 | ||
| 324 | 450 | /// Makes `target`'s branches and tags match `source`'s. With | |
| 325 | 451 | /// `Prune::No`, refs only the target has are kept. `Err` in the inner | |
| 326 | 452 | /// result is a reason to show a person. | |
| 327 | − | pub async fn copy(source: &Endpoint, target: &Endpoint, prune: Prune) -> Result<std::result::Result<Copied, String>> { | |
| 453 | + | pub async fn copy(source: &Endpoint, target: &Endpoint, prune: Prune) -> Result<std::result::Result<Copied, Problem>> { | |
| 328 | 454 | let theirs = match advertise(source, "git-upload-pack").await? { | |
| 329 | 455 | Ok(refs) => refs, | |
| 330 | − | Err(reason) => return Ok(Err(reason)), | |
| 456 | + | Err(problem) => return Ok(Err(problem)), | |
| 331 | 457 | }; | |
| 332 | 458 | let ours = match advertise(target, "git-receive-pack").await? { | |
| 333 | 459 | Ok(refs) => refs, | |
| 334 | − | Err(reason) => return Ok(Err(format!("Writing the copy failed: {reason}"))), | |
| 460 | + | Err(problem) => return Ok(Err(problem.map(|reason| format!("Writing the copy failed: {reason}")))), | |
| 335 | 461 | }; | |
| 336 | 462 | let commands = plan(&theirs, &ours, prune); | |
| 337 | 463 | let mut copied = Copied { | |
| ⋯ | |||
| 341 | 467 | if commands.is_empty() { | |
| 342 | 468 | return Ok(Ok(copied)); | |
| 343 | 469 | } | |
| 344 | − | let held: HashSet<&String> = ours.refs.values().collect(); | |
| 470 | + | let results = match transfer(source, target, &ours, &commands).await? { | |
| 471 | + | Ok(results) => results, | |
| 472 | + | Err(problem) => return Ok(Err(problem)), | |
| 473 | + | }; | |
| 474 | + | let problems: Vec<String> = results | |
| 475 | + | .iter() | |
| 476 | + | .filter_map(|(name, problem)| problem.as_ref().map(|problem| format!("{name}: {problem}"))) | |
| 477 | + | .collect(); | |
| 478 | + | if !problems.is_empty() { | |
| 479 | + | return Ok(Err(Problem::Refused(format!("Some refs were not updated: {}", problems.join("; "))))); | |
| 480 | + | } | |
| 481 | + | for (name, old, new) in commands { | |
| 482 | + | if new == ZERO_ID { | |
| 483 | + | if let Some(old) = old { | |
| 484 | + | copied.deleted_from.insert(name.clone(), old); | |
| 485 | + | } | |
| 486 | + | copied.deleted.push(name); | |
| 487 | + | } else { | |
| 488 | + | copied.updated.push((name, old, new)); | |
| 489 | + | } | |
| 490 | + | } | |
| 491 | + | Ok(Ok(copied)) | |
| 492 | + | } | |
| 493 | + | ||
| 494 | + | /// Sends `commands` to `target` in one push, with a pack from `source` | |
| 495 | + | /// holding what the target lacks (`held`: what its refs already name). | |
| 496 | + | /// Each command comes back with why it was refused, if it was. | |
| 497 | + | async fn transfer( | |
| 498 | + | source: &Endpoint, | |
| 499 | + | target: &Endpoint, | |
| 500 | + | held: &Advertised, | |
| 501 | + | commands: &[Command], | |
| 502 | + | ) -> Result<std::result::Result<Vec<(String, Option<String>)>, Problem>> { | |
| 503 | + | let have: HashSet<&String> = held.refs.values().collect(); | |
| 345 | 504 | let mut wants: Vec<String> = commands | |
| 346 | 505 | .iter() | |
| 347 | 506 | .map(|(_, _, new)| new) | |
| 348 | − | .filter(|new| *new != ZERO_ID && !held.contains(new)) | |
| 507 | + | .filter(|new| *new != ZERO_ID && !have.contains(new)) | |
| 349 | 508 | .cloned() | |
| 350 | 509 | .collect(); | |
| 351 | 510 | wants.sort(); | |
| ⋯ | |||
| 353 | 512 | let pack = if wants.is_empty() { | |
| 354 | 513 | None | |
| 355 | 514 | } else { | |
| 356 | − | let haves: Vec<String> = ours.refs.values().cloned().collect::<HashSet<_>>().into_iter().collect(); | |
| 515 | + | let haves: Vec<String> = held.refs.values().cloned().collect::<HashSet<_>>().into_iter().collect(); | |
| 357 | 516 | match fetch_pack(source, &wants, &haves).await? { | |
| 358 | 517 | Ok(pack) => Some(pack), | |
| 359 | − | Err(reason) => return Ok(Err(reason)), | |
| 518 | + | Err(problem) => return Ok(Err(problem)), | |
| 360 | 519 | } | |
| 361 | 520 | }; | |
| 362 | − | let body = receive_request(&commands, pack); | |
| 521 | + | let body = receive_request(commands, pack); | |
| 363 | 522 | let url = format!("{}/git-receive-pack", target.url); | |
| 364 | − | let mut response = Fetch::Request(request(Method::Post, &url, target, Some(("git-receive-pack", body)))?) | |
| 523 | + | let Ok(mut response) = Fetch::Request(request(Method::Post, &url, target, Some(("git-receive-pack", body)))?) | |
| 365 | 524 | .send() | |
| 366 | − | .await?; | |
| 525 | + | .await | |
| 526 | + | else { | |
| 527 | + | return Ok(Err(Problem::Unreachable("The repository stopped answering while receiving.".to_owned()))); | |
| 528 | + | }; | |
| 367 | 529 | let report = response.bytes().await?; | |
| 368 | 530 | if response.status_code() != 200 { | |
| 369 | − | return Ok(Err(format!("The push was refused ({}).", response.status_code()))); | |
| 531 | + | let status = response.status_code(); | |
| 532 | + | return Ok(Err(problem_for_status(status, format!("The push was refused ({status}).")))); | |
| 370 | 533 | } | |
| 371 | − | let problems = refused(&report, &commands); | |
| 372 | − | if !problems.is_empty() { | |
| 373 | − | return Ok(Err(format!("Some refs were not updated: {}", problems.join("; ")))); | |
| 374 | − | } | |
| 375 | − | for (name, old, new) in commands { | |
| 376 | − | if new == ZERO_ID { | |
| 377 | − | copied.deleted.push(name); | |
| 378 | − | } else { | |
| 379 | − | copied.updated.push((name, old, new)); | |
| 380 | − | } | |
| 381 | − | } | |
| 382 | − | Ok(Ok(copied)) | |
| 534 | + | Ok(Ok(outcomes(&report, commands))) | |
| 535 | + | } | |
| 536 | + | ||
| 537 | + | /// Each command with why it was refused, if the report says it was. | |
| 538 | + | pub fn outcomes(report: &[u8], commands: &[Command]) -> Vec<(String, Option<String>)> { | |
| 539 | + | let refused = refused(report, commands); | |
| 540 | + | commands | |
| 541 | + | .iter() | |
| 542 | + | .map(|(name, _, _)| { | |
| 543 | + | let prefix = format!("{name}: "); | |
| 544 | + | (name.clone(), refused.iter().find_map(|line| line.strip_prefix(&prefix).map(str::to_owned))) | |
| 545 | + | }) | |
| 546 | + | .collect() | |
| 383 | 547 | } | |
| 384 | 548 | ||
| 385 | 549 | /// The pushes an import announces, one for each ref it made: the default | |
| ⋯ | |||
| 408 | 572 | impl<S: GitStore> Repos<S> { | |
| 409 | 573 | /// `mirror`: a mirror catching up with the host it mirrors, or a | |
| 410 | 574 | /// repository pushing its refs out to one. Each branch moved on g1t is | |
| 411 | − | /// announced as a push, so deployments and checks follow it. | |
| 575 | + | /// announced as a push copied in, so what follows a mirror's state | |
| 576 | + | /// (workflows, deployments) can tell. | |
| 412 | 577 | pub(crate) async fn mirror(&self, a: MirrorArgs) -> Result<Outcome<Mirrored>> { | |
| 413 | 578 | let Some(repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 414 | 579 | return Ok(not_found()); | |
| ⋯ | |||
| 431 | 596 | }; | |
| 432 | 597 | let access = self.store.open(&store_key(&repo)).await?.access(scope).await?; | |
| 433 | 598 | let ours = Endpoint::bearer(&access.remote, &access.token); | |
| 434 | − | let theirs = Endpoint::github(&url, &a.token); | |
| 599 | + | let theirs = Endpoint::remote(&url, a.username.as_deref(), &a.token); | |
| 435 | 600 | let copied = match a.direction { | |
| 436 | 601 | MirrorDirection::Pull => { | |
| 437 | 602 | let copied = copy(&theirs, &ours, Prune::Yes).await?; | |
| ⋯ | |||
| 442 | 607 | }; | |
| 443 | 608 | let copied = match copied { | |
| 444 | 609 | Ok(copied) => copied, | |
| 445 | − | Err(reason) => return Ok(Outcome::fail(FailureCode::Conflict, reason)), | |
| 610 | + | Err(problem) => return Ok(Outcome::fail(problem.code(), problem.to_string())), | |
| 446 | 611 | }; | |
| 612 | + | let mut replaced = Vec::new(); | |
| 447 | 613 | if a.direction == MirrorDirection::Pull { | |
| 614 | + | let moved: Vec<(String, Option<String>, String)> = copied | |
| 615 | + | .updated | |
| 616 | + | .iter() | |
| 617 | + | .cloned() | |
| 618 | + | .chain( | |
| 619 | + | copied | |
| 620 | + | .deleted_from | |
| 621 | + | .iter() | |
| 622 | + | .map(|(name, old)| (name.clone(), Some(old.clone()), ZERO_ID.to_owned())), | |
| 623 | + | ) | |
| 624 | + | .collect(); | |
| 625 | + | replaced = self.keep_replaced(&repo, &access, &moved).await?; | |
| 448 | 626 | for (name, old, new) in &copied.updated { | |
| 449 | 627 | if name.starts_with("refs/heads/") { | |
| 450 | − | self.publish_push(&repo, name, old.as_deref(), new, None).await?; | |
| 628 | + | self.publish_mirrored_push(&repo, name, old.as_deref(), new).await?; | |
| 451 | 629 | } | |
| 452 | 630 | } | |
| 453 | 631 | } | |
| 454 | 632 | Ok(Outcome::Ok(Mirrored { | |
| 455 | 633 | updated: copied.updated.into_iter().map(|(name, _, _)| name).collect(), | |
| 456 | 634 | deleted: copied.deleted, | |
| 635 | + | replaced, | |
| 457 | 636 | })) | |
| 458 | 637 | } | |
| 638 | + | ||
| 639 | + | /// Keeps each commit a pull stopped pointing at, when the ref's new | |
| 640 | + | /// commit does not contain it: a force-push or deletion on the remote. | |
| 641 | + | /// Returns the refs that keep them. Expired ones are let go at the same | |
| 642 | + | /// time. | |
| 643 | + | async fn keep_replaced(&self, repo: &Repo, access: &GitAccess, moved: &[(String, Option<String>, String)]) -> Result<Vec<String>> { | |
| 644 | + | let git = self.store.open(&store_key(repo)).await?; | |
| 645 | + | let now = now_ms(); | |
| 646 | + | let mut kept = Vec::new(); | |
| 647 | + | for (name, old, new) in moved { | |
| 648 | + | let Some(old) = old.as_deref() else { continue }; | |
| 649 | + | let lost = if new == ZERO_ID || name.starts_with("refs/tags/") { | |
| 650 | + | true | |
| 651 | + | } else { | |
| 652 | + | match git.log(name, REPLACED_HISTORY).await { | |
| 653 | + | Ok(history) => !descends_from(&git, &history, old).await.unwrap_or(true), | |
| 654 | + | Err(_) => false, | |
| 655 | + | } | |
| 656 | + | }; | |
| 657 | + | if !lost { | |
| 658 | + | continue; | |
| 659 | + | } | |
| 660 | + | let keep = replaced_name(name, now); | |
| 661 | + | if let Ok(Ok(())) = push_ref(access, &keep, None, old, Some(EMPTY_PACK.to_vec())).await { | |
| 662 | + | kept.push(keep); | |
| 663 | + | } | |
| 664 | + | } | |
| 665 | + | if !kept.is_empty() { | |
| 666 | + | let ours = Endpoint::bearer(&access.remote, &access.token); | |
| 667 | + | if let Ok(Ok(bytes)) = advertise_raw(&ours, "git-receive-pack").await { | |
| 668 | + | for (name, hash) in replaced_refs(&bytes).into_iter().filter(|(name, _)| replaced_expired(name, now)) { | |
| 669 | + | let _ = push_ref(access, &name, Some(&hash), ZERO_ID, None).await; | |
| 670 | + | } | |
| 671 | + | } | |
| 672 | + | self.refs_moved(&repo.id).await; | |
| 673 | + | } | |
| 674 | + | Ok(kept) | |
| 675 | + | } | |
| 676 | + | ||
| 677 | + | /// `mirror_refs`: both sides' branches and tags. | |
| 678 | + | pub(crate) async fn mirror_refs(&self, a: MirrorRefsArgs) -> Result<Outcome<MirrorRefs>> { | |
| 679 | + | let Some(repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 680 | + | return Ok(not_found()); | |
| 681 | + | }; | |
| 682 | + | let access = self.store.open(&store_key(&repo)).await?.access(Scope::Read).await?; | |
| 683 | + | let ours = match advertise(&Endpoint::bearer(&access.remote, &access.token), "git-upload-pack").await? { | |
| 684 | + | Ok(ours) => ours, | |
| 685 | + | Err(problem) => return Ok(Outcome::fail(FailureCode::Conflict, problem.to_string())), | |
| 686 | + | }; | |
| 687 | + | // No address: only g1t's side was asked for. | |
| 688 | + | if a.url.is_empty() { | |
| 689 | + | return Ok(Outcome::Ok(MirrorRefs { ours: ours.refs, theirs: None, unreachable: None })); | |
| 690 | + | } | |
| 691 | + | let Some(url) = import::clean_url(&a.url) else { | |
| 692 | + | return Ok(Outcome::fail(FailureCode::Invalid, "That is not an https repository address.")); | |
| 693 | + | }; | |
| 694 | + | let (theirs, unreachable) = match advertise(&Endpoint::remote(&url, a.username.as_deref(), &a.token), "git-upload-pack").await? { | |
| 695 | + | Ok(theirs) => (Some(theirs.refs), None), | |
| 696 | + | Err(Problem::Unreachable(reason)) => (None, Some(reason)), | |
| 697 | + | Err(problem) => return Ok(Outcome::fail(problem.code(), problem.to_string())), | |
| 698 | + | }; | |
| 699 | + | Ok(Outcome::Ok(MirrorRefs { | |
| 700 | + | ours: ours.refs, | |
| 701 | + | theirs, | |
| 702 | + | unreachable, | |
| 703 | + | })) | |
| 704 | + | } | |
| 705 | + | ||
| 706 | + | /// `mirror_apply`: moves chosen refs either way, each only from the | |
| 707 | + | /// commit the caller saw. Pushes go out in one request, pulls come in | |
| 708 | + | /// in another. | |
| 709 | + | pub(crate) async fn mirror_apply(&self, a: MirrorApplyArgs) -> Result<Outcome<MirrorApplied>> { | |
| 710 | + | let Some(repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 711 | + | return Ok(not_found()); | |
| 712 | + | }; | |
| 713 | + | let Some(url) = import::clean_url(&a.url) else { | |
| 714 | + | return Ok(Outcome::fail(FailureCode::Invalid, "That is not an https repository address.")); | |
| 715 | + | }; | |
| 716 | + | let repo = match self.unpaused(repo).await? { | |
| 717 | + | Ok(repo) => repo, | |
| 718 | + | Err((code, message)) => return Ok(Outcome::fail(code, message)), | |
| 719 | + | }; | |
| 720 | + | let access = self.store.open(&store_key(&repo)).await?.access(Scope::Write).await?; | |
| 721 | + | let ours = Endpoint::bearer(&access.remote, &access.token); | |
| 722 | + | let theirs = Endpoint::remote(&url, a.username.as_deref(), &a.token); | |
| 723 | + | let mut moved = Vec::new(); | |
| 724 | + | for direction in [MirrorDirection::Push, MirrorDirection::Pull] { | |
| 725 | + | let moves: Vec<_> = a.moves.iter().filter(|m| m.direction == direction).collect(); | |
| 726 | + | if moves.is_empty() { | |
| 727 | + | continue; | |
| 728 | + | } | |
| 729 | + | let commands: Vec<Command> = moves | |
| 730 | + | .iter() | |
| 731 | + | .map(|m| { | |
| 732 | + | ( | |
| 733 | + | m.to.clone().unwrap_or_else(|| m.git_ref.clone()), | |
| 734 | + | m.old.clone(), | |
| 735 | + | m.new.clone().unwrap_or_else(|| ZERO_ID.to_owned()), | |
| 736 | + | ) | |
| 737 | + | }) | |
| 738 | + | .collect(); | |
| 739 | + | let (source, target) = match direction { | |
| 740 | + | MirrorDirection::Push => (&ours, &theirs), | |
| 741 | + | MirrorDirection::Pull => (&theirs, &ours), | |
| 742 | + | }; | |
| 743 | + | let held = match advertise(target, "git-receive-pack").await? { | |
| 744 | + | Ok(held) => held, | |
| 745 | + | Err(problem) => return Ok(Outcome::fail(problem.code(), problem.to_string())), | |
| 746 | + | }; | |
| 747 | + | let results = match transfer(source, target, &held, &commands).await? { | |
| 748 | + | Ok(results) => results, | |
| 749 | + | Err(problem) => return Ok(Outcome::fail(problem.code(), problem.to_string())), | |
| 750 | + | }; | |
| 751 | + | let mut landed = Vec::new(); | |
| 752 | + | for ((name, problem), command) in results.into_iter().zip(&commands) { | |
| 753 | + | if problem.is_none() { | |
| 754 | + | landed.push(command.clone()); | |
| 755 | + | } | |
| 756 | + | moved.push(RefMoved { | |
| 757 | + | git_ref: name, | |
| 758 | + | direction, | |
| 759 | + | problem, | |
| 760 | + | replaced: None, | |
| 761 | + | }); | |
| 762 | + | } | |
| 763 | + | if direction == MirrorDirection::Pull && !landed.is_empty() { | |
| 764 | + | self.refs_moved(&repo.id).await; | |
| 765 | + | for keep in self.keep_replaced(&repo, &access, &landed).await? { | |
| 766 | + | let from = replaced_from(&keep); | |
| 767 | + | if let Some(entry) = moved | |
| 768 | + | .iter_mut() | |
| 769 | + | .find(|m| m.direction == direction && Some(&m.git_ref) == from.as_ref()) | |
| 770 | + | { | |
| 771 | + | entry.replaced = Some(keep); | |
| 772 | + | } | |
| 773 | + | } | |
| 774 | + | for (name, old, new) in &landed { | |
| 775 | + | if name.starts_with("refs/heads/") && new != ZERO_ID { | |
| 776 | + | self.publish_mirrored_push(&repo, name, old.as_deref(), new).await?; | |
| 777 | + | } | |
| 778 | + | } | |
| 779 | + | } | |
| 780 | + | } | |
| 781 | + | Ok(Outcome::Ok(MirrorApplied { moved })) | |
| 782 | + | } | |
| 783 | + | ||
| 784 | + | /// `set_mirror`: what the integrations service decided about a | |
| 785 | + | /// repository's mirror, kept on its row. | |
| 786 | + | pub(crate) async fn set_mirror(&self, a: SetMirrorArgs) -> Result<Outcome<Repo>> { | |
| 787 | + | let Some(mut repo) = self.registry.by_id(&a.repo_id).await? else { | |
| 788 | + | return Ok(not_found()); | |
| 789 | + | }; | |
| 790 | + | self.registry.set_mirror(&repo.id, a.mirror.as_ref()).await?; | |
| 791 | + | repo.mirror = a.mirror; | |
| 792 | + | Ok(Outcome::Ok(repo)) | |
| 793 | + | } | |
| 459 | 794 | } | |
| 460 | 795 | ||
| 461 | 796 | #[cfg(test)] | |
| ⋯ | |||
| 585 | 920 | } | |
| 586 | 921 | ||
| 587 | 922 | #[test] | |
| 923 | + | fn replaced_commits_are_kept_by_name_and_time() { | |
| 924 | + | let kept = replaced_name("refs/heads/feature/x", 1_000); | |
| 925 | + | assert_eq!(kept, "refs/g1t/replaced/heads/feature/x/1000"); | |
| 926 | + | assert_eq!(replaced_from(&kept).as_deref(), Some("refs/heads/feature/x")); | |
| 927 | + | assert!(!replaced_expired(&kept, 1_000 + REPLACED_KEPT_DAYS * 86_400_000)); | |
| 928 | + | assert!(replaced_expired(&kept, 1_001 + REPLACED_KEPT_DAYS * 86_400_000)); | |
| 929 | + | assert!(!replaced_expired("refs/g1t/replaced/heads/main/not-a-time", u64::MAX)); | |
| 930 | + | let bytes = [ | |
| 931 | + | pkt_line(&format!("{A} refs/heads/main\0report-status\n")), | |
| 932 | + | pkt_line(&format!("{B} {kept}\n")), | |
| 933 | + | b"0000".to_vec(), | |
| 934 | + | ] | |
| 935 | + | .concat(); | |
| 936 | + | assert_eq!(replaced_refs(&bytes), vec![(kept.clone(), B.to_owned())]); | |
| 937 | + | assert!(parse_advertisement(&bytes).refs.get(&kept).is_none(), "kept commits are not branches"); | |
| 938 | + | } | |
| 939 | + | ||
| 940 | + | #[test] | |
| 941 | + | fn a_server_error_means_the_host_may_be_down() { | |
| 942 | + | assert_eq!(problem_for_status(503, "x").code(), FailureCode::Unavailable); | |
| 943 | + | assert_eq!(problem_for_status(429, "x").code(), FailureCode::Unavailable); | |
| 944 | + | assert_eq!(problem_for_status(403, "x").code(), FailureCode::Conflict); | |
| 945 | + | let endpoint = Endpoint::remote("https://git.example/a.git", Some("chase"), "t"); | |
| 946 | + | assert_eq!(endpoint.authorization, format!("Basic {}", base64("chase:t"))); | |
| 947 | + | assert_eq!(Endpoint::remote("https://x", Some(" "), "t").authorization, Endpoint::github("https://x", "t").authorization); | |
| 948 | + | } | |
| 949 | + | ||
| 950 | + | #[test] | |
| 951 | + | fn each_command_comes_back_with_its_refusal() { | |
| 952 | + | let commands = vec![ | |
| 953 | + | ("refs/heads/main".to_owned(), None, A.to_owned()), | |
| 954 | + | ("refs/heads/x".to_owned(), None, B.to_owned()), | |
| 955 | + | ]; | |
| 956 | + | let report = [ | |
| 957 | + | pkt_line("unpack ok\n"), | |
| 958 | + | pkt_line("ok refs/heads/main\n"), | |
| 959 | + | pkt_line("ng refs/heads/x protected branch hook declined\n"), | |
| 960 | + | b"0000".to_vec(), | |
| 961 | + | ] | |
| 962 | + | .concat(); | |
| 963 | + | assert_eq!(outcomes(&report, &commands), vec![ | |
| 964 | + | ("refs/heads/main".to_owned(), None), | |
| 965 | + | ("refs/heads/x".to_owned(), Some("protected branch hook declined".to_owned())), | |
| 966 | + | ]); | |
| 967 | + | } | |
| 968 | + | ||
| 969 | + | #[test] | |
| 588 | 970 | fn refusals_are_reported_by_ref() { | |
| 589 | 971 | let commands = vec![ | |
| 590 | 972 | ("refs/heads/main".to_owned(), None, A.to_owned()), | |
| 33 | 33 | website: Option<String>, | |
| 34 | 34 | #[serde(default)] | |
| 35 | 35 | archived_at: Option<String>, | |
| 36 | + | /// JSON `RepoMirror`; absent on rows read before the column existed. | |
| 37 | + | #[serde(default)] | |
| 38 | + | mirror: Option<String>, | |
| 36 | 39 | #[serde(default)] | |
| 37 | 40 | deleted_at: Option<String>, | |
| 38 | 41 | /// Bumped by everything that changes the repository's refs; see | |
| ⋯ | |||
| 246 | 249 | .unwrap_or_default(), | |
| 247 | 250 | website: row.website, | |
| 248 | 251 | archived_at: row.archived_at, | |
| 252 | + | mirror: row.mirror.as_deref().and_then(|mirror| serde_json::from_str(mirror).ok()), | |
| 249 | 253 | }; | |
| 250 | 254 | if let Some(store) = &row.store { | |
| 251 | 255 | remember_store(&repo, store); | |
| ⋯ | |||
| 795 | 799 | .prepare( | |
| 796 | 800 | "INSERT INTO repos | |
| 797 | 801 | (id, namespace, name, description, is_private, owner_id, | |
| 798 | − | default_branch, fork_of, created_at, store) | |
| 799 | − | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", | |
| 802 | + | default_branch, fork_of, created_at, store, mirror) | |
| 803 | + | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", | |
| 800 | 804 | ) | |
| 801 | 805 | .bind(&[ | |
| 802 | 806 | repo.id.as_str().into(), | |
| ⋯ | |||
| 809 | 813 | optional(&repo.fork_of), | |
| 810 | 814 | repo.created_at.as_str().into(), | |
| 811 | 815 | store_key(repo).into(), | |
| 816 | + | repo.mirror | |
| 817 | + | .as_ref() | |
| 818 | + | .and_then(|mirror| serde_json::to_string(mirror).ok()) | |
| 819 | + | .map_or(JsValue::NULL, JsValue::from), | |
| 812 | 820 | ])? | |
| 813 | 821 | .run() | |
| 814 | 822 | .await?; | |
| ⋯ | |||
| 853 | 861 | topics: None, | |
| 854 | 862 | website: None, | |
| 855 | 863 | archived_at: None, | |
| 864 | + | mirror: None, | |
| 856 | 865 | deleted_at: None, | |
| 857 | 866 | refs_version: version, | |
| 858 | 867 | refs_open_until: None, | |
| ⋯ | |||
| 915 | 924 | topics: Vec::new(), | |
| 916 | 925 | website: None, | |
| 917 | 926 | archived_at: None, | |
| 927 | + | mirror: None, | |
| 918 | 928 | } | |
| 919 | 929 | } | |
| 920 | 930 | ||
| 200 | 200 | topics: Vec::new(), | |
| 201 | 201 | website: None, | |
| 202 | 202 | archived_at: None, | |
| 203 | + | mirror: None, | |
| 203 | 204 | } | |
| 204 | 205 | } | |
| 205 | 206 |
| 418 | 418 | topics: Vec::new(), | |
| 419 | 419 | website: None, | |
| 420 | 420 | archived_at: None, | |
| 421 | + | mirror: None, | |
| 421 | 422 | }; | |
| 422 | 423 | assert!(is_named(&repo, "acme/rocket")); | |
| 423 | 424 | assert!(is_named(&repo, "ACME/Rocket")); |
| 23 | 23 | use worker::Result; | |
| 24 | 24 | use worker::wasm_bindgen::JsValue; | |
| 25 | 25 | ||
| 26 | − | use crate::retired::writable; | |
| 27 | 26 | use crate::{Work, allowed}; | |
| 28 | 27 | ||
| 29 | 28 | /// Unwraps an `Outcome`, returning its failure from the enclosing method. | |
| ⋯ | |||
| 245 | 244 | async fn reportable(&self, path: &RepoPath, actor: &User) -> Result<Outcome<Repo>> { | |
| 246 | 245 | let repo = check!(self.repo(path, &Some(actor.clone())).await?); | |
| 247 | 246 | check!(allowed(Some(actor), &repo, Capability::Push)); | |
| 248 | − | check!(writable(&repo)); | |
| 247 | + | check!(crate::retired::not_archived(&repo)); | |
| 249 | 248 | Ok(Outcome::Ok(repo)) | |
| 250 | 249 | } | |
| 251 | 250 | ||
| 2 | 2 | //! renamed: what issues, pull requests and agents do about each. | |
| 3 | 3 | //! | |
| 4 | 4 | //! An archived repository is read-only: its issues and pull requests are | |
| 5 | − | //! locked and nothing new starts on it. A deleted one looks missing (repos | |
| 5 | + | //! locked and nothing new starts on it. So is a mirror standing by (see | |
| 6 | + | //! `g1t_contracts::mirrors`): its work happens where it is mirrored from, | |
| 7 | + | //! until someone takes over on g1t. A deleted one looks missing (repos | |
| 6 | 8 | //! hides it) and keeps its rows for a restore. A purged one is gone, and | |
| 7 | 9 | //! every row kept for it goes with it. | |
| 8 | 10 | ||
| ⋯ | |||
| 17 | 19 | use crate::Work; | |
| 18 | 20 | ||
| 19 | 21 | /// `Ok` when `repo` may be changed; refused, saying why, when it is | |
| 20 | − | /// archived. | |
| 22 | + | /// archived or a mirror that is not taken over. | |
| 21 | 23 | pub(crate) fn writable(repo: &Repo) -> Outcome<()> { | |
| 24 | + | match repo.read_only_reason() { | |
| 25 | + | Some(reason) => Outcome::fail(FailureCode::Forbidden, reason), | |
| 26 | + | None => Outcome::Ok(()), | |
| 27 | + | } | |
| 28 | + | } | |
| 29 | + | ||
| 30 | + | /// `Ok` unless `repo` is archived. For what stays g1t's own on a mirror: | |
| 31 | + | /// its settings and rules, and the statuses and checks reported on its | |
| 32 | + | /// commits (CI failover reports them). | |
| 33 | + | pub(crate) fn not_archived(repo: &Repo) -> Outcome<()> { | |
| 22 | 34 | if repo.archived() { | |
| 23 | 35 | Outcome::fail(FailureCode::Forbidden, archived_message(&repo.namespace, &repo.name)) | |
| 24 | 36 | } else { | |
| ⋯ | |||
| 261 | 273 | topics: Vec::new(), | |
| 262 | 274 | website: None, | |
| 263 | 275 | archived_at: archived_at.map(str::to_owned), | |
| 276 | + | mirror: None, | |
| 264 | 277 | } | |
| 265 | 278 | } | |
| 266 | 279 | ||
| 424 | 424 | let Some(actor) = viewer else { | |
| 425 | 425 | return Ok(Outcome::fail(FailureCode::Unauthenticated, "Sign in first.")); | |
| 426 | 426 | }; | |
| 427 | − | check!(crate::retired::writable(&repo)); | |
| 427 | + | check!(crate::retired::not_archived(&repo)); | |
| 428 | 428 | if !actor.verified { | |
| 429 | 429 | return Ok(Outcome::fail(FailureCode::Forbidden, crate::UNVERIFIED)); | |
| 430 | 430 | } |
| 114 | 114 | Outcome::Ok(repo) => repo, | |
| 115 | 115 | Outcome::Fail(failure) => return Ok(Outcome::Fail(failure)), | |
| 116 | 116 | }; | |
| 117 | − | if let Outcome::Fail(failure) = crate::retired::writable(&repo) { | |
| 117 | + | if let Outcome::Fail(failure) = crate::retired::not_archived(&repo) { | |
| 118 | 118 | return Ok(Outcome::Fail(failure)); | |
| 119 | 119 | } | |
| 120 | 120 | if !a.actor.verified { |