| 1 | //! Repository backups: a nightly `git bundle` of every repository whose |
| 2 | //! refs changed, kept outside the git store (docs/ARTIFACTS.md, R11). |
| 3 | //! |
| 4 | //! The repos service decides what is due and keeps the bundles and their |
| 5 | //! manifests (services/repos/src/backups.rs). It cannot run git, so the |
| 6 | //! bundle is cut where git runs: a sandbox the runner starts, which only |
| 7 | //! ever calls out, as a merge check does. |
| 8 | //! |
| 9 | //! 1. Each night the repos service queues the repositories whose refs |
| 10 | //! moved since their last backup. |
| 11 | //! 2. The runner's sweep claims a few at a time (`claim_backups`) and starts |
| 12 | //! a sandbox for each, with the job's id and token and nothing else. |
| 13 | //! 3. The sandbox asks for its job (`POST api.g1t.sh/backups/{job}/spec`): |
| 14 | //! a read-only git credential for the repository, minutes long, and the |
| 15 | //! commits the last bundle ended at. It clones, cuts the bundle, sends |
| 16 | //! it in parts (`PUT .../parts/{n}`), and says what it holds |
| 17 | //! (`POST .../complete`), or why it could not (`POST .../fail`). |
| 18 | //! |
| 19 | //! The job's token, in the `x-g1t-backup-token` header, is the only |
| 20 | //! credential the sandbox holds for g1t; it lasts as long as the job. |
| 21 | //! |
| 22 | //! What the runner and the repos service exchange is camelCase, as between |
| 23 | //! every service. What the sandbox sends and is sent is snake_case: it is |
| 24 | //! the API's. |
| 25 | |
| 26 | use std::collections::BTreeMap; |
| 27 | |
| 28 | use serde::{Deserialize, Serialize}; |
| 29 | |
| 30 | use crate::repos::RepoPath; |
| 31 | |
| 32 | /// A bundle is sent in parts of this size; the last may be smaller. |
| 33 | pub const PART_BYTES: u64 = 32 * 1024 * 1024; |
| 34 | /// The header the sandbox sends its job's token in. |
| 35 | pub const TOKEN_HEADER: &str = "x-g1t-backup-token"; |
| 36 | |
| 37 | /// `claim_backups`: up to `limit` queued backups, so long as no more than |
| 38 | /// `max_running` are then running. Returns `Vec<BackupClaim>`. |
| 39 | #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] |
| 40 | #[serde(rename_all = "camelCase")] |
| 41 | pub struct ClaimBackupsArgs { |
| 42 | pub limit: u32, |
| 43 | pub max_running: u32, |
| 44 | } |
| 45 | |
| 46 | /// One backup to start. |
| 47 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 48 | #[serde(rename_all = "camelCase")] |
| 49 | pub struct BackupClaim { |
| 50 | pub job_id: String, |
| 51 | /// Lets the sandbox, and nothing else, do this job. |
| 52 | pub token: String, |
| 53 | pub repo_id: String, |
| 54 | /// Where the repository is now: for the sandbox's name and the logs. |
| 55 | pub path: RepoPath, |
| 56 | } |
| 57 | |
| 58 | /// What kind of bundle a job cuts. |
| 59 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 60 | #[serde(rename_all = "snake_case")] |
| 61 | pub enum BackupKind { |
| 62 | /// Everything the repository has. |
| 63 | Full, |
| 64 | /// What is new since the last bundle: its prerequisites are the commits |
| 65 | /// the last bundle's refs pointed to. |
| 66 | Incremental, |
| 67 | } |
| 68 | |
| 69 | impl BackupKind { |
| 70 | /// How a bundle's file name says what it is. |
| 71 | pub fn suffix(self) -> &'static str { |
| 72 | match self { |
| 73 | BackupKind::Full => "full", |
| 74 | BackupKind::Incremental => "incr", |
| 75 | } |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | /// `backup_spec`, `backup_part` and the job's other calls: which job, and |
| 80 | /// its token. |
| 81 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 82 | pub struct BackupJobArgs { |
| 83 | pub job_id: String, |
| 84 | pub token: String, |
| 85 | } |
| 86 | |
| 87 | /// The job, as the sandbox is given it. `Outcome<BackupSpec>`. |
| 88 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 89 | pub struct BackupSpec { |
| 90 | pub kind: BackupKind, |
| 91 | /// The repository in the git store, and a read-only credential for it |
| 92 | /// that lasts minutes (sent as `Authorization: Bearer`). |
| 93 | pub remote: String, |
| 94 | pub git_token: String, |
| 95 | /// For an incremental bundle: the commits it may leave out, and every |
| 96 | /// commit they reach. Empty for a full one. |
| 97 | pub prerequisites: Vec<String>, |
| 98 | /// The refs the last bundle held. When the clone has exactly these, |
| 99 | /// nothing has changed and no bundle is cut. |
| 100 | pub previous_refs: BTreeMap<String, String>, |
| 101 | pub part_bytes: u64, |
| 102 | } |
| 103 | |
| 104 | /// A part the repos service has kept: what completing the upload needs. |
| 105 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 106 | pub struct BackupPart { |
| 107 | pub number: u16, |
| 108 | pub etag: String, |
| 109 | } |
| 110 | |
| 111 | /// `backup_complete`: the bundle is cut and sent. `Outcome<bool>`. |
| 112 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 113 | pub struct BackupComplete { |
| 114 | pub job_id: String, |
| 115 | pub token: String, |
| 116 | /// Every ref the bundle holds (`git for-each-ref` of the clone, and |
| 117 | /// `HEAD`), by name. |
| 118 | pub refs: BTreeMap<String, String>, |
| 119 | /// The bundle's size; 0 when there was nothing new to bundle. |
| 120 | pub size: u64, |
| 121 | #[serde(default)] |
| 122 | pub sha256: Option<String>, |
| 123 | #[serde(default)] |
| 124 | pub parts: Vec<BackupPart>, |
| 125 | /// What the clone read from the git store, for its meters. |
| 126 | #[serde(default)] |
| 127 | pub fetched_bytes: u64, |
| 128 | } |
| 129 | |
| 130 | /// `backup_fail`: the job could not be done. `Outcome<bool>`. |
| 131 | #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] |
| 132 | pub struct BackupFail { |
| 133 | pub job_id: String, |
| 134 | pub token: String, |
| 135 | pub error: String, |
| 136 | #[serde(default)] |
| 137 | pub fetched_bytes: u64, |
| 138 | } |