pr_01m47d15m3e54sn21z27rpy5n9/crates/runner/src/actions/blobs.rs
| 1 | //! Artifacts and the cache: `actions/upload-artifact`, |
| 2 | //! `actions/download-artifact` and `actions/cache`, done natively against |
| 3 | //! g1t, which keeps them in R2. Artifacts belong to the run; cache entries |
| 4 | //! to the repository, found by exact key or by the newest under a |
| 5 | //! `restore-keys` prefix. |
| 6 | |
| 7 | use std::collections::BTreeMap; |
| 8 | use std::io::Read; |
| 9 | use std::path::Path; |
| 10 | use std::process::Command; |
| 11 | use std::time::Duration; |
| 12 | |
| 13 | use super::process::{self, Commands, Ended}; |
| 14 | use super::{Job, Post, PostRun}; |
| 15 | |
| 16 | /// The largest upload g1t takes, as the platform limits a request. |
| 17 | const MAX_UPLOAD: u64 = 60 * 1024 * 1024; |
| 18 | |
| 19 | fn lines(text: &str) -> Vec<String> { |
| 20 | text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect() |
| 21 | } |
| 22 | |
| 23 | fn quote(text: &str) -> String { |
| 24 | format!("'{}'", text.replace('\'', "'\\''")) |
| 25 | } |
| 26 | |
| 27 | fn safe_name(name: &str) -> bool { |
| 28 | !name.is_empty() && name.len() <= 200 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ' ')) && !name.starts_with('.') |
| 29 | } |
| 30 | |
| 31 | impl Job { |
| 32 | fn url(&self, rest: &str) -> String { |
| 33 | format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job) |
| 34 | } |
| 35 | |
| 36 | fn auth(&self) -> String { |
| 37 | format!("Bearer {}", self.log.api.token) |
| 38 | } |
| 39 | |
| 40 | /// Runs a shell line, logging its output; whether it succeeded. |
| 41 | fn shell(&mut self, script: &str) -> bool { |
| 42 | let mut command = Command::new("bash"); |
| 43 | command.args(["-c", script]).current_dir(&self.workspace); |
| 44 | let mut commands = Commands::default(); |
| 45 | matches!(process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands), Ok(Ended::Exited(0))) |
| 46 | } |
| 47 | |
| 48 | fn upload(&mut self, rest: &str, file: &Path) -> Result<u64, String> { |
| 49 | let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len(); |
| 50 | if size > MAX_UPLOAD { |
| 51 | return Err(format!("it is {} MB, more than g1t takes at once ({} MB)", size / 1_048_576, MAX_UPLOAD / 1_048_576)); |
| 52 | } |
| 53 | let bytes = std::fs::read(file).map_err(|e| e.to_string())?; |
| 54 | ureq::put(&self.url(rest)) |
| 55 | .set("authorization", &self.auth()) |
| 56 | .set("content-type", "application/gzip") |
| 57 | .timeout(Duration::from_secs(600)) |
| 58 | .send_bytes(&bytes) |
| 59 | .map_err(|e| e.to_string())?; |
| 60 | Ok(size) |
| 61 | } |
| 62 | |
| 63 | /// Downloads into `file`; `Ok(None)` when there is nothing there. |
| 64 | fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> { |
| 65 | let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).call() { |
| 66 | Ok(response) => response, |
| 67 | Err(ureq::Error::Status(404, _)) => return Ok(None), |
| 68 | Err(error) => return Err(error.to_string()), |
| 69 | }; |
| 70 | let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default(); |
| 71 | let mut bytes = Vec::new(); |
| 72 | response.into_reader().take(MAX_UPLOAD * 2).read_to_end(&mut bytes).map_err(|e| e.to_string())?; |
| 73 | std::fs::write(file, bytes).map_err(|e| e.to_string())?; |
| 74 | Ok(Some(matched)) |
| 75 | } |
| 76 | |
| 77 | /// `actions/upload-artifact`. |
| 78 | pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 79 | let name = with.get("name").filter(|n| !n.is_empty()).cloned().unwrap_or_else(|| "artifact".into()); |
| 80 | if !safe_name(&name) { |
| 81 | self.log.line(&format!("##[error]`{name}` is not an artifact name g1t takes: letters, digits, spaces, `-`, `_` and `.`.")); |
| 82 | return (false, BTreeMap::new()); |
| 83 | } |
| 84 | let paths = lines(with.get("path").map(String::as_str).unwrap_or_default()); |
| 85 | let missing = with.get("if-no-files-found").map(String::as_str).unwrap_or("warn").to_owned(); |
| 86 | let archive = self.temp.join(format!("artifact-{name}.tgz")); |
| 87 | // As on GitHub: one folder uploads its contents; otherwise paths |
| 88 | // are kept relative to the workspace. |
| 89 | let single_dir = paths.len() == 1 && self.workspace.join(&paths[0]).is_dir(); |
| 90 | let script = if single_dir { |
| 91 | format!("tar -czf {} -C {} .", quote(&archive.display().to_string()), quote(&paths[0])) |
| 92 | } else { |
| 93 | let patterns: Vec<String> = paths.iter().filter(|p| !p.starts_with('!')).map(|p| p.replace('\'', "")).collect(); |
| 94 | format!( |
| 95 | "shopt -s globstar nullglob dotglob; files=( {} ); if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; tar -czf {} -- \"${{files[@]}}\"", |
| 96 | patterns.join(" "), |
| 97 | quote(&archive.display().to_string()) |
| 98 | ) |
| 99 | }; |
| 100 | let mut command = Command::new("bash"); |
| 101 | command.args(["-c", &script]).current_dir(&self.workspace); |
| 102 | let mut commands = Commands::default(); |
| 103 | match process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands) { |
| 104 | Ok(Ended::Exited(0)) => {} |
| 105 | Ok(Ended::Exited(3)) => { |
| 106 | let message = format!("No files were found at {}.", paths.join(", ")); |
| 107 | return match missing.as_str() { |
| 108 | "error" => { |
| 109 | self.log.line(&format!("##[error]{message}")); |
| 110 | (false, BTreeMap::new()) |
| 111 | } |
| 112 | "ignore" => (true, BTreeMap::new()), |
| 113 | _ => { |
| 114 | self.log.line(&format!("##[warning]{message} Nothing was uploaded.")); |
| 115 | (true, BTreeMap::new()) |
| 116 | } |
| 117 | }; |
| 118 | } |
| 119 | _ => { |
| 120 | self.log.line("##[error]The files could not be packed."); |
| 121 | return (false, BTreeMap::new()); |
| 122 | } |
| 123 | } |
| 124 | match self.upload(&format!("artifacts/{name}"), &archive) { |
| 125 | Ok(size) => { |
| 126 | self.log.line(&format!("Uploaded artifact {name} ({} KB). It is kept with the run for 14 days.", size.div_ceil(1024))); |
| 127 | let mut outputs = BTreeMap::new(); |
| 128 | outputs.insert("artifact-id".into(), name.clone()); |
| 129 | (true, outputs) |
| 130 | } |
| 131 | Err(error) => { |
| 132 | self.log.line(&format!("##[error]The artifact could not be uploaded: {error}")); |
| 133 | (false, BTreeMap::new()) |
| 134 | } |
| 135 | } |
| 136 | } |
| 137 | |
| 138 | /// `actions/download-artifact`: one by name, or every artifact of the |
| 139 | /// run, each into a folder of its name. |
| 140 | pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 141 | let dest = with.get("path").filter(|p| !p.is_empty()).map_or(self.workspace.clone(), |p| self.workspace.join(p)); |
| 142 | let names: Vec<String> = match with.get("name").filter(|n| !n.is_empty()) { |
| 143 | Some(name) => vec![name.clone()], |
| 144 | None => { |
| 145 | let listed = ureq::get(&self.url("artifacts")).set("authorization", &self.auth()).call().ok().and_then(|r| r.into_json::<Vec<serde_json::Value>>().ok()).unwrap_or_default(); |
| 146 | listed.iter().filter_map(|a| a["name"].as_str().map(str::to_owned)).collect() |
| 147 | } |
| 148 | }; |
| 149 | let merge = with.get("merge-multiple").is_some_and(|m| m == "true"); |
| 150 | let single = with.get("name").is_some_and(|n| !n.is_empty()); |
| 151 | for name in &names { |
| 152 | let archive = self.temp.join(format!("download-{name}.tgz")); |
| 153 | match self.download(&format!("artifacts/{name}"), &archive) { |
| 154 | Ok(Some(_)) => {} |
| 155 | Ok(None) => { |
| 156 | self.log.line(&format!("##[error]This run has no artifact called {name}.")); |
| 157 | return (false, BTreeMap::new()); |
| 158 | } |
| 159 | Err(error) => { |
| 160 | self.log.line(&format!("##[error]The artifact {name} could not be downloaded: {error}")); |
| 161 | return (false, BTreeMap::new()); |
| 162 | } |
| 163 | } |
| 164 | let target = if single || merge { dest.clone() } else { dest.join(name) }; |
| 165 | let _ = std::fs::create_dir_all(&target); |
| 166 | if !self.shell(&format!("tar -xzf {} -C {}", quote(&archive.display().to_string()), quote(&target.display().to_string()))) { |
| 167 | return (false, BTreeMap::new()); |
| 168 | } |
| 169 | self.log.line(&format!("Downloaded artifact {name} into {}", target.display())); |
| 170 | } |
| 171 | let mut outputs = BTreeMap::new(); |
| 172 | outputs.insert("download-path".into(), dest.display().to_string()); |
| 173 | (true, outputs) |
| 174 | } |
| 175 | |
| 176 | fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> { |
| 177 | let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into()); |
| 178 | lines(with.get("path").map(String::as_str).unwrap_or_default()) |
| 179 | .into_iter() |
| 180 | .map(|p| { |
| 181 | let p = if let Some(rest) = p.strip_prefix("~/") { format!("{home}/{rest}") } else { p }; |
| 182 | if p.starts_with('/') { p } else { self.workspace.join(p).display().to_string() } |
| 183 | }) |
| 184 | .collect() |
| 185 | } |
| 186 | |
| 187 | /// `actions/cache` and `actions/cache/restore`: restores what it can, |
| 188 | /// and for `actions/cache`, saves at the end of the job on a miss. |
| 189 | pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) { |
| 190 | let key = with.get("key").cloned().unwrap_or_default(); |
| 191 | if key.is_empty() { |
| 192 | self.log.line("##[error]The cache needs a `key`."); |
| 193 | return (false, BTreeMap::new()); |
| 194 | } |
| 195 | let paths = self.cache_paths(with); |
| 196 | let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default()); |
| 197 | let query = format!( |
| 198 | "cache?key={}&restore={}", |
| 199 | urlencode(&key), |
| 200 | urlencode(&restore.join("\n")) |
| 201 | ); |
| 202 | let archive = self.temp.join("cache-restore.tgz"); |
| 203 | let mut outputs = BTreeMap::new(); |
| 204 | let exact = match self.download(&query, &archive) { |
| 205 | Ok(Some(matched)) => { |
| 206 | let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true"); |
| 207 | if !lookup_only && !self.shell(&format!("tar -xzPf {}", quote(&archive.display().to_string()))) { |
| 208 | self.log.line("##[warning]The cache was found but could not be unpacked."); |
| 209 | } |
| 210 | self.log.line(&format!("Cache restored from key: {matched}")); |
| 211 | outputs.insert("cache-matched-key".into(), matched.clone()); |
| 212 | matched == key |
| 213 | } |
| 214 | Ok(None) => { |
| 215 | self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", "))); |
| 216 | if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") { |
| 217 | self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set."); |
| 218 | return (false, outputs); |
| 219 | } |
| 220 | false |
| 221 | } |
| 222 | Err(error) => { |
| 223 | self.log.line(&format!("##[warning]The cache could not be read: {error}")); |
| 224 | false |
| 225 | } |
| 226 | }; |
| 227 | outputs.insert("cache-hit".into(), exact.to_string()); |
| 228 | outputs.insert("cache-primary-key".into(), key.clone()); |
| 229 | if save_after && !exact { |
| 230 | self.posts.push(Post { |
| 231 | name: format!("Post {title}"), |
| 232 | condition: "success()".into(), |
| 233 | env: BTreeMap::new(), |
| 234 | run: PostRun::CacheSave { key, paths }, |
| 235 | }); |
| 236 | } |
| 237 | (true, outputs) |
| 238 | } |
| 239 | |
| 240 | /// Saves paths under a key, unless the key is taken. |
| 241 | pub(crate) fn cache_save(&mut self, key: &str, paths: &[String]) -> bool { |
| 242 | if paths.is_empty() { |
| 243 | self.log.line("##[warning]Nothing to cache: no `path`."); |
| 244 | return true; |
| 245 | } |
| 246 | let archive = self.temp.join("cache-save.tgz"); |
| 247 | let list: Vec<String> = paths.iter().filter(|p| Path::new(p).exists()).map(|p| quote(p)).collect(); |
| 248 | if list.is_empty() { |
| 249 | self.log.line("##[warning]None of the cache's paths exist; nothing was saved."); |
| 250 | return true; |
| 251 | } |
| 252 | if !self.shell(&format!("tar -czPf {} {}", quote(&archive.display().to_string()), list.join(" "))) { |
| 253 | self.log.line("##[warning]The cache could not be packed; nothing was saved."); |
| 254 | return true; |
| 255 | } |
| 256 | match self.upload(&format!("cache?key={}", urlencode(key)), &archive) { |
| 257 | Ok(size) => self.log.line(&format!("Cache saved with key: {key} ({} KB)", size.div_ceil(1024))), |
| 258 | // A cache that cannot be saved does not fail the job, as on GitHub. |
| 259 | Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")), |
| 260 | } |
| 261 | true |
| 262 | } |
| 263 | |
| 264 | /// `actions/cache/save`. |
| 265 | pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 266 | let key = with.get("key").cloned().unwrap_or_default(); |
| 267 | let paths = self.cache_paths(with); |
| 268 | (self.cache_save(&key, &paths), BTreeMap::new()) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | fn urlencode(text: &str) -> String { |
| 273 | let mut out = String::new(); |
| 274 | for byte in text.bytes() { |
| 275 | if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { |
| 276 | out.push(byte as char); |
| 277 | } else { |
| 278 | out.push_str(&format!("%{byte:02X}")); |
| 279 | } |
| 280 | } |
| 281 | out |
| 282 | } |
| 283 | |
| 284 | #[cfg(test)] |
| 285 | mod tests { |
| 286 | use super::*; |
| 287 | |
| 288 | #[test] |
| 289 | fn keys_are_encoded_and_names_checked() { |
| 290 | assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202"); |
| 291 | assert!(safe_name("coverage report")); |
| 292 | assert!(!safe_name("../etc")); |
| 293 | assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]); |
| 294 | } |
| 295 | } |