g1t/crates/runner/src/actions/blobs.rs
| 1 | //! Artifacts and the cache: `actions/upload-artifact`, |
| 2 | //! `actions/download-artifact` and `actions/cache`, done natively against |
| 3 | //! g1t. Artifacts belong to the run; cache entries to the repository, found |
| 4 | //! by exact key or by the newest under a `restore-keys` prefix. |
| 5 | //! |
| 6 | //! A cache entry is a tar archive, compressed with zstd where the machine |
| 7 | //! has it (gzip otherwise), of up to 2 GB: uploaded in parts, and |
| 8 | //! downloaded straight to a file. Its `path` takes globs (`**` included) |
| 9 | //! and `!` patterns that leave paths out, as `actions/cache` does. |
| 10 | |
| 11 | use std::collections::BTreeMap; |
| 12 | use std::io::Read; |
| 13 | use std::path::Path; |
| 14 | use std::process::Command; |
| 15 | use std::time::{Duration, Instant}; |
| 16 | |
| 17 | use super::process::{self, Commands, Ended}; |
| 18 | use super::{Job, Post, PostRun}; |
| 19 | |
| 20 | /// The largest artifact g1t takes at once, as the platform limits a request. |
| 21 | const MAX_UPLOAD: u64 = 60 * 1024 * 1024; |
| 22 | /// The largest cache entry, compressed (`g1t_contracts::actions::CACHE_MAX_ENTRY_BYTES`). |
| 23 | const MAX_CACHE_ENTRY: u64 = 2 * 1024 * 1024 * 1024; |
| 24 | |
| 25 | fn lines(text: &str) -> Vec<String> { |
| 26 | text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect() |
| 27 | } |
| 28 | |
| 29 | fn quote(text: &str) -> String { |
| 30 | format!("'{}'", text.replace('\'', "'\\''")) |
| 31 | } |
| 32 | |
| 33 | fn safe_name(name: &str) -> bool { |
| 34 | !name.is_empty() && name.len() <= 200 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ' ')) && !name.starts_with('.') |
| 35 | } |
| 36 | |
| 37 | impl Job { |
| 38 | fn url(&self, rest: &str) -> String { |
| 39 | format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job) |
| 40 | } |
| 41 | |
| 42 | fn auth(&self) -> String { |
| 43 | format!("Bearer {}", self.log.api.token) |
| 44 | } |
| 45 | |
| 46 | /// Runs a shell line, logging its output; whether it succeeded. |
| 47 | fn shell(&mut self, script: &str) -> bool { |
| 48 | let mut command = Command::new("bash"); |
| 49 | command.args(["-c", script]).current_dir(&self.workspace); |
| 50 | let mut commands = Commands::default(); |
| 51 | matches!(process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands), Ok(Ended::Exited(0))) |
| 52 | } |
| 53 | |
| 54 | fn upload(&mut self, rest: &str, file: &Path) -> Result<u64, String> { |
| 55 | let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len(); |
| 56 | if size > MAX_UPLOAD { |
| 57 | return Err(format!("it is {} MB, more than g1t takes at once ({} MB)", size / 1_048_576, MAX_UPLOAD / 1_048_576)); |
| 58 | } |
| 59 | let bytes = std::fs::read(file).map_err(|e| e.to_string())?; |
| 60 | ureq::put(&self.url(rest)) |
| 61 | .set("authorization", &self.auth()) |
| 62 | .set("content-type", "application/gzip") |
| 63 | .timeout(Duration::from_secs(600)) |
| 64 | .send_bytes(&bytes) |
| 65 | .map_err(|e| e.to_string())?; |
| 66 | Ok(size) |
| 67 | } |
| 68 | |
| 69 | /// Downloads into `file`, as it comes; `Ok(None)` when there is |
| 70 | /// nothing there. |
| 71 | fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> { |
| 72 | let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(1800)).call() { |
| 73 | Ok(response) => response, |
| 74 | Err(ureq::Error::Status(404, _)) => return Ok(None), |
| 75 | Err(error) => return Err(error.to_string()), |
| 76 | }; |
| 77 | let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default(); |
| 78 | let mut out = std::fs::File::create(file).map_err(|e| e.to_string())?; |
| 79 | std::io::copy(&mut response.into_reader().take(MAX_CACHE_ENTRY + 1024), &mut out).map_err(|e| e.to_string())?; |
| 80 | Ok(Some(matched)) |
| 81 | } |
| 82 | |
| 83 | /// Saves `file` as the cache entry `key`, in parts. `Ok(false)` when |
| 84 | /// the key is already cached. |
| 85 | fn upload_cache(&mut self, key: &str, file: &Path) -> Result<bool, String> { |
| 86 | let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len(); |
| 87 | if size > MAX_CACHE_ENTRY { |
| 88 | return Err(format!("it is {} MB, more than a cache entry may be ({} MB)", size / 1_048_576, MAX_CACHE_ENTRY / 1_048_576)); |
| 89 | } |
| 90 | let started = match ureq::post(&self.url(&format!("cache/uploads?key={}&size={size}", urlencode(key)))) |
| 91 | .set("authorization", &self.auth()) |
| 92 | .timeout(Duration::from_secs(60)) |
| 93 | .call() |
| 94 | { |
| 95 | Ok(response) => response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?, |
| 96 | Err(ureq::Error::Status(409, _)) => return Ok(false), |
| 97 | Err(ureq::Error::Status(_, response)) => return Err(refusal(response)), |
| 98 | Err(error) => return Err(error.to_string()), |
| 99 | }; |
| 100 | let id = started["id"].as_str().unwrap_or_default().to_owned(); |
| 101 | let upload = started["upload"].as_str().unwrap_or_default().to_owned(); |
| 102 | let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024); |
| 103 | let base = format!("cache/uploads/{}", urlencode(&id)); |
| 104 | let sent = self.send_parts(&base, &upload, file, part_bytes); |
| 105 | let parts = match sent { |
| 106 | Ok(parts) => parts, |
| 107 | Err(error) => { |
| 108 | let _ = ureq::delete(&self.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &self.auth()).call(); |
| 109 | return Err(error); |
| 110 | } |
| 111 | }; |
| 112 | ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload)))) |
| 113 | .set("authorization", &self.auth()) |
| 114 | .timeout(Duration::from_secs(120)) |
| 115 | .send_json(serde_json::json!({ "size": size, "parts": parts })) |
| 116 | .map_err(|e| match e { |
| 117 | ureq::Error::Status(_, response) => refusal(response), |
| 118 | other => other.to_string(), |
| 119 | })?; |
| 120 | Ok(true) |
| 121 | } |
| 122 | |
| 123 | /// Sends `file` in parts of `part_bytes`; each part's number and etag. |
| 124 | fn send_parts(&mut self, base: &str, upload: &str, file: &Path, part_bytes: u64) -> Result<Vec<serde_json::Value>, String> { |
| 125 | let mut reader = std::fs::File::open(file).map_err(|e| e.to_string())?; |
| 126 | let mut parts = Vec::new(); |
| 127 | let mut buffer = vec![0u8; part_bytes as usize]; |
| 128 | for number in 1u32.. { |
| 129 | let mut filled = 0; |
| 130 | while filled < buffer.len() { |
| 131 | let read = reader.read(&mut buffer[filled..]).map_err(|e| e.to_string())?; |
| 132 | if read == 0 { |
| 133 | break; |
| 134 | } |
| 135 | filled += read; |
| 136 | } |
| 137 | if filled == 0 && number > 1 { |
| 138 | break; |
| 139 | } |
| 140 | let url = self.url(&format!("{base}/{number}?upload={}", urlencode(upload))); |
| 141 | // A part that fails is sent again, twice at most. |
| 142 | let mut tries = 0; |
| 143 | let answer = loop { |
| 144 | tries += 1; |
| 145 | match ureq::put(&url).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).send_bytes(&buffer[..filled]) { |
| 146 | Ok(response) => break response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?, |
| 147 | Err(ureq::Error::Status(status, response)) if status < 500 => return Err(refusal(response)), |
| 148 | Err(error) if tries >= 3 => return Err(error.to_string()), |
| 149 | Err(_) => std::thread::sleep(Duration::from_secs(2 * tries)), |
| 150 | } |
| 151 | }; |
| 152 | parts.push(serde_json::json!({ "part": number, "etag": answer["etag"] })); |
| 153 | if filled < buffer.len() { |
| 154 | break; |
| 155 | } |
| 156 | } |
| 157 | Ok(parts) |
| 158 | } |
| 159 | |
| 160 | /// `actions/upload-artifact`. |
| 161 | pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 162 | let name = with.get("name").filter(|n| !n.is_empty()).cloned().unwrap_or_else(|| "artifact".into()); |
| 163 | if !safe_name(&name) { |
| 164 | self.log.line(&format!("##[error]`{name}` is not an artifact name g1t takes: letters, digits, spaces, `-`, `_` and `.`.")); |
| 165 | return (false, BTreeMap::new()); |
| 166 | } |
| 167 | let paths = lines(with.get("path").map(String::as_str).unwrap_or_default()); |
| 168 | let missing = with.get("if-no-files-found").map(String::as_str).unwrap_or("warn").to_owned(); |
| 169 | let archive = self.temp.join(format!("artifact-{name}.tgz")); |
| 170 | // As on GitHub: one folder uploads its contents; otherwise paths |
| 171 | // are kept relative to the workspace. |
| 172 | let single_dir = paths.len() == 1 && self.workspace.join(&paths[0]).is_dir(); |
| 173 | let script = if single_dir { |
| 174 | format!("tar -czf {} -C {} .", quote(&archive.display().to_string()), quote(&paths[0])) |
| 175 | } else { |
| 176 | let patterns: Vec<String> = paths.iter().filter(|p| !p.starts_with('!')).map(|p| p.replace('\'', "")).collect(); |
| 177 | format!( |
| 178 | "shopt -s globstar nullglob dotglob; files=( {} ); if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; tar -czf {} -- \"${{files[@]}}\"", |
| 179 | patterns.join(" "), |
| 180 | quote(&archive.display().to_string()) |
| 181 | ) |
| 182 | }; |
| 183 | let mut command = Command::new("bash"); |
| 184 | command.args(["-c", &script]).current_dir(&self.workspace); |
| 185 | let mut commands = Commands::default(); |
| 186 | match process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands) { |
| 187 | Ok(Ended::Exited(0)) => {} |
| 188 | Ok(Ended::Exited(3)) => { |
| 189 | let message = format!("No files were found at {}.", paths.join(", ")); |
| 190 | return match missing.as_str() { |
| 191 | "error" => { |
| 192 | self.log.line(&format!("##[error]{message}")); |
| 193 | (false, BTreeMap::new()) |
| 194 | } |
| 195 | "ignore" => (true, BTreeMap::new()), |
| 196 | _ => { |
| 197 | self.log.line(&format!("##[warning]{message} Nothing was uploaded.")); |
| 198 | (true, BTreeMap::new()) |
| 199 | } |
| 200 | }; |
| 201 | } |
| 202 | _ => { |
| 203 | self.log.line("##[error]The files could not be packed."); |
| 204 | return (false, BTreeMap::new()); |
| 205 | } |
| 206 | } |
| 207 | match self.upload(&format!("artifacts/{name}"), &archive) { |
| 208 | Ok(size) => { |
| 209 | self.log.line(&format!("Uploaded artifact {name} ({} KB). It is kept with the run for 14 days.", size.div_ceil(1024))); |
| 210 | let mut outputs = BTreeMap::new(); |
| 211 | outputs.insert("artifact-id".into(), name.clone()); |
| 212 | (true, outputs) |
| 213 | } |
| 214 | Err(error) => { |
| 215 | self.log.line(&format!("##[error]The artifact could not be uploaded: {error}")); |
| 216 | (false, BTreeMap::new()) |
| 217 | } |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | /// `actions/download-artifact`: one by name, or every artifact of the |
| 222 | /// run, each into a folder of its name. |
| 223 | pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 224 | let dest = with.get("path").filter(|p| !p.is_empty()).map_or(self.workspace.clone(), |p| self.workspace.join(p)); |
| 225 | let names: Vec<String> = match with.get("name").filter(|n| !n.is_empty()) { |
| 226 | Some(name) => vec![name.clone()], |
| 227 | None => { |
| 228 | let listed = ureq::get(&self.url("artifacts")).set("authorization", &self.auth()).call().ok().and_then(|r| r.into_json::<Vec<serde_json::Value>>().ok()).unwrap_or_default(); |
| 229 | listed.iter().filter_map(|a| a["name"].as_str().map(str::to_owned)).collect() |
| 230 | } |
| 231 | }; |
| 232 | let merge = with.get("merge-multiple").is_some_and(|m| m == "true"); |
| 233 | let single = with.get("name").is_some_and(|n| !n.is_empty()); |
| 234 | for name in &names { |
| 235 | let archive = self.temp.join(format!("download-{name}.tgz")); |
| 236 | match self.download(&format!("artifacts/{name}"), &archive) { |
| 237 | Ok(Some(_)) => {} |
| 238 | Ok(None) => { |
| 239 | self.log.line(&format!("##[error]This run has no artifact called {name}.")); |
| 240 | return (false, BTreeMap::new()); |
| 241 | } |
| 242 | Err(error) => { |
| 243 | self.log.line(&format!("##[error]The artifact {name} could not be downloaded: {error}")); |
| 244 | return (false, BTreeMap::new()); |
| 245 | } |
| 246 | } |
| 247 | let target = if single || merge { dest.clone() } else { dest.join(name) }; |
| 248 | let _ = std::fs::create_dir_all(&target); |
| 249 | if !self.shell(&format!("tar -xzf {} -C {}", quote(&archive.display().to_string()), quote(&target.display().to_string()))) { |
| 250 | return (false, BTreeMap::new()); |
| 251 | } |
| 252 | self.log.line(&format!("Downloaded artifact {name} into {}", target.display())); |
| 253 | } |
| 254 | let mut outputs = BTreeMap::new(); |
| 255 | outputs.insert("download-path".into(), dest.display().to_string()); |
| 256 | (true, outputs) |
| 257 | } |
| 258 | |
| 259 | /// The cache's `path`, each made absolute (`~/` is the home folder, |
| 260 | /// anything else is under the workspace), `!` patterns kept as they |
| 261 | /// came, with their `!`. |
| 262 | fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> { |
| 263 | let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into()); |
| 264 | let workspace = self.workspace.display().to_string(); |
| 265 | cache_patterns(with.get("path").map(String::as_str).unwrap_or_default(), &home, &workspace) |
| 266 | } |
| 267 | |
| 268 | /// `actions/cache` and `actions/cache/restore`: restores what it can, |
| 269 | /// and for `actions/cache`, saves at the end of the job on a miss. |
| 270 | pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) { |
| 271 | let key = with.get("key").cloned().unwrap_or_default(); |
| 272 | if key.is_empty() { |
| 273 | self.log.line("##[error]The cache needs a `key`."); |
| 274 | return (false, BTreeMap::new()); |
| 275 | } |
| 276 | let paths = self.cache_paths(with); |
| 277 | let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default()); |
| 278 | let query = format!( |
| 279 | "cache?key={}&restore={}", |
| 280 | urlencode(&key), |
| 281 | urlencode(&restore.join("\n")) |
| 282 | ); |
| 283 | let archive = self.temp.join("cache-restore.tar"); |
| 284 | let started = Instant::now(); |
| 285 | let mut outputs = BTreeMap::new(); |
| 286 | let exact = match self.download(&query, &archive) { |
| 287 | Ok(Some(matched)) => { |
| 288 | let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true"); |
| 289 | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 290 | // tar finds out from the archive whether it is zstd or gzip. |
| 291 | if !lookup_only && !self.shell(&format!("tar -xPf {}", quote(&archive.display().to_string()))) { |
| 292 | self.log.line("##[warning]The cache was found but could not be unpacked."); |
| 293 | } |
| 294 | let _ = std::fs::remove_file(&archive); |
| 295 | self.log.line(&format!("Cache restored from key: {matched} ({} in {:.1}s)", megabytes(size), started.elapsed().as_secs_f64())); |
| 296 | outputs.insert("cache-matched-key".into(), matched.clone()); |
| 297 | matched == key |
| 298 | } |
| 299 | Ok(None) => { |
| 300 | self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", "))); |
| 301 | if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") { |
| 302 | self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set."); |
| 303 | return (false, outputs); |
| 304 | } |
| 305 | false |
| 306 | } |
| 307 | Err(error) => { |
| 308 | self.log.line(&format!("##[warning]The cache could not be read: {error}")); |
| 309 | false |
| 310 | } |
| 311 | }; |
| 312 | outputs.insert("cache-hit".into(), exact.to_string()); |
| 313 | outputs.insert("cache-primary-key".into(), key.clone()); |
| 314 | if save_after && !exact { |
| 315 | self.posts.push(Post { |
| 316 | name: format!("Post {title}"), |
| 317 | condition: "success()".into(), |
| 318 | env: BTreeMap::new(), |
| 319 | run: PostRun::CacheSave { key, paths }, |
| 320 | }); |
| 321 | } |
| 322 | (true, outputs) |
| 323 | } |
| 324 | |
| 325 | /// Saves paths under a key, unless the key is taken. |
| 326 | pub(crate) fn cache_save(&mut self, key: &str, paths: &[String]) -> bool { |
| 327 | if paths.iter().all(|p| p.starts_with('!')) { |
| 328 | self.log.line("##[warning]Nothing to cache: no `path`."); |
| 329 | return true; |
| 330 | } |
| 331 | let archive = self.temp.join("cache-save.tar"); |
| 332 | let started = Instant::now(); |
| 333 | match self.run_shell(&pack_script(paths, &archive.display().to_string())) { |
| 334 | Some(0) => {} |
| 335 | Some(3) => { |
| 336 | self.log.line("##[warning]None of the cache's paths exist; nothing was saved."); |
| 337 | return true; |
| 338 | } |
| 339 | _ => { |
| 340 | self.log.line("##[warning]The cache could not be packed; nothing was saved."); |
| 341 | return true; |
| 342 | } |
| 343 | } |
| 344 | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 345 | let packed = started.elapsed().as_secs_f64(); |
| 346 | match self.upload_cache(key, &archive) { |
| 347 | Ok(true) => self.log.line(&format!( |
| 348 | "Cache saved with key: {key} ({}, packed in {packed:.1}s, sent in {:.1}s)", |
| 349 | megabytes(size), |
| 350 | started.elapsed().as_secs_f64() - packed |
| 351 | )), |
| 352 | Ok(false) => self.log.line(&format!("Cache not saved: {key} is already cached.")), |
| 353 | // A cache that cannot be saved does not fail the job, as on GitHub. |
| 354 | Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")), |
| 355 | } |
| 356 | let _ = std::fs::remove_file(&archive); |
| 357 | true |
| 358 | } |
| 359 | |
| 360 | /// Runs a shell line, logging its output; its exit code. |
| 361 | fn run_shell(&mut self, script: &str) -> Option<i32> { |
| 362 | let mut command = Command::new("bash"); |
| 363 | command.args(["-c", script]).current_dir(&self.workspace); |
| 364 | let mut commands = Commands::default(); |
| 365 | match process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands) { |
| 366 | Ok(Ended::Exited(code)) => Some(code), |
| 367 | _ => None, |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | /// `actions/cache/save`. |
| 372 | pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 373 | let key = with.get("key").cloned().unwrap_or_default(); |
| 374 | let paths = self.cache_paths(with); |
| 375 | (self.cache_save(&key, &paths), BTreeMap::new()) |
| 376 | } |
| 377 | } |
| 378 | |
| 379 | /// The message of a refused request, from its JSON body. |
| 380 | fn refusal(response: ureq::Response) -> String { |
| 381 | let status = response.status(); |
| 382 | let body: serde_json::Value = response.into_json().unwrap_or_default(); |
| 383 | body["error"]["message"].as_str().map_or_else(|| format!("g1t answered {status}"), str::to_owned) |
| 384 | } |
| 385 | |
| 386 | fn megabytes(bytes: u64) -> String { |
| 387 | if bytes < 1_048_576 { format!("{} KB", bytes.div_ceil(1024)) } else { format!("{:.1} MB", bytes as f64 / 1_048_576.0) } |
| 388 | } |
| 389 | |
| 390 | /// The lines of a cache's `path`, absolute: `~/` is `home`, a relative |
| 391 | /// path is under `workspace`. A `!` pattern keeps its `!`. |
| 392 | fn cache_patterns(path: &str, home: &str, workspace: &str) -> Vec<String> { |
| 393 | lines(path) |
| 394 | .into_iter() |
| 395 | .map(|line| { |
| 396 | let (bang, p) = match line.strip_prefix('!') { |
| 397 | Some(rest) => ("!", rest.trim().to_owned()), |
| 398 | None => ("", line), |
| 399 | }; |
| 400 | let p = if p == "~" { |
| 401 | home.to_owned() |
| 402 | } else if let Some(rest) = p.strip_prefix("~/") { |
| 403 | format!("{home}/{rest}") |
| 404 | } else { |
| 405 | p |
| 406 | }; |
| 407 | let p = if p.starts_with('/') { p } else { format!("{}/{}", workspace.trim_end_matches('/'), p.trim_start_matches("./")) }; |
| 408 | format!("{bang}{}", p.trim_end_matches('/')) |
| 409 | }) |
| 410 | .collect() |
| 411 | } |
| 412 | |
| 413 | /// A path pattern as a word bash expands as a glob: everything but `*`, |
| 414 | /// `?` and `[...]` escaped, so spaces and quotes stay literal. |
| 415 | fn glob_word(pattern: &str) -> String { |
| 416 | let mut out = String::new(); |
| 417 | for c in pattern.chars() { |
| 418 | if c.is_ascii_alphanumeric() || matches!(c, '*' | '?' | '[' | ']' | '/' | '.' | '-' | '_' | '~' | '+' | ',' | '=' | '@' | ':') { |
| 419 | out.push(c); |
| 420 | } else { |
| 421 | out.push('\\'); |
| 422 | out.push(c); |
| 423 | } |
| 424 | } |
| 425 | out |
| 426 | } |
| 427 | |
| 428 | /// The script that packs a cache: its patterns expanded (`**` reaching |
| 429 | /// any depth), `!` patterns left out, into a tar archive compressed with |
| 430 | /// zstd where there is one, else gzip. Exits 3 when nothing matched. |
| 431 | fn pack_script(patterns: &[String], archive: &str) -> String { |
| 432 | let includes: Vec<String> = patterns.iter().filter(|p| !p.starts_with('!')).map(|p| glob_word(p)).collect(); |
| 433 | let excludes: Vec<String> = patterns |
| 434 | .iter() |
| 435 | .filter_map(|p| p.strip_prefix('!')) |
| 436 | // tar's patterns: `*` already crosses `/`, so `**` is the same. |
| 437 | .map(|p| format!("--exclude={}", quote(&p.replace("**", "*")))) |
| 438 | .collect(); |
| 439 | format!( |
| 440 | "set -o pipefail; shopt -s globstar nullglob dotglob; found=( {} ); files=(); \ |
| 441 | for f in \"${{found[@]}}\"; do if [ -e \"$f\" ]; then files+=(\"$f\"); fi; done; \ |
| 442 | if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; \ |
| 443 | if command -v zstd >/dev/null; then compress='zstd -T0 -3'; else compress=gzip; fi; \ |
| 444 | tar -cPf {} -I \"$compress\" {} -- \"${{files[@]}}\"", |
| 445 | includes.join(" "), |
| 446 | quote(archive), |
| 447 | excludes.join(" ") |
| 448 | ) |
| 449 | } |
| 450 | |
| 451 | fn urlencode(text: &str) -> String { |
| 452 | let mut out = String::new(); |
| 453 | for byte in text.bytes() { |
| 454 | if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { |
| 455 | out.push(byte as char); |
| 456 | } else { |
| 457 | out.push_str(&format!("%{byte:02X}")); |
| 458 | } |
| 459 | } |
| 460 | out |
| 461 | } |
| 462 | |
| 463 | #[cfg(test)] |
| 464 | mod tests { |
| 465 | use super::*; |
| 466 | |
| 467 | #[test] |
| 468 | fn cache_paths_take_home_globs_and_exclusions() { |
| 469 | let paths = cache_patterns("~/.cargo/registry/cache\ntarget/*/release/\n!target/**/incremental\n./dist\n/abs/x\n~", "/home/node", "/w/repo/"); |
| 470 | assert_eq!( |
| 471 | paths, |
| 472 | ["/home/node/.cargo/registry/cache", "/w/repo/target/*/release", "!/w/repo/target/**/incremental", "/w/repo/dist", "/abs/x", "/home/node"] |
| 473 | ); |
| 474 | assert_eq!(glob_word("/w/my repo/target/**/*.rlib"), "/w/my\\ repo/target/**/*.rlib"); |
| 475 | assert_eq!(glob_word("/w/a'b"), "/w/a\\'b"); |
| 476 | } |
| 477 | |
| 478 | #[test] |
| 479 | fn packing_expands_globs_and_leaves_exclusions_out() { |
| 480 | let script = pack_script(&["/w/target/*/release".into(), "!/w/target/**/incremental".into()], "/t/c.tar"); |
| 481 | assert!(script.contains("found=( /w/target/*/release )"), "{script}"); |
| 482 | assert!(script.contains("--exclude='/w/target/*/incremental'"), "{script}"); |
| 483 | assert!(script.contains("zstd -T0"), "{script}"); |
| 484 | assert!(script.contains("exit 3"), "{script}"); |
| 485 | } |
| 486 | |
| 487 | /// Packs and unpacks for real, where bash and tar are (not on Windows). |
| 488 | #[test] |
| 489 | #[cfg(unix)] |
| 490 | fn a_packed_cache_unpacks_without_what_was_left_out() { |
| 491 | let dir = std::env::temp_dir().join(format!("g1t-cache-test-{}", std::process::id())); |
| 492 | let _ = std::fs::remove_dir_all(&dir); |
| 493 | for file in ["target/release/deps/a.rlib", "target/release/incremental/x.bin", "target/wasm/release/deps/b.rlib", "src/main.rs"] { |
| 494 | let path = dir.join(file); |
| 495 | std::fs::create_dir_all(path.parent().unwrap()).unwrap(); |
| 496 | std::fs::write(&path, file).unwrap(); |
| 497 | } |
| 498 | let root = dir.display().to_string(); |
| 499 | let patterns = cache_patterns("target/**/deps\ntarget/release/incremental\n!target/**/incremental", "/home/node", &root); |
| 500 | let archive = dir.join("c.tar").display().to_string(); |
| 501 | let status = Command::new("bash").args(["-c", &pack_script(&patterns, &archive)]).status().unwrap(); |
| 502 | assert!(status.success()); |
| 503 | let listed = Command::new("tar").args(["-tPf", &archive]).output().unwrap(); |
| 504 | let listed = String::from_utf8_lossy(&listed.stdout); |
| 505 | assert!(listed.contains("deps/a.rlib") && listed.contains("deps/b.rlib"), "{listed}"); |
| 506 | assert!(!listed.contains("incremental") && !listed.contains("main.rs"), "{listed}"); |
| 507 | let none = Command::new("bash").args(["-c", &pack_script(&[format!("{root}/nothing/*")], &archive)]).status().unwrap(); |
| 508 | assert_eq!(none.code(), Some(3)); |
| 509 | let _ = std::fs::remove_dir_all(&dir); |
| 510 | } |
| 511 | |
| 512 | #[test] |
| 513 | fn keys_are_encoded_and_names_checked() { |
| 514 | assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202"); |
| 515 | assert!(safe_name("coverage report")); |
| 516 | assert!(!safe_name("../etc")); |
| 517 | assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]); |
| 518 | } |
| 519 | } |