| 1 | //! Artifacts and the cache: `actions/upload-artifact`, its `merge`, |
| 2 | //! `actions/download-artifact` and `actions/cache`, done natively against |
| 3 | //! g1t. Artifacts belong to the run; cache entries to the repository, found |
| 4 | //! by exact key or by the newest under a `restore-keys` prefix. |
| 5 | //! |
| 6 | //! An artifact is a ZIP file, as GitHub's v4 actions make it (zip.rs), of |
| 7 | //! the files its `path` finds (glob.rs): uploaded in parts with its SHA-256, |
| 8 | //! and downloaded straight to a file before it is unpacked. |
| 9 | //! |
| 10 | //! A cache entry is a tar archive, compressed with zstd where the machine |
| 11 | //! has it (gzip otherwise), of up to 2 GB: uploaded in parts, and |
| 12 | //! downloaded straight to a file. Its `path` takes globs (`**` included) |
| 13 | //! and `!` patterns that leave paths out, as `actions/cache` does. Each is |
| 14 | //! saved and found under a version, the hash of its `path` and compression, |
| 15 | //! and g1t keeps it under the ref whose run saved it. |
| 16 | |
| 17 | use std::collections::BTreeMap; |
| 18 | use std::io::{Read, Write}; |
| 19 | use std::path::Path; |
| 20 | use std::process::Command; |
| 21 | use std::time::{Duration, Instant}; |
| 22 | |
| 23 | use sha2::{Digest, Sha256}; |
| 24 | |
| 25 | use super::process::{self, Commands, Ended}; |
| 26 | use super::{Job, Post, PostRun, glob, zip}; |
| 27 | |
| 28 | /// The largest cache entry, compressed (`g1t_contracts::actions::CACHE_MAX_ENTRY_BYTES`). |
| 29 | const MAX_CACHE_ENTRY: u64 = 2 * 1024 * 1024 * 1024; |
| 30 | |
| 31 | /// How often a long transfer says how far it has got. |
| 32 | const PROGRESS_EVERY: Duration = Duration::from_secs(5); |
| 33 | |
| 34 | /// A transfer's progress as GitHub's cache action words it: |
| 35 | /// `Received 46137344 of 523190823 (8.8%), 43.9 MBs/sec`. |
| 36 | fn progress(verb: &str, done: u64, total: Option<u64>, took: Duration) -> String { |
| 37 | let rate = done as f64 / 1_048_576.0 / took.as_secs_f64().max(0.001); |
| 38 | match total.filter(|total| *total > 0) { |
| 39 | Some(total) => format!("{verb} {done} of {total} ({:.1}%), {rate:.1} MBs/sec", done as f64 * 100.0 / total as f64), |
| 40 | None => format!("{verb} {done} bytes, {rate:.1} MBs/sec"), |
| 41 | } |
| 42 | } |
| 43 | |
| 44 | fn lines(text: &str) -> Vec<String> { |
| 45 | text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect() |
| 46 | } |
| 47 | |
| 48 | fn quote(text: &str) -> String { |
| 49 | format!("'{}'", text.replace('\'', "'\\''")) |
| 50 | } |
| 51 | |
| 52 | impl Job { |
| 53 | fn url(&self, rest: &str) -> String { |
| 54 | format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job) |
| 55 | } |
| 56 | |
| 57 | fn auth(&self) -> String { |
| 58 | format!("Bearer {}", self.log.api.token) |
| 59 | } |
| 60 | |
| 61 | /// Runs a shell line, logging its output; whether it succeeded. |
| 62 | fn shell(&mut self, script: &str) -> bool { |
| 63 | let mut command = Command::new("bash"); |
| 64 | command.args(["-c", script]).current_dir(&self.workspace); |
| 65 | let mut commands = Commands::default(); |
| 66 | matches!(process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0))) |
| 67 | } |
| 68 | |
| 69 | /// Downloads into `file`, as it comes; `Ok(None)` when there is |
| 70 | /// nothing there. |
| 71 | fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> { |
| 72 | let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(1800)).call() { |
| 73 | Ok(response) => response, |
| 74 | Err(ureq::Error::Status(404, _)) => return Ok(None), |
| 75 | Err(error) => return Err(error.to_string()), |
| 76 | }; |
| 77 | let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default(); |
| 78 | let total = response.header("content-length").and_then(|n| n.parse::<u64>().ok()); |
| 79 | let mut out = std::io::BufWriter::new(std::fs::File::create(file).map_err(|e| e.to_string())?); |
| 80 | self.receive(&mut response.into_reader().take(MAX_CACHE_ENTRY + 1024), total, |bytes| out.write_all(bytes))?; |
| 81 | out.flush().map_err(|e| e.to_string())?; |
| 82 | Ok(Some(matched)) |
| 83 | } |
| 84 | |
| 85 | /// Reads a download to its end, handing `each` every piece, and says |
| 86 | /// how it is going every few seconds as GitHub's cache and artifact |
| 87 | /// actions do (`Received 46137344 of 523190823 (8.8%), 43.9 MBs/sec`), |
| 88 | /// so a long download is never a silent step. Its total in bytes. |
| 89 | fn receive(&mut self, reader: &mut dyn Read, total: Option<u64>, mut each: impl FnMut(&[u8]) -> std::io::Result<()>) -> Result<u64, String> { |
| 90 | let started = Instant::now(); |
| 91 | let mut told = Instant::now(); |
| 92 | let mut received = 0u64; |
| 93 | let mut buffer = vec![0u8; 256 * 1024]; |
| 94 | loop { |
| 95 | let n = reader.read(&mut buffer).map_err(|e| e.to_string())?; |
| 96 | if n == 0 { |
| 97 | break; |
| 98 | } |
| 99 | each(&buffer[..n]).map_err(|e| e.to_string())?; |
| 100 | received += n as u64; |
| 101 | if told.elapsed() >= PROGRESS_EVERY { |
| 102 | told = Instant::now(); |
| 103 | self.log.line(&progress("Received", received, total, started.elapsed())); |
| 104 | } |
| 105 | self.log.tick(); |
| 106 | } |
| 107 | Ok(received) |
| 108 | } |
| 109 | |
| 110 | /// Saves `file` as the cache entry `key`, in parts. `Ok(false)` when |
| 111 | /// the key is already cached. |
| 112 | fn upload_cache(&mut self, key: &str, version: &str, file: &Path) -> Result<bool, String> { |
| 113 | let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len(); |
| 114 | if size > MAX_CACHE_ENTRY { |
| 115 | return Err(format!("it is {} MB, more than a cache entry may be ({} MB)", size / 1_048_576, MAX_CACHE_ENTRY / 1_048_576)); |
| 116 | } |
| 117 | let started = match ureq::post(&self.url(&format!("cache/uploads?key={}&size={size}&version={}", urlencode(key), urlencode(version)))) |
| 118 | .set("authorization", &self.auth()) |
| 119 | .timeout(Duration::from_secs(60)) |
| 120 | .call() |
| 121 | { |
| 122 | Ok(response) => response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?, |
| 123 | Err(ureq::Error::Status(409, _)) => return Ok(false), |
| 124 | Err(ureq::Error::Status(_, response)) => return Err(refusal(response)), |
| 125 | Err(error) => return Err(error.to_string()), |
| 126 | }; |
| 127 | let id = started["id"].as_str().unwrap_or_default().to_owned(); |
| 128 | let upload = started["upload"].as_str().unwrap_or_default().to_owned(); |
| 129 | let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024); |
| 130 | let base = format!("cache/uploads/{}", urlencode(&id)); |
| 131 | let sent = self.send_parts(&base, &upload, file, part_bytes); |
| 132 | let parts = match sent { |
| 133 | Ok(parts) => parts, |
| 134 | Err(error) => { |
| 135 | let _ = ureq::delete(&self.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &self.auth()).call(); |
| 136 | return Err(error); |
| 137 | } |
| 138 | }; |
| 139 | ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload)))) |
| 140 | .set("authorization", &self.auth()) |
| 141 | .timeout(Duration::from_secs(120)) |
| 142 | .send_json(serde_json::json!({ "size": size, "parts": parts })) |
| 143 | .map_err(|e| match e { |
| 144 | ureq::Error::Status(_, response) => refusal(response), |
| 145 | other => other.to_string(), |
| 146 | })?; |
| 147 | Ok(true) |
| 148 | } |
| 149 | |
| 150 | /// Sends `file` in parts of `part_bytes`; each part's number and etag. |
| 151 | fn send_parts(&mut self, base: &str, upload: &str, file: &Path, part_bytes: u64) -> Result<Vec<serde_json::Value>, String> { |
| 152 | let mut reader = std::fs::File::open(file).map_err(|e| e.to_string())?; |
| 153 | let size = reader.metadata().map(|m| m.len()).unwrap_or(0); |
| 154 | let (started, mut told, mut sent) = (Instant::now(), Instant::now(), 0u64); |
| 155 | let mut parts = Vec::new(); |
| 156 | let mut buffer = vec![0u8; part_bytes as usize]; |
| 157 | for number in 1u32.. { |
| 158 | let mut filled = 0; |
| 159 | while filled < buffer.len() { |
| 160 | let read = reader.read(&mut buffer[filled..]).map_err(|e| e.to_string())?; |
| 161 | if read == 0 { |
| 162 | break; |
| 163 | } |
| 164 | filled += read; |
| 165 | } |
| 166 | if filled == 0 && number > 1 { |
| 167 | break; |
| 168 | } |
| 169 | let url = self.url(&format!("{base}/{number}?upload={}", urlencode(upload))); |
| 170 | // A part that fails is sent again, twice at most. |
| 171 | let mut tries = 0; |
| 172 | let answer = loop { |
| 173 | tries += 1; |
| 174 | match ureq::put(&url).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).send_bytes(&buffer[..filled]) { |
| 175 | Ok(response) => break response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?, |
| 176 | Err(ureq::Error::Status(status, response)) if status < 500 => return Err(refusal(response)), |
| 177 | Err(error) if tries >= 3 => return Err(error.to_string()), |
| 178 | Err(_) => std::thread::sleep(Duration::from_secs(2 * tries)), |
| 179 | } |
| 180 | }; |
| 181 | parts.push(serde_json::json!({ "part": number, "etag": answer["etag"] })); |
| 182 | sent += filled as u64; |
| 183 | if filled < buffer.len() { |
| 184 | break; |
| 185 | } |
| 186 | // More to send: say how far it has got, as a download does. |
| 187 | if told.elapsed() >= PROGRESS_EVERY { |
| 188 | told = Instant::now(); |
| 189 | self.log.line(&progress("Sent", sent, Some(size), started.elapsed())); |
| 190 | } |
| 191 | self.log.tick(); |
| 192 | } |
| 193 | Ok(parts) |
| 194 | } |
| 195 | |
| 196 | /// A value of the `github` context, as text. |
| 197 | fn github_value(&self, key: &str) -> String { |
| 198 | match self.contexts.get("github").and_then(|github| github.get(key)) { |
| 199 | Some(serde_json::Value::String(text)) => text.clone(), |
| 200 | Some(serde_json::Value::Null) | None => String::new(), |
| 201 | Some(other) => other.to_string(), |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | fn home(&self) -> String { |
| 206 | self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into()) |
| 207 | } |
| 208 | |
| 209 | /// `actions/upload-artifact`: the files `path` names, packed into one |
| 210 | /// ZIP and uploaded in parts. |
| 211 | pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 212 | let name = input(with, "name").unwrap_or("artifact").to_owned(); |
| 213 | let missing = input(with, "if-no-files-found").unwrap_or("warn").to_ascii_lowercase(); |
| 214 | let checked = check_name(&name) |
| 215 | .and_then(|()| keep(with, true)) |
| 216 | .and_then(|keep| if matches!(missing.as_str(), "warn" | "error" | "ignore") { Ok(keep) } else { Err(format!("`if-no-files-found` is `{missing}`; it takes warn, error or ignore.")) }); |
| 217 | let keep = match checked { |
| 218 | Ok(keep) => keep, |
| 219 | Err(message) => { |
| 220 | self.log.line(&format!("##[error]{message}")); |
| 221 | return (false, BTreeMap::new()); |
| 222 | } |
| 223 | }; |
| 224 | let paths = lines(input(with, "path").unwrap_or_default()); |
| 225 | let found = glob::find(&paths, &self.workspace.display().to_string(), &self.home(), keep.hidden); |
| 226 | if found.files.is_empty() { |
| 227 | let message = format!("No files were found with the provided path: {}. No artifacts will be uploaded.", paths.join(", ")); |
| 228 | return match missing.as_str() { |
| 229 | "error" => { |
| 230 | self.log.line(&format!("##[error]{message}")); |
| 231 | (false, BTreeMap::new()) |
| 232 | } |
| 233 | "ignore" => { |
| 234 | self.log.line(&message); |
| 235 | (true, BTreeMap::new()) |
| 236 | } |
| 237 | _ => { |
| 238 | self.log.line(&format!("##[warning]{message}")); |
| 239 | (true, BTreeMap::new()) |
| 240 | } |
| 241 | }; |
| 242 | } |
| 243 | self.log.line(&format!("Found {} with the provided path, stored relative to {}.", count(found.files.len(), "file"), found.root)); |
| 244 | self.send_artifact(&name, &found.files, &keep) |
| 245 | } |
| 246 | |
| 247 | /// Packs `files` into a ZIP and uploads it as the artifact `name`; |
| 248 | /// whether it worked, and the outputs `upload-artifact` sets. |
| 249 | fn send_artifact(&mut self, name: &str, files: &[(String, std::path::PathBuf)], keep: &Keep) -> (bool, BTreeMap<String, String>) { |
| 250 | let archive = self.temp.join(format!("artifact-{}.zip", super::rand_id())); |
| 251 | let sent = zip::write(&archive, files, keep.level) |
| 252 | .map_err(|e| format!("The files could not be packed: {e}")) |
| 253 | .and_then(|packed| self.post_artifact(name, &archive, keep).map(|done| (packed, done))); |
| 254 | let _ = std::fs::remove_file(&archive); |
| 255 | let (packed, done) = match sent { |
| 256 | Ok(sent) => sent, |
| 257 | Err(error) => { |
| 258 | self.log.line(&format!("##[error]{error}")); |
| 259 | return (false, BTreeMap::new()); |
| 260 | } |
| 261 | }; |
| 262 | let kept = done.retention.map(|days| format!(" It is kept for {}.", count(days as usize, "day"))).unwrap_or_default(); |
| 263 | self.log.line(&format!("Uploaded artifact {name} ({}, {}).{kept}", megabytes(packed.size), count(packed.files, "file"))); |
| 264 | let url = format!( |
| 265 | "{}/{}/actions/runs/{}/artifacts/{}", |
| 266 | self.github_value("server_url").trim_end_matches('/'), |
| 267 | self.github_value("repository"), |
| 268 | self.github_value("run_id"), |
| 269 | done.id |
| 270 | ); |
| 271 | self.log.line(&format!("Artifact download URL: {url}")); |
| 272 | let mut outputs = BTreeMap::new(); |
| 273 | outputs.insert("artifact-id".into(), done.id.to_string()); |
| 274 | outputs.insert("artifact-url".into(), url); |
| 275 | outputs.insert("artifact-digest".into(), done.digest); |
| 276 | (true, outputs) |
| 277 | } |
| 278 | |
| 279 | /// Uploads the ZIP file `archive` as the artifact `name`, in parts, |
| 280 | /// giving the upload up when a part or the end fails. |
| 281 | fn post_artifact(&mut self, name: &str, archive: &Path, keep: &Keep) -> Result<Sent, String> { |
| 282 | let size = std::fs::metadata(archive).map_err(|e| e.to_string())?.len(); |
| 283 | let digest = sha256_file(archive).map_err(|e| e.to_string())?; |
| 284 | let query = format!( |
| 285 | "artifacts/uploads?name={}&size={size}&retention_days={}&overwrite={}&format=zip", |
| 286 | urlencode(name), |
| 287 | keep.retention, |
| 288 | keep.overwrite |
| 289 | ); |
| 290 | let failed = |e: ureq::Error| match e { |
| 291 | ureq::Error::Status(_, response) => format!("The artifact could not be uploaded: {}", refusal(response)), |
| 292 | other => format!("The artifact could not be uploaded: {other}"), |
| 293 | }; |
| 294 | let started = ureq::post(&self.url(&query)) |
| 295 | .set("authorization", &self.auth()) |
| 296 | .timeout(Duration::from_secs(60)) |
| 297 | .call() |
| 298 | .map_err(failed)? |
| 299 | .into_json::<serde_json::Value>() |
| 300 | .map_err(|e| e.to_string())?; |
| 301 | let id = number(&started["id"]).ok_or("g1t did not say which artifact the upload is")?; |
| 302 | let upload = started["upload"].as_str().unwrap_or_default().to_owned(); |
| 303 | let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024); |
| 304 | let retention = started["retention_days"].as_u64(); |
| 305 | if let Some(applied) = retention |
| 306 | && keep.retention != 0 |
| 307 | && applied != u64::from(keep.retention) |
| 308 | { |
| 309 | self.log.line(&format!( |
| 310 | "##[notice]The artifact is kept for {}, not the {} asked for: the most this repository keeps artifacts.", |
| 311 | count(applied as usize, "day"), |
| 312 | keep.retention |
| 313 | )); |
| 314 | } |
| 315 | let base = format!("artifacts/uploads/{id}"); |
| 316 | let give_up = |job: &Job| { |
| 317 | let _ = ureq::delete(&job.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &job.auth()).call(); |
| 318 | }; |
| 319 | let parts = match self.send_parts(&base, &upload, archive, part_bytes) { |
| 320 | Ok(parts) => parts, |
| 321 | Err(error) => { |
| 322 | give_up(self); |
| 323 | return Err(format!("The artifact could not be uploaded: {error}")); |
| 324 | } |
| 325 | }; |
| 326 | let done = ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload)))) |
| 327 | .set("authorization", &self.auth()) |
| 328 | .timeout(Duration::from_secs(120)) |
| 329 | .send_json(serde_json::json!({ "size": size, "parts": parts, "digest": format!("sha256:{digest}") })); |
| 330 | let done = match done { |
| 331 | Ok(response) => response.into_json::<serde_json::Value>().unwrap_or_default(), |
| 332 | Err(error) => { |
| 333 | give_up(self); |
| 334 | return Err(failed(error)); |
| 335 | } |
| 336 | }; |
| 337 | Ok(Sent { id: number(&done["id"]).unwrap_or(id), digest, retention: retention.map(|d| d as u32).or((keep.retention != 0).then_some(keep.retention)) }) |
| 338 | } |
| 339 | |
| 340 | /// The artifacts of this run, or of the run `run_id` of this |
| 341 | /// repository; one per name, the newest. |
| 342 | fn list_artifacts(&mut self, run_id: Option<&str>) -> Result<Vec<Listed>, String> { |
| 343 | let rest = match run_id { |
| 344 | Some(run) => format!("artifacts?run_id={}", urlencode(run)), |
| 345 | None => "artifacts".to_owned(), |
| 346 | }; |
| 347 | let listed = ureq::get(&self.url(&rest)) |
| 348 | .set("authorization", &self.auth()) |
| 349 | .timeout(Duration::from_secs(60)) |
| 350 | .call() |
| 351 | .map_err(|e| match e { |
| 352 | ureq::Error::Status(_, response) => refusal(response), |
| 353 | other => other.to_string(), |
| 354 | })? |
| 355 | .into_json::<Vec<serde_json::Value>>() |
| 356 | .map_err(|e| e.to_string())?; |
| 357 | Ok(newest(listed.iter().filter_map(Listed::from_json).collect())) |
| 358 | } |
| 359 | |
| 360 | /// Downloads an artifact to a file, as it comes, and unpacks it into |
| 361 | /// `into`. |
| 362 | fn fetch_artifact(&mut self, artifact: &Listed, into: &Path) -> Result<(), String> { |
| 363 | let file = self.temp.join(format!("download-{}.zip", super::rand_id())); |
| 364 | let fetched = self.fetch_to(artifact, &file).and_then(|format| { |
| 365 | if format == "tgz" { |
| 366 | std::fs::create_dir_all(into).map_err(|e| e.to_string())?; |
| 367 | let script = format!("tar{} -xzf {} -C {}", tar_local(), quote(&file.display().to_string()), quote(&into.display().to_string())); |
| 368 | if self.shell(&script) { Ok(()) } else { Err("it could not be unpacked".into()) } |
| 369 | } else { |
| 370 | zip::extract(&file, into).map(|_| ()).map_err(|e| format!("it could not be unpacked: {e}")) |
| 371 | } |
| 372 | }); |
| 373 | let _ = std::fs::remove_file(&file); |
| 374 | fetched |
| 375 | } |
| 376 | |
| 377 | /// Downloads an artifact into `file`, checking its digest; how it is |
| 378 | /// packed, `zip` or `tgz`. |
| 379 | fn fetch_to(&mut self, artifact: &Listed, file: &Path) -> Result<String, String> { |
| 380 | let response = match ureq::get(&self.url(&format!("artifacts/{}/download", artifact.id))) |
| 381 | .set("authorization", &self.auth()) |
| 382 | .timeout(Duration::from_secs(4 * 3600)) |
| 383 | .call() |
| 384 | { |
| 385 | Ok(response) => response, |
| 386 | Err(ureq::Error::Status(404, _)) => return Err("it is gone: it expired or was deleted".into()), |
| 387 | Err(ureq::Error::Status(_, response)) => return Err(refusal(response)), |
| 388 | Err(error) => return Err(error.to_string()), |
| 389 | }; |
| 390 | let format = response.header("x-g1t-format").map(str::to_owned).unwrap_or_else(|| artifact.format.clone()); |
| 391 | let total = response.header("content-length").and_then(|n| n.parse::<u64>().ok()).or(Some(artifact.size)); |
| 392 | let mut reader = response.into_reader(); |
| 393 | let mut out = std::io::BufWriter::new(std::fs::File::create(file).map_err(|e| e.to_string())?); |
| 394 | let mut hasher = Sha256::new(); |
| 395 | self.receive(&mut reader, total, |bytes| { |
| 396 | hasher.update(bytes); |
| 397 | out.write_all(bytes) |
| 398 | })?; |
| 399 | out.flush().map_err(|e| e.to_string())?; |
| 400 | let got = hex::encode(hasher.finalize()); |
| 401 | if let Some(expected) = artifact.digest.as_deref().map(|d| d.trim_start_matches("sha256:")) |
| 402 | && !expected.is_empty() |
| 403 | && !expected.eq_ignore_ascii_case(&got) |
| 404 | { |
| 405 | self.log.line(&format!("##[warning]Artifact {} does not match its digest (sha256:{got}, not sha256:{expected}): it may have been changed since it was uploaded.", artifact.name)); |
| 406 | } |
| 407 | Ok(format) |
| 408 | } |
| 409 | |
| 410 | /// Where artifacts are downloaded: `path`, under the workspace unless |
| 411 | /// absolute, `~` the home folder. |
| 412 | fn download_dir(&self, path: Option<&str>) -> std::path::PathBuf { |
| 413 | match path { |
| 414 | None => self.workspace.clone(), |
| 415 | Some(path) => { |
| 416 | let home = self.home(); |
| 417 | let path = if path == "~" { |
| 418 | home |
| 419 | } else if let Some(rest) = path.strip_prefix("~/") { |
| 420 | format!("{}/{rest}", home.trim_end_matches('/')) |
| 421 | } else { |
| 422 | path.to_owned() |
| 423 | }; |
| 424 | self.workspace.join(path) |
| 425 | } |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | /// `actions/download-artifact`: one artifact by name straight into |
| 430 | /// `path`, or the run's artifacts (by `pattern` or `artifact-ids`) each |
| 431 | /// into a folder of its name, or all into `path` with `merge-multiple`. |
| 432 | pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 433 | let fail = |job: &mut Job, message: &str| { |
| 434 | job.log.line(&format!("##[error]{message}")); |
| 435 | (false, BTreeMap::new()) |
| 436 | }; |
| 437 | let name = input(with, "name"); |
| 438 | let ids = match artifact_ids(input(with, "artifact-ids").unwrap_or_default()) { |
| 439 | Ok(ids) => ids, |
| 440 | Err(message) => return fail(self, &message), |
| 441 | }; |
| 442 | if name.is_some() && !ids.is_empty() { |
| 443 | return fail(self, "Inputs 'name' and 'artifact-ids' cannot be used together. Please specify only one."); |
| 444 | } |
| 445 | let merge = match flag(with, "merge-multiple", false) { |
| 446 | Ok(merge) => merge, |
| 447 | Err(message) => return fail(self, &message), |
| 448 | }; |
| 449 | let dest = self.download_dir(input(with, "path")); |
| 450 | // Another run's artifacts, as v4 has it: with a token and a run id. |
| 451 | let run_id = match (input(with, "github-token"), input(with, "run-id")) { |
| 452 | (Some(_), Some(run)) => { |
| 453 | let own = self.github_value("repository"); |
| 454 | let repository = input(with, "repository").unwrap_or(&own); |
| 455 | if !repository.eq_ignore_ascii_case(&own) { |
| 456 | return fail(self, &format!("g1t downloads artifacts from other runs of the same repository only; {repository} is not this run's repository, {own}.")); |
| 457 | } |
| 458 | Some(run.to_owned()) |
| 459 | } |
| 460 | _ => None, |
| 461 | }; |
| 462 | let listed = match self.list_artifacts(run_id.as_deref()) { |
| 463 | Ok(listed) => listed, |
| 464 | Err(error) => return fail(self, &format!("The artifacts could not be listed: {error}")), |
| 465 | }; |
| 466 | let chosen: Vec<Listed> = if let Some(name) = name { |
| 467 | match listed.into_iter().find(|a| a.name == name) { |
| 468 | Some(artifact) => vec![artifact], |
| 469 | None => return fail(self, &format!("Unable to download artifact(s): Artifact not found for name: {name}")), |
| 470 | } |
| 471 | } else if !ids.is_empty() { |
| 472 | let chosen: Vec<Listed> = listed.into_iter().filter(|a| ids.contains(&a.id)).collect(); |
| 473 | if chosen.is_empty() { |
| 474 | return fail(self, "Unable to download artifact(s): None of the provided artifact IDs were found."); |
| 475 | } |
| 476 | if chosen.len() < ids.len() { |
| 477 | self.log.line(&format!("##[warning]Could only find {} of {} artifact IDs.", chosen.len(), ids.len())); |
| 478 | } |
| 479 | chosen |
| 480 | } else { |
| 481 | match input(with, "pattern") { |
| 482 | Some(pattern) => listed.into_iter().filter(|a| glob::matches_part(pattern, &a.name)).collect(), |
| 483 | None => listed, |
| 484 | } |
| 485 | }; |
| 486 | let mut outputs = BTreeMap::new(); |
| 487 | outputs.insert("download-path".into(), dest.display().to_string()); |
| 488 | if chosen.is_empty() { |
| 489 | match input(with, "pattern") { |
| 490 | Some(pattern) => self.log.line(&format!("No artifacts matched the pattern {pattern}; nothing was downloaded.")), |
| 491 | None => self.log.line("The run has no artifacts; nothing was downloaded."), |
| 492 | } |
| 493 | return (true, outputs); |
| 494 | } |
| 495 | let targets = download_targets(&dest, &chosen, name.is_some(), !ids.is_empty(), merge); |
| 496 | for (artifact, target) in chosen.iter().zip(targets) { |
| 497 | let Some(target) = target else { |
| 498 | return fail(self, &format!("The artifact {} cannot be downloaded into a folder of its name.", artifact.name)); |
| 499 | }; |
| 500 | if let Err(error) = self.fetch_artifact(artifact, &target) { |
| 501 | return fail(self, &format!("The artifact {} could not be downloaded: {error}", artifact.name)); |
| 502 | } |
| 503 | self.log.line(&format!("Downloaded artifact {} ({}) into {}", artifact.name, megabytes(artifact.size), target.display())); |
| 504 | } |
| 505 | if chosen.len() > 1 { |
| 506 | self.log.line(&format!("Downloaded {}.", count(chosen.len(), "artifact"))); |
| 507 | } |
| 508 | (true, outputs) |
| 509 | } |
| 510 | |
| 511 | /// `actions/upload-artifact/merge`: the run's artifacts matching |
| 512 | /// `pattern`, downloaded together and uploaded again as one. |
| 513 | pub(crate) fn merge_artifacts(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 514 | let fail = |job: &mut Job, message: &str| { |
| 515 | job.log.line(&format!("##[error]{message}")); |
| 516 | (false, BTreeMap::new()) |
| 517 | }; |
| 518 | let name = input(with, "name").unwrap_or("merged-artifacts").to_owned(); |
| 519 | let pattern = input(with, "pattern").unwrap_or("*").to_owned(); |
| 520 | let options = check_name(&name).and_then(|()| { |
| 521 | let keep = keep(with, false)?; |
| 522 | Ok((keep, flag(with, "separate-directories", false)?, flag(with, "delete-merged", false)?)) |
| 523 | }); |
| 524 | let (keep, separate, delete) = match options { |
| 525 | Ok(options) => options, |
| 526 | Err(message) => return fail(self, &message), |
| 527 | }; |
| 528 | let listed = match self.list_artifacts(None) { |
| 529 | Ok(listed) => listed, |
| 530 | Err(error) => return fail(self, &format!("The artifacts could not be listed: {error}")), |
| 531 | }; |
| 532 | let chosen: Vec<Listed> = listed.into_iter().filter(|a| glob::matches_part(&pattern, &a.name)).collect(); |
| 533 | if chosen.is_empty() { |
| 534 | return fail(self, &format!("No artifacts found matching pattern '{pattern}'.")); |
| 535 | } |
| 536 | self.log.line(&format!("Merging {}: {}", count(chosen.len(), "artifact"), chosen.iter().map(|a| a.name.as_str()).collect::<Vec<_>>().join(", "))); |
| 537 | let folder = self.temp.join(format!("merge-{}", super::rand_id())); |
| 538 | let targets = download_targets(&folder, &chosen, false, false, !separate); |
| 539 | let mut result = None; |
| 540 | for (artifact, target) in chosen.iter().zip(targets) { |
| 541 | let Some(target) = target else { |
| 542 | result = Some(fail(self, &format!("The artifact {} cannot be merged into a folder of its name.", artifact.name))); |
| 543 | break; |
| 544 | }; |
| 545 | if let Err(error) = self.fetch_artifact(artifact, &target) { |
| 546 | result = Some(fail(self, &format!("The artifact {} could not be downloaded: {error}", artifact.name))); |
| 547 | break; |
| 548 | } |
| 549 | } |
| 550 | let result = result.unwrap_or_else(|| { |
| 551 | let root = folder.display().to_string(); |
| 552 | let found = glob::find(std::slice::from_ref(&root), &root, &self.home(), keep.hidden); |
| 553 | if found.files.is_empty() { |
| 554 | fail(self, "The artifacts matched hold no files to merge.") |
| 555 | } else { |
| 556 | self.send_artifact(&name, &found.files, &keep) |
| 557 | } |
| 558 | }); |
| 559 | let _ = std::fs::remove_dir_all(&folder); |
| 560 | if result.0 && delete { |
| 561 | for artifact in &chosen { |
| 562 | match ureq::delete(&self.url(&format!("artifacts/{}", artifact.id))).set("authorization", &self.auth()).timeout(Duration::from_secs(60)).call() { |
| 563 | Ok(_) => self.log.line(&format!("Deleted artifact {}.", artifact.name)), |
| 564 | Err(ureq::Error::Status(_, response)) => self.log.line(&format!("##[warning]Artifact {} could not be deleted: {}", artifact.name, refusal(response))), |
| 565 | Err(error) => self.log.line(&format!("##[warning]Artifact {} could not be deleted: {error}", artifact.name)), |
| 566 | } |
| 567 | } |
| 568 | } |
| 569 | result |
| 570 | } |
| 571 | |
| 572 | /// The cache's `path`, each made absolute (`~/` is the home folder, |
| 573 | /// anything else is under the workspace), `!` patterns kept as they |
| 574 | /// came, with their `!`. |
| 575 | fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> { |
| 576 | let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into()); |
| 577 | let workspace = self.workspace.display().to_string(); |
| 578 | cache_patterns(with.get("path").map(String::as_str).unwrap_or_default(), &home, &workspace) |
| 579 | } |
| 580 | |
| 581 | /// `actions/cache` and `actions/cache/restore`: restores what it can, |
| 582 | /// and for `actions/cache`, saves at the end of the job on a miss. |
| 583 | pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) { |
| 584 | let key = with.get("key").cloned().unwrap_or_default(); |
| 585 | if key.is_empty() { |
| 586 | self.log.line("##[error]The cache needs a `key`."); |
| 587 | return (false, BTreeMap::new()); |
| 588 | } |
| 589 | let paths = self.cache_paths(with); |
| 590 | let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression()); |
| 591 | let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default()); |
| 592 | let query = format!( |
| 593 | "cache?key={}&restore={}&version={}", |
| 594 | urlencode(&key), |
| 595 | urlencode(&restore.join("\n")), |
| 596 | urlencode(&version) |
| 597 | ); |
| 598 | let archive = self.temp.join("cache-restore.tar"); |
| 599 | let started = Instant::now(); |
| 600 | let mut outputs = BTreeMap::new(); |
| 601 | let exact = match self.download(&query, &archive) { |
| 602 | Ok(Some(matched)) => { |
| 603 | let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true"); |
| 604 | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 605 | // tar finds out from the archive whether it is zstd or gzip. |
| 606 | if !lookup_only && !self.shell(&format!("tar{} -xPf {}", tar_local(), quote(&archive.display().to_string()))) { |
| 607 | self.log.line("##[warning]The cache was found but could not be unpacked."); |
| 608 | } |
| 609 | let _ = std::fs::remove_file(&archive); |
| 610 | self.log.line(&format!("Cache restored from key: {matched} ({} in {:.1}s)", megabytes(size), started.elapsed().as_secs_f64())); |
| 611 | outputs.insert("cache-matched-key".into(), matched.clone()); |
| 612 | matched == key |
| 613 | } |
| 614 | Ok(None) => { |
| 615 | self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", "))); |
| 616 | if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") { |
| 617 | self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set."); |
| 618 | return (false, outputs); |
| 619 | } |
| 620 | false |
| 621 | } |
| 622 | Err(error) => { |
| 623 | self.log.line(&format!("##[warning]The cache could not be read: {error}")); |
| 624 | false |
| 625 | } |
| 626 | }; |
| 627 | outputs.insert("cache-hit".into(), exact.to_string()); |
| 628 | outputs.insert("cache-primary-key".into(), key.clone()); |
| 629 | if save_after && !exact { |
| 630 | self.posts.push(Post { |
| 631 | name: format!("Post {title}"), |
| 632 | condition: "success()".into(), |
| 633 | env: BTreeMap::new(), |
| 634 | run: PostRun::CacheSave { key, paths, version }, |
| 635 | }); |
| 636 | } |
| 637 | (true, outputs) |
| 638 | } |
| 639 | |
| 640 | /// Saves paths under a key, unless the key is taken. |
| 641 | pub(crate) fn cache_save(&mut self, key: &str, paths: &[String], version: &str) -> bool { |
| 642 | if paths.iter().all(|p| p.starts_with('!')) { |
| 643 | self.log.line("##[warning]Nothing to cache: no `path`."); |
| 644 | return true; |
| 645 | } |
| 646 | let archive = self.temp.join("cache-save.tar"); |
| 647 | let started = Instant::now(); |
| 648 | match self.run_shell(&pack_script(paths, &archive.display().to_string())) { |
| 649 | Some(0) => {} |
| 650 | Some(3) => { |
| 651 | self.log.line("##[warning]None of the cache's paths exist; nothing was saved."); |
| 652 | return true; |
| 653 | } |
| 654 | _ => { |
| 655 | self.log.line("##[warning]The cache could not be packed; nothing was saved."); |
| 656 | return true; |
| 657 | } |
| 658 | } |
| 659 | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 660 | let packed = started.elapsed().as_secs_f64(); |
| 661 | match self.upload_cache(key, version, &archive) { |
| 662 | Ok(true) => self.log.line(&format!( |
| 663 | "Cache saved with key: {key} ({}, packed in {packed:.1}s, sent in {:.1}s)", |
| 664 | megabytes(size), |
| 665 | started.elapsed().as_secs_f64() - packed |
| 666 | )), |
| 667 | Ok(false) => self.log.line(&format!("Cache not saved: {key} is already cached.")), |
| 668 | // A cache that cannot be saved does not fail the job, as on GitHub. |
| 669 | Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")), |
| 670 | } |
| 671 | let _ = std::fs::remove_file(&archive); |
| 672 | true |
| 673 | } |
| 674 | |
| 675 | /// Runs a shell line, logging its output; its exit code. |
| 676 | fn run_shell(&mut self, script: &str) -> Option<i32> { |
| 677 | let mut command = Command::new("bash"); |
| 678 | command.args(["-c", script]).current_dir(&self.workspace); |
| 679 | let mut commands = Commands::default(); |
| 680 | match process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands) { |
| 681 | Ok(Ended::Exited(code)) => Some(code), |
| 682 | _ => None, |
| 683 | } |
| 684 | } |
| 685 | |
| 686 | /// `actions/cache/save`. |
| 687 | pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 688 | let key = with.get("key").cloned().unwrap_or_default(); |
| 689 | let paths = self.cache_paths(with); |
| 690 | let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression()); |
| 691 | (self.cache_save(&key, &paths, &version), BTreeMap::new()) |
| 692 | } |
| 693 | } |
| 694 | |
| 695 | /// An input, trimmed; `None` when empty. |
| 696 | fn input<'a>(with: &'a BTreeMap<String, String>, key: &str) -> Option<&'a str> { |
| 697 | with.get(key).map(|v| v.trim()).filter(|v| !v.is_empty()) |
| 698 | } |
| 699 | |
| 700 | /// A true-or-false input, as `core.getBooleanInput` reads it. |
| 701 | fn flag(with: &BTreeMap<String, String>, key: &str, default: bool) -> Result<bool, String> { |
| 702 | match input(with, key) { |
| 703 | None => Ok(default), |
| 704 | Some("true" | "True" | "TRUE") => Ok(true), |
| 705 | Some("false" | "False" | "FALSE") => Ok(false), |
| 706 | Some(other) => Err(format!("`{key}` is `{other}`; it takes true or false.")), |
| 707 | } |
| 708 | } |
| 709 | |
| 710 | /// `1 file`, `3 files`. |
| 711 | fn count(n: usize, what: &str) -> String { |
| 712 | if n == 1 { format!("1 {what}") } else { format!("{n} {what}s") } |
| 713 | } |
| 714 | |
| 715 | /// An id that came as a number or as text. |
| 716 | fn number(value: &serde_json::Value) -> Option<u64> { |
| 717 | value.as_u64().or_else(|| value.as_str().and_then(|s| s.parse().ok())) |
| 718 | } |
| 719 | |
| 720 | /// Whether a name is one GitHub takes for an artifact: not empty, at most |
| 721 | /// 256 characters, none of `" : < > | * ? \ /` or a line break. |
| 722 | fn check_name(name: &str) -> Result<(), String> { |
| 723 | if name.is_empty() { |
| 724 | return Err("The artifact needs a name.".into()); |
| 725 | } |
| 726 | if name.chars().count() > 256 { |
| 727 | return Err(format!("The artifact name `{name}` is longer than 256 characters.")); |
| 728 | } |
| 729 | if let Some(bad) = name.chars().find(|c| matches!(c, '"' | ':' | '<' | '>' | '|' | '*' | '?' | '\r' | '\n' | '\\' | '/')) { |
| 730 | let shown = match bad { |
| 731 | '\r' => "a carriage return".to_owned(), |
| 732 | '\n' => "a line break".to_owned(), |
| 733 | c => format!("`{c}`"), |
| 734 | }; |
| 735 | return Err(format!("The artifact name `{name}` is not valid: it contains {shown}. A name may not contain \" : < > | * ? \\ / or line breaks.")); |
| 736 | } |
| 737 | Ok(()) |
| 738 | } |
| 739 | |
| 740 | /// How an artifact is packed and kept, from the inputs `upload-artifact` |
| 741 | /// and its merge share. |
| 742 | #[derive(Debug, PartialEq)] |
| 743 | struct Keep { |
| 744 | /// Days; 0 for the repository's own setting. |
| 745 | retention: u32, |
| 746 | /// 0 (stored) to 9. |
| 747 | level: u32, |
| 748 | overwrite: bool, |
| 749 | /// Whether files and folders whose names start with `.` are taken. |
| 750 | hidden: bool, |
| 751 | } |
| 752 | |
| 753 | fn keep(with: &BTreeMap<String, String>, takes_overwrite: bool) -> Result<Keep, String> { |
| 754 | let retention = match input(with, "retention-days") { |
| 755 | None => 0, |
| 756 | Some(text) => match text.parse::<u32>() { |
| 757 | Ok(days @ 0..=90) => days, |
| 758 | _ => return Err(format!("`retention-days` is `{text}`; it takes a number of days from 1 to 90, or nothing for the repository's setting.")), |
| 759 | }, |
| 760 | }; |
| 761 | let level = match input(with, "compression-level") { |
| 762 | None => 6, |
| 763 | Some(text) => match text.parse::<u32>() { |
| 764 | Ok(level @ 0..=9) => level, |
| 765 | _ => return Err(format!("`compression-level` is `{text}`; it takes 0 (no compression) to 9.")), |
| 766 | }, |
| 767 | }; |
| 768 | Ok(Keep { |
| 769 | retention, |
| 770 | level, |
| 771 | overwrite: takes_overwrite && flag(with, "overwrite", false)?, |
| 772 | hidden: flag(with, "include-hidden-files", false)?, |
| 773 | }) |
| 774 | } |
| 775 | |
| 776 | /// What finishing an upload gave: the artifact's id, the ZIP's SHA-256 in |
| 777 | /// hex, and the days it is kept, when known. |
| 778 | struct Sent { |
| 779 | id: u64, |
| 780 | digest: String, |
| 781 | retention: Option<u32>, |
| 782 | } |
| 783 | |
| 784 | /// An artifact as g1t lists it. |
| 785 | #[derive(Debug, Clone, PartialEq)] |
| 786 | struct Listed { |
| 787 | id: u64, |
| 788 | name: String, |
| 789 | size: u64, |
| 790 | digest: Option<String>, |
| 791 | /// `zip`, or `tgz` for an artifact an older runner stored. |
| 792 | format: String, |
| 793 | } |
| 794 | |
| 795 | impl Listed { |
| 796 | fn from_json(value: &serde_json::Value) -> Option<Listed> { |
| 797 | Some(Listed { |
| 798 | id: number(&value["id"])?, |
| 799 | name: value["name"].as_str()?.to_owned(), |
| 800 | size: value["size"].as_u64().unwrap_or(0), |
| 801 | digest: value["digest"].as_str().map(str::to_owned), |
| 802 | format: value["format"].as_str().unwrap_or("zip").to_owned(), |
| 803 | }) |
| 804 | } |
| 805 | } |
| 806 | |
| 807 | /// One artifact per name, the newest (highest id), in name order. |
| 808 | fn newest(listed: Vec<Listed>) -> Vec<Listed> { |
| 809 | let mut by_name: BTreeMap<String, Listed> = BTreeMap::new(); |
| 810 | for artifact in listed { |
| 811 | if by_name.get(&artifact.name).is_none_or(|kept| kept.id < artifact.id) { |
| 812 | by_name.insert(artifact.name.clone(), artifact); |
| 813 | } |
| 814 | } |
| 815 | by_name.into_values().collect() |
| 816 | } |
| 817 | |
| 818 | /// `artifact-ids`: numbers, separated by commas. |
| 819 | fn artifact_ids(text: &str) -> Result<Vec<u64>, String> { |
| 820 | text.split(',') |
| 821 | .map(str::trim) |
| 822 | .filter(|id| !id.is_empty()) |
| 823 | .map(|id| id.parse::<u64>().map_err(|_| format!("`artifact-ids` takes artifact ids, numbers separated by commas; `{id}` is not one."))) |
| 824 | .collect() |
| 825 | } |
| 826 | |
| 827 | /// The folder each artifact is unpacked into: `dest` itself for one |
| 828 | /// artifact by name, for `merge-multiple`, and for a single artifact by |
| 829 | /// id; otherwise a folder of the artifact's name in `dest`. `None` for a |
| 830 | /// name that cannot be a folder. |
| 831 | fn download_targets(dest: &Path, chosen: &[Listed], by_name: bool, by_ids: bool, merge: bool) -> Vec<Option<std::path::PathBuf>> { |
| 832 | let straight = by_name || merge || (by_ids && chosen.len() == 1); |
| 833 | chosen |
| 834 | .iter() |
| 835 | .map(|artifact| { |
| 836 | if straight { |
| 837 | Some(dest.to_path_buf()) |
| 838 | } else if artifact.name.is_empty() || artifact.name == "." || artifact.name == ".." || artifact.name.contains(['/', '\\']) { |
| 839 | None |
| 840 | } else { |
| 841 | Some(dest.join(&artifact.name)) |
| 842 | } |
| 843 | }) |
| 844 | .collect() |
| 845 | } |
| 846 | |
| 847 | /// The SHA-256 of a file, in hex, read as it comes. |
| 848 | fn sha256_file(path: &Path) -> std::io::Result<String> { |
| 849 | let mut file = std::fs::File::open(path)?; |
| 850 | let mut hasher = Sha256::new(); |
| 851 | let mut buffer = vec![0u8; 256 * 1024]; |
| 852 | loop { |
| 853 | let n = file.read(&mut buffer)?; |
| 854 | if n == 0 { |
| 855 | break; |
| 856 | } |
| 857 | hasher.update(&buffer[..n]); |
| 858 | } |
| 859 | Ok(hex::encode(hasher.finalize())) |
| 860 | } |
| 861 | |
| 862 | /// The message of a refused request, from its JSON body. |
| 863 | fn refusal(response: ureq::Response) -> String { |
| 864 | let status = response.status(); |
| 865 | let body: serde_json::Value = response.into_json().unwrap_or_default(); |
| 866 | body["error"]["message"].as_str().map_or_else(|| format!("g1t answered {status}"), str::to_owned) |
| 867 | } |
| 868 | |
| 869 | fn megabytes(bytes: u64) -> String { |
| 870 | if bytes < 1_048_576 { format!("{} KB", bytes.div_ceil(1024)) } else { format!("{:.1} MB", bytes as f64 / 1_048_576.0) } |
| 871 | } |
| 872 | |
| 873 | /// How this machine compresses cache entries: zstd where it has it, else |
| 874 | /// gzip, as `pack_script` decides. |
| 875 | fn compression() -> &'static str { |
| 876 | let zstd = Command::new("sh").args(["-c", "command -v zstd"]).output().is_ok_and(|out| out.status.success()); |
| 877 | if zstd { "zstd" } else { "gzip" } |
| 878 | } |
| 879 | |
| 880 | /// An entry's version: the hash of its `path` lines, as written, and its |
| 881 | /// compression, as `actions/cache` makes one. The same key saved for other |
| 882 | /// paths, or packed another way, is another entry. |
| 883 | fn cache_version(path: &str, compression: &str) -> String { |
| 884 | use sha2::{Digest, Sha256}; |
| 885 | let mut parts = lines(path); |
| 886 | parts.push(compression.to_owned()); |
| 887 | hex::encode(Sha256::digest(parts.join("|").as_bytes())) |
| 888 | } |
| 889 | |
| 890 | /// The lines of a cache's `path`, absolute: `~/` is `home`, a relative |
| 891 | /// path is under `workspace`. A `!` pattern keeps its `!`. |
| 892 | fn cache_patterns(path: &str, home: &str, workspace: &str) -> Vec<String> { |
| 893 | lines(path) |
| 894 | .into_iter() |
| 895 | .map(|line| { |
| 896 | let (bang, p) = match line.strip_prefix('!') { |
| 897 | Some(rest) => ("!", rest.trim().to_owned()), |
| 898 | None => ("", line), |
| 899 | }; |
| 900 | let p = if p == "~" { |
| 901 | home.to_owned() |
| 902 | } else if let Some(rest) = p.strip_prefix("~/") { |
| 903 | format!("{home}/{rest}") |
| 904 | } else { |
| 905 | p |
| 906 | }; |
| 907 | let p = if p.starts_with('/') { p } else { format!("{}/{}", workspace.trim_end_matches('/'), p.trim_start_matches("./")) }; |
| 908 | format!("{bang}{}", p.trim_end_matches('/')) |
| 909 | }) |
| 910 | .collect() |
| 911 | } |
| 912 | |
| 913 | /// A path pattern as a word bash expands as a glob: everything but `*`, |
| 914 | /// `?` and `[...]` escaped, so spaces and quotes stay literal. |
| 915 | fn glob_word(pattern: &str) -> String { |
| 916 | let mut out = String::new(); |
| 917 | for c in pattern.chars() { |
| 918 | if c.is_ascii_alphanumeric() || matches!(c, '*' | '?' | '[' | ']' | '/' | '.' | '-' | '_' | '~' | '+' | ',' | '=' | '@' | ':') { |
| 919 | out.push(c); |
| 920 | } else { |
| 921 | out.push('\\'); |
| 922 | out.push(c); |
| 923 | } |
| 924 | } |
| 925 | out |
| 926 | } |
| 927 | |
| 928 | /// The script that packs a cache: its patterns expanded (`**` reaching |
| 929 | /// any depth), `!` patterns left out, into a tar archive compressed with |
| 930 | /// zstd where there is one, else gzip. Exits 3 when nothing matched. |
| 931 | fn pack_script(patterns: &[String], archive: &str) -> String { |
| 932 | let includes: Vec<String> = patterns.iter().filter(|p| !p.starts_with('!')).map(|p| glob_word(p)).collect(); |
| 933 | let excludes: Vec<String> = patterns |
| 934 | .iter() |
| 935 | .filter_map(|p| p.strip_prefix('!')) |
| 936 | // tar's patterns: `*` already crosses `/`, so `**` is the same. |
| 937 | .map(|p| format!("--exclude={}", quote(&p.replace("**", "*")))) |
| 938 | .collect(); |
| 939 | format!( |
| 940 | "set -o pipefail; shopt -s globstar nullglob dotglob; found=( {} ); files=(); \ |
| 941 | for f in \"${{found[@]}}\"; do if [ -e \"$f\" ]; then files+=(\"$f\"); fi; done; \ |
| 942 | if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; \ |
| 943 | if command -v zstd >/dev/null; then compress='zstd -T0 -3'; else compress=gzip; fi; \ |
| 944 | tar{} -cPf {} -I \"$compress\" {} -- \"${{files[@]}}\"", |
| 945 | includes.join(" "), |
| 946 | tar_local(), |
| 947 | quote(archive), |
| 948 | excludes.join(" ") |
| 949 | ) |
| 950 | } |
| 951 | |
| 952 | /// GNU tar on Windows reads `C:...` as a host and a file; `--force-local` |
| 953 | /// makes every path a local one. Elsewhere the flag is left off, since |
| 954 | /// bsdtar (macOS) doesn't know it. |
| 955 | pub(crate) fn tar_local() -> &'static str { |
| 956 | if cfg!(windows) { |
| 957 | " --force-local" |
| 958 | } else { |
| 959 | "" |
| 960 | } |
| 961 | } |
| 962 | |
| 963 | fn urlencode(text: &str) -> String { |
| 964 | let mut out = String::new(); |
| 965 | for byte in text.bytes() { |
| 966 | if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { |
| 967 | out.push(byte as char); |
| 968 | } else { |
| 969 | out.push_str(&format!("%{byte:02X}")); |
| 970 | } |
| 971 | } |
| 972 | out |
| 973 | } |
| 974 | |
| 975 | #[cfg(test)] |
| 976 | mod tests { |
| 977 | use super::*; |
| 978 | |
| 979 | #[test] |
| 980 | fn a_cache_entrys_version_follows_its_paths_and_compression() { |
| 981 | let version = cache_version("~/.cargo/registry\ntarget", "zstd"); |
| 982 | assert_eq!(version.len(), 64); |
| 983 | assert_eq!(version, cache_version("~/.cargo/registry\n\ntarget\n", "zstd"), "blank lines do not count"); |
| 984 | assert_ne!(version, cache_version("~/.cargo/registry", "zstd")); |
| 985 | assert_ne!(version, cache_version("~/.cargo/registry\ntarget", "gzip")); |
| 986 | } |
| 987 | |
| 988 | #[test] |
| 989 | fn cache_paths_take_home_globs_and_exclusions() { |
| 990 | let paths = cache_patterns("~/.cargo/registry/cache\ntarget/*/release/\n!target/**/incremental\n./dist\n/abs/x\n~", "/home/node", "/w/repo/"); |
| 991 | assert_eq!( |
| 992 | paths, |
| 993 | ["/home/node/.cargo/registry/cache", "/w/repo/target/*/release", "!/w/repo/target/**/incremental", "/w/repo/dist", "/abs/x", "/home/node"] |
| 994 | ); |
| 995 | assert_eq!(glob_word("/w/my repo/target/**/*.rlib"), "/w/my\\ repo/target/**/*.rlib"); |
| 996 | assert_eq!(glob_word("/w/a'b"), "/w/a\\'b"); |
| 997 | } |
| 998 | |
| 999 | #[test] |
| 1000 | fn packing_keeps_windows_paths_local() { |
| 1001 | let script = pack_script(&["C:/w/target".into()], "C:/t/c.tar"); |
| 1002 | assert_eq!(script.contains("--force-local"), cfg!(windows), "{script}"); |
| 1003 | } |
| 1004 | |
| 1005 | #[test] |
| 1006 | fn packing_expands_globs_and_leaves_exclusions_out() { |
| 1007 | let script = pack_script(&["/w/target/*/release".into(), "!/w/target/**/incremental".into()], "/t/c.tar"); |
| 1008 | assert!(script.contains("found=( /w/target/*/release )"), "{script}"); |
| 1009 | assert!(script.contains("--exclude='/w/target/*/incremental'"), "{script}"); |
| 1010 | assert!(script.contains("zstd -T0"), "{script}"); |
| 1011 | assert!(script.contains("exit 3"), "{script}"); |
| 1012 | } |
| 1013 | |
| 1014 | /// Packs and unpacks for real, where bash and tar are (not on Windows). |
| 1015 | #[test] |
| 1016 | #[cfg(unix)] |
| 1017 | fn a_packed_cache_unpacks_without_what_was_left_out() { |
| 1018 | let dir = std::env::temp_dir().join(format!("g1t-cache-test-{}", std::process::id())); |
| 1019 | let _ = std::fs::remove_dir_all(&dir); |
| 1020 | for file in ["target/release/deps/a.rlib", "target/release/incremental/x.bin", "target/wasm/release/deps/b.rlib", "src/main.rs"] { |
| 1021 | let path = dir.join(file); |
| 1022 | std::fs::create_dir_all(path.parent().unwrap()).unwrap(); |
| 1023 | std::fs::write(&path, file).unwrap(); |
| 1024 | } |
| 1025 | let root = dir.display().to_string(); |
| 1026 | let patterns = cache_patterns("target/**/deps\ntarget/release/incremental\n!target/**/incremental", "/home/node", &root); |
| 1027 | let archive = dir.join("c.tar").display().to_string(); |
| 1028 | let status = Command::new("bash").args(["-c", &pack_script(&patterns, &archive)]).status().unwrap(); |
| 1029 | assert!(status.success()); |
| 1030 | let listed = Command::new("tar").args(["-tPf", &archive]).output().unwrap(); |
| 1031 | let listed = String::from_utf8_lossy(&listed.stdout); |
| 1032 | assert!(listed.contains("deps/a.rlib") && listed.contains("deps/b.rlib"), "{listed}"); |
| 1033 | assert!(!listed.contains("incremental") && !listed.contains("main.rs"), "{listed}"); |
| 1034 | let none = Command::new("bash").args(["-c", &pack_script(&[format!("{root}/nothing/*")], &archive)]).status().unwrap(); |
| 1035 | assert_eq!(none.code(), Some(3)); |
| 1036 | let _ = std::fs::remove_dir_all(&dir); |
| 1037 | } |
| 1038 | |
| 1039 | #[test] |
| 1040 | fn transfers_say_how_far_they_have_got_as_githubs_do() { |
| 1041 | assert_eq!(progress("Received", 46_137_344, Some(523_190_823), Duration::from_secs(1)), "Received 46137344 of 523190823 (8.8%), 44.0 MBs/sec"); |
| 1042 | assert_eq!(progress("Sent", 1_048_576, None, Duration::from_secs(2)), "Sent 1048576 bytes, 0.5 MBs/sec"); |
| 1043 | assert_eq!(progress("Received", 10, Some(0), Duration::ZERO), "Received 10 bytes, 0.0 MBs/sec"); |
| 1044 | } |
| 1045 | |
| 1046 | #[test] |
| 1047 | fn keys_are_encoded_and_names_checked() { |
| 1048 | assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202"); |
| 1049 | assert!(check_name("coverage report").is_ok()); |
| 1050 | assert!(check_name("dist-linux_x64.1 (debug)").is_ok()); |
| 1051 | assert!(check_name("ünïcødé").is_ok()); |
| 1052 | assert!(check_name(&"a".repeat(256)).is_ok()); |
| 1053 | assert!(check_name(&"a".repeat(257)).is_err()); |
| 1054 | assert!(check_name("").is_err()); |
| 1055 | for bad in ["a/b", "a\\b", "a:b", "a*b", "a?b", "a\"b", "a<b", "a>b", "a|b", "a\nb", "a\rb"] { |
| 1056 | assert!(check_name(bad).is_err(), "{bad}"); |
| 1057 | } |
| 1058 | assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]); |
| 1059 | } |
| 1060 | |
| 1061 | fn with(pairs: &[(&str, &str)]) -> BTreeMap<String, String> { |
| 1062 | pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() |
| 1063 | } |
| 1064 | |
| 1065 | #[test] |
| 1066 | fn upload_inputs_are_read_as_v4_reads_them() { |
| 1067 | assert_eq!(keep(&with(&[]), true).unwrap(), Keep { retention: 0, level: 6, overwrite: false, hidden: false }); |
| 1068 | let set = with(&[("retention-days", "14"), ("compression-level", "0"), ("overwrite", "true"), ("include-hidden-files", "True")]); |
| 1069 | assert_eq!(keep(&set, true).unwrap(), Keep { retention: 14, level: 0, overwrite: true, hidden: true }); |
| 1070 | // The merge takes no `overwrite`. |
| 1071 | assert!(!keep(&set, false).unwrap().overwrite); |
| 1072 | assert_eq!(keep(&with(&[("retention-days", "0")]), true).unwrap().retention, 0); |
| 1073 | assert!(keep(&with(&[("retention-days", "91")]), true).is_err()); |
| 1074 | assert!(keep(&with(&[("retention-days", "-1")]), true).is_err()); |
| 1075 | assert!(keep(&with(&[("retention-days", "two")]), true).is_err()); |
| 1076 | assert!(keep(&with(&[("compression-level", "10")]), true).is_err()); |
| 1077 | assert!(keep(&with(&[("overwrite", "yes")]), true).is_err()); |
| 1078 | assert_eq!(artifact_ids(" 12, 34 ,,").unwrap(), [12, 34]); |
| 1079 | assert!(artifact_ids("12,abc").is_err()); |
| 1080 | assert_eq!(count(1, "file"), "1 file"); |
| 1081 | assert_eq!(count(37, "file"), "37 files"); |
| 1082 | } |
| 1083 | |
| 1084 | fn listed(id: u64, name: &str) -> Listed { |
| 1085 | Listed { id, name: name.into(), size: 0, digest: None, format: "zip".into() } |
| 1086 | } |
| 1087 | |
| 1088 | #[test] |
| 1089 | fn listings_read_and_keep_the_newest_of_a_name() { |
| 1090 | let json = serde_json::json!([ |
| 1091 | { "id": 1, "name": "dist", "size": 10, "digest": null, "format": "tgz", "created_at": "x", "expires_at": "y" }, |
| 1092 | { "id": "3", "name": "dist", "size": 30, "digest": "sha256:ab", "format": "zip" }, |
| 1093 | { "id": 2, "name": "logs", "size": 5 }, |
| 1094 | { "name": "no id" } |
| 1095 | ]); |
| 1096 | let all: Vec<Listed> = json.as_array().unwrap().iter().filter_map(Listed::from_json).collect(); |
| 1097 | assert_eq!(all.len(), 3); |
| 1098 | assert_eq!(all[0].format, "tgz"); |
| 1099 | let kept = newest(all); |
| 1100 | assert_eq!(kept.iter().map(|a| (a.id, a.name.as_str())).collect::<Vec<_>>(), [(3, "dist"), (2, "logs")]); |
| 1101 | assert_eq!(kept[0].digest.as_deref(), Some("sha256:ab")); |
| 1102 | } |
| 1103 | |
| 1104 | #[test] |
| 1105 | fn downloads_land_where_v4_puts_them() { |
| 1106 | let dest = Path::new("/w/out"); |
| 1107 | let one = [listed(1, "dist")]; |
| 1108 | let two = [listed(1, "dist"), listed(2, "logs")]; |
| 1109 | let at = |targets: Vec<Option<std::path::PathBuf>>| targets.into_iter().map(|t| t.unwrap()).collect::<Vec<_>>(); |
| 1110 | // By name: straight into `path`. |
| 1111 | assert_eq!(at(download_targets(dest, &one, true, false, false)), [dest.to_path_buf()]); |
| 1112 | // Every artifact, or by pattern: a folder each. |
| 1113 | assert_eq!(at(download_targets(dest, &two, false, false, false)), [dest.join("dist"), dest.join("logs")]); |
| 1114 | assert_eq!(at(download_targets(dest, &one, false, false, false)), [dest.join("dist")]); |
| 1115 | // merge-multiple: all into `path`. |
| 1116 | assert_eq!(at(download_targets(dest, &two, false, false, true)), [dest.to_path_buf(), dest.to_path_buf()]); |
| 1117 | // By id: one straight into `path`, several a folder each. |
| 1118 | assert_eq!(at(download_targets(dest, &one, false, true, false)), [dest.to_path_buf()]); |
| 1119 | assert_eq!(at(download_targets(dest, &two, false, true, false)), [dest.join("dist"), dest.join("logs")]); |
| 1120 | // A name that is no folder is refused. |
| 1121 | assert!(download_targets(dest, &[listed(1, ".."), listed(2, "x")], false, false, false)[0].is_none()); |
| 1122 | assert!(glob::matches_part("dist-*", "dist-linux")); |
| 1123 | assert!(!glob::matches_part("dist-*", "logs")); |
| 1124 | } |
| 1125 | } |