| 23 | 23 | | use sha2::{Digest, Sha256}; |
| 24 | 24 | | |
| 25 | 25 | | use super::process::{self, Commands, Ended}; |
| 26 | | − | use super::{Job, Post, PostRun, glob, zip}; |
| 26 | + | use super::{Job, glob, zip}; |
| 27 | 27 | | |
| 28 | | − | /// The largest cache entry, compressed (`g1t_contracts::actions::CACHE_MAX_ENTRY_BYTES`). |
| 29 | | − | const MAX_CACHE_ENTRY: u64 = 2 * 1024 * 1024 * 1024; |
| 30 | 28 | | |
| 31 | 29 | | /// How often a long transfer says how far it has got. |
| 32 | 30 | | const PROGRESS_EVERY: Duration = Duration::from_secs(5); |
| ⋯ |
| 64 | 62 | | command.args(["-c", script]).current_dir(&self.workspace); |
| 65 | 63 | | let mut commands = Commands::default(); |
| 66 | 64 | | matches!(process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0))) |
| 67 | | − | } |
| 68 | | − | |
| 69 | | − | /// Downloads into `file`, as it comes; `Ok(None)` when there is |
| 70 | | − | /// nothing there. |
| 71 | | − | fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> { |
| 72 | | − | let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(1800)).call() { |
| 73 | | − | Ok(response) => response, |
| 74 | | − | Err(ureq::Error::Status(404, _)) => return Ok(None), |
| 75 | | − | Err(error) => return Err(error.to_string()), |
| 76 | | − | }; |
| 77 | | − | let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default(); |
| 78 | | − | let total = response.header("content-length").and_then(|n| n.parse::<u64>().ok()); |
| 79 | | − | let mut out = std::io::BufWriter::new(std::fs::File::create(file).map_err(|e| e.to_string())?); |
| 80 | | − | self.receive(&mut response.into_reader().take(MAX_CACHE_ENTRY + 1024), total, |bytes| out.write_all(bytes))?; |
| 81 | | − | out.flush().map_err(|e| e.to_string())?; |
| 82 | | − | Ok(Some(matched)) |
| 83 | 65 | | } |
| 84 | 66 | | |
| 85 | 67 | | /// Reads a download to its end, handing `each` every piece, and says |
| ⋯ |
| 105 | 87 | | self.log.tick(); |
| 106 | 88 | | } |
| 107 | 89 | | Ok(received) |
| 108 | | − | } |
| 109 | | − | |
| 110 | | − | /// Saves `file` as the cache entry `key`, in parts. `Ok(false)` when |
| 111 | | − | /// the key is already cached. |
| 112 | | − | fn upload_cache(&mut self, key: &str, version: &str, file: &Path) -> Result<bool, String> { |
| 113 | | − | let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len(); |
| 114 | | − | if size > MAX_CACHE_ENTRY { |
| 115 | | − | return Err(format!("it is {} MB, more than a cache entry may be ({} MB)", size / 1_048_576, MAX_CACHE_ENTRY / 1_048_576)); |
| 116 | | − | } |
| 117 | | − | let started = match ureq::post(&self.url(&format!("cache/uploads?key={}&size={size}&version={}", urlencode(key), urlencode(version)))) |
| 118 | | − | .set("authorization", &self.auth()) |
| 119 | | − | .timeout(Duration::from_secs(60)) |
| 120 | | − | .call() |
| 121 | | − | { |
| 122 | | − | Ok(response) => response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?, |
| 123 | | − | Err(ureq::Error::Status(409, _)) => return Ok(false), |
| 124 | | − | Err(ureq::Error::Status(_, response)) => return Err(refusal(response)), |
| 125 | | − | Err(error) => return Err(error.to_string()), |
| 126 | | − | }; |
| 127 | | − | let id = started["id"].as_str().unwrap_or_default().to_owned(); |
| 128 | | − | let upload = started["upload"].as_str().unwrap_or_default().to_owned(); |
| 129 | | − | let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024); |
| 130 | | − | let base = format!("cache/uploads/{}", urlencode(&id)); |
| 131 | | − | let sent = self.send_parts(&base, &upload, file, part_bytes); |
| 132 | | − | let parts = match sent { |
| 133 | | − | Ok(parts) => parts, |
| 134 | | − | Err(error) => { |
| 135 | | − | let _ = ureq::delete(&self.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &self.auth()).call(); |
| 136 | | − | return Err(error); |
| 137 | | − | } |
| 138 | | − | }; |
| 139 | | − | ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload)))) |
| 140 | | − | .set("authorization", &self.auth()) |
| 141 | | − | .timeout(Duration::from_secs(120)) |
| 142 | | − | .send_json(serde_json::json!({ "size": size, "parts": parts })) |
| 143 | | − | .map_err(|e| match e { |
| 144 | | − | ureq::Error::Status(_, response) => refusal(response), |
| 145 | | − | other => other.to_string(), |
| 146 | | − | })?; |
| 147 | | − | Ok(true) |
| 148 | 90 | | } |
| 149 | 91 | | |
| 150 | 92 | | /// Sends `file` in parts of `part_bytes`; each part's number and etag. |
| ⋯ |
| 567 | 509 | | } |
| 568 | 510 | | } |
| 569 | 511 | | result |
| 570 | | − | } |
| 571 | | − | |
| 572 | | − | /// The cache's `path`, each made absolute (`~/` is the home folder, |
| 573 | | − | /// anything else is under the workspace), `!` patterns kept as they |
| 574 | | − | /// came, with their `!`. |
| 575 | | − | fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> { |
| 576 | | − | let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into()); |
| 577 | | − | let workspace = self.workspace.display().to_string(); |
| 578 | | − | cache_patterns(with.get("path").map(String::as_str).unwrap_or_default(), &home, &workspace) |
| 579 | | − | } |
| 580 | | − | |
| 581 | | − | /// `actions/cache` and `actions/cache/restore`: restores what it can, |
| 582 | | − | /// and for `actions/cache`, saves at the end of the job on a miss. |
| 583 | | − | pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) { |
| 584 | | − | let key = with.get("key").cloned().unwrap_or_default(); |
| 585 | | − | if key.is_empty() { |
| 586 | | − | self.log.line("##[error]The cache needs a `key`."); |
| 587 | | − | return (false, BTreeMap::new()); |
| 588 | | − | } |
| 589 | | − | let paths = self.cache_paths(with); |
| 590 | | − | let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression()); |
| 591 | | − | let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default()); |
| 592 | | − | let query = format!( |
| 593 | | − | "cache?key={}&restore={}&version={}", |
| 594 | | − | urlencode(&key), |
| 595 | | − | urlencode(&restore.join("\n")), |
| 596 | | − | urlencode(&version) |
| 597 | | − | ); |
| 598 | | − | let archive = self.temp.join("cache-restore.tar"); |
| 599 | | − | let started = Instant::now(); |
| 600 | | − | let mut outputs = BTreeMap::new(); |
| 601 | | − | let exact = match self.download(&query, &archive) { |
| 602 | | − | Ok(Some(matched)) => { |
| 603 | | − | let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true"); |
| 604 | | − | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 605 | | − | // tar finds out from the archive whether it is zstd or gzip. |
| 606 | | − | if !lookup_only && !self.shell(&format!("tar{} -xPf {}", tar_local(), quote(&archive.display().to_string()))) { |
| 607 | | − | self.log.line("##[warning]The cache was found but could not be unpacked."); |
| 608 | | − | } |
| 609 | | − | let _ = std::fs::remove_file(&archive); |
| 610 | | − | self.log.line(&format!("Cache restored from key: {matched} ({} in {:.1}s)", megabytes(size), started.elapsed().as_secs_f64())); |
| 611 | | − | outputs.insert("cache-matched-key".into(), matched.clone()); |
| 612 | | − | matched == key |
| 613 | | − | } |
| 614 | | − | Ok(None) => { |
| 615 | | − | self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", "))); |
| 616 | | − | if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") { |
| 617 | | − | self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set."); |
| 618 | | − | return (false, outputs); |
| 619 | | − | } |
| 620 | | − | false |
| 621 | | − | } |
| 622 | | − | Err(error) => { |
| 623 | | − | self.log.line(&format!("##[warning]The cache could not be read: {error}")); |
| 624 | | − | false |
| 625 | | − | } |
| 626 | | − | }; |
| 627 | | − | outputs.insert("cache-hit".into(), exact.to_string()); |
| 628 | | − | outputs.insert("cache-primary-key".into(), key.clone()); |
| 629 | | − | if save_after && !exact { |
| 630 | | − | self.posts.push(Post { |
| 631 | | − | name: format!("Post {title}"), |
| 632 | | − | condition: "success()".into(), |
| 633 | | − | env: BTreeMap::new(), |
| 634 | | − | run: PostRun::CacheSave { key, paths, version }, |
| 635 | | − | }); |
| 636 | | − | } |
| 637 | | − | (true, outputs) |
| 638 | | − | } |
| 639 | | − | |
| 640 | | − | /// Saves paths under a key, unless the key is taken. |
| 641 | | − | pub(crate) fn cache_save(&mut self, key: &str, paths: &[String], version: &str) -> bool { |
| 642 | | − | if paths.iter().all(|p| p.starts_with('!')) { |
| 643 | | − | self.log.line("##[warning]Nothing to cache: no `path`."); |
| 644 | | − | return true; |
| 645 | | − | } |
| 646 | | − | let archive = self.temp.join("cache-save.tar"); |
| 647 | | − | let started = Instant::now(); |
| 648 | | − | match self.run_shell(&pack_script(paths, &archive.display().to_string())) { |
| 649 | | − | Some(0) => {} |
| 650 | | − | Some(3) => { |
| 651 | | − | self.log.line("##[warning]None of the cache's paths exist; nothing was saved."); |
| 652 | | − | return true; |
| 653 | | − | } |
| 654 | | − | _ => { |
| 655 | | − | self.log.line("##[warning]The cache could not be packed; nothing was saved."); |
| 656 | | − | return true; |
| 657 | | − | } |
| 658 | | − | } |
| 659 | | − | let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0); |
| 660 | | − | let packed = started.elapsed().as_secs_f64(); |
| 661 | | − | match self.upload_cache(key, version, &archive) { |
| 662 | | − | Ok(true) => self.log.line(&format!( |
| 663 | | − | "Cache saved with key: {key} ({}, packed in {packed:.1}s, sent in {:.1}s)", |
| 664 | | − | megabytes(size), |
| 665 | | − | started.elapsed().as_secs_f64() - packed |
| 666 | | − | )), |
| 667 | | − | Ok(false) => self.log.line(&format!("Cache not saved: {key} is already cached.")), |
| 668 | | − | // A cache that cannot be saved does not fail the job, as on GitHub. |
| 669 | | − | Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")), |
| 670 | | − | } |
| 671 | | − | let _ = std::fs::remove_file(&archive); |
| 672 | | − | true |
| 673 | | − | } |
| 674 | | − | |
| 675 | | − | /// Runs a shell line, logging its output; its exit code. |
| 676 | | − | fn run_shell(&mut self, script: &str) -> Option<i32> { |
| 677 | | − | let mut command = Command::new("bash"); |
| 678 | | − | command.args(["-c", script]).current_dir(&self.workspace); |
| 679 | | − | let mut commands = Commands::default(); |
| 680 | | − | match process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands) { |
| 681 | | − | Ok(Ended::Exited(code)) => Some(code), |
| 682 | | − | _ => None, |
| 683 | | − | } |
| 684 | 512 | | } |
| 685 | 513 | | |
| 686 | | − | /// `actions/cache/save`. |
| 687 | | − | pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) { |
| 688 | | − | let key = with.get("key").cloned().unwrap_or_default(); |
| 689 | | − | let paths = self.cache_paths(with); |
| 690 | | − | let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression()); |
| 691 | | − | (self.cache_save(&key, &paths, &version), BTreeMap::new()) |
| 692 | | − | } |
| 693 | 514 | | } |
| 694 | 515 | | |
| 695 | 516 | | /// An input, trimmed; `None` when empty. |
| ⋯ |
| 868 | 689 | | |
| 869 | 690 | | fn megabytes(bytes: u64) -> String { |
| 870 | 691 | | if bytes < 1_048_576 { format!("{} KB", bytes.div_ceil(1024)) } else { format!("{:.1} MB", bytes as f64 / 1_048_576.0) } |
| 871 | | − | } |
| 872 | | − | |
| 873 | | − | /// How this machine compresses cache entries: zstd where it has it, else |
| 874 | | − | /// gzip, as `pack_script` decides. |
| 875 | | − | fn compression() -> &'static str { |
| 876 | | − | let zstd = Command::new("sh").args(["-c", "command -v zstd"]).output().is_ok_and(|out| out.status.success()); |
| 877 | | − | if zstd { "zstd" } else { "gzip" } |
| 878 | | − | } |
| 879 | | − | |
| 880 | | − | /// An entry's version: the hash of its `path` lines, as written, and its |
| 881 | | − | /// compression, as `actions/cache` makes one. The same key saved for other |
| 882 | | − | /// paths, or packed another way, is another entry. |
| 883 | | − | fn cache_version(path: &str, compression: &str) -> String { |
| 884 | | − | use sha2::{Digest, Sha256}; |
| 885 | | − | let mut parts = lines(path); |
| 886 | | − | parts.push(compression.to_owned()); |
| 887 | | − | hex::encode(Sha256::digest(parts.join("|").as_bytes())) |
| 888 | | − | } |
| 889 | | − | |
| 890 | | − | /// The lines of a cache's `path`, absolute: `~/` is `home`, a relative |
| 891 | | − | /// path is under `workspace`. A `!` pattern keeps its `!`. |
| 892 | | − | fn cache_patterns(path: &str, home: &str, workspace: &str) -> Vec<String> { |
| 893 | | − | lines(path) |
| 894 | | − | .into_iter() |
| 895 | | − | .map(|line| { |
| 896 | | − | let (bang, p) = match line.strip_prefix('!') { |
| 897 | | − | Some(rest) => ("!", rest.trim().to_owned()), |
| 898 | | − | None => ("", line), |
| 899 | | − | }; |
| 900 | | − | let p = if p == "~" { |
| 901 | | − | home.to_owned() |
| 902 | | − | } else if let Some(rest) = p.strip_prefix("~/") { |
| 903 | | − | format!("{home}/{rest}") |
| 904 | | − | } else { |
| 905 | | − | p |
| 906 | | − | }; |
| 907 | | − | let p = if p.starts_with('/') { p } else { format!("{}/{}", workspace.trim_end_matches('/'), p.trim_start_matches("./")) }; |
| 908 | | − | format!("{bang}{}", p.trim_end_matches('/')) |
| 909 | | − | }) |
| 910 | | − | .collect() |
| 911 | | − | } |
| 912 | | − | |
| 913 | | − | /// A path pattern as a word bash expands as a glob: everything but `*`, |
| 914 | | − | /// `?` and `[...]` escaped, so spaces and quotes stay literal. |
| 915 | | − | fn glob_word(pattern: &str) -> String { |
| 916 | | − | let mut out = String::new(); |
| 917 | | − | for c in pattern.chars() { |
| 918 | | − | if c.is_ascii_alphanumeric() || matches!(c, '*' | '?' | '[' | ']' | '/' | '.' | '-' | '_' | '~' | '+' | ',' | '=' | '@' | ':') { |
| 919 | | − | out.push(c); |
| 920 | | − | } else { |
| 921 | | − | out.push('\\'); |
| 922 | | − | out.push(c); |
| 923 | | − | } |
| 924 | | − | } |
| 925 | | − | out |
| 926 | | − | } |
| 927 | | − | |
| 928 | | − | /// The script that packs a cache: its patterns expanded (`**` reaching |
| 929 | | − | /// any depth), `!` patterns left out, into a tar archive compressed with |
| 930 | | − | /// zstd where there is one, else gzip. Exits 3 when nothing matched. |
| 931 | | − | fn pack_script(patterns: &[String], archive: &str) -> String { |
| 932 | | − | let includes: Vec<String> = patterns.iter().filter(|p| !p.starts_with('!')).map(|p| glob_word(p)).collect(); |
| 933 | | − | let excludes: Vec<String> = patterns |
| 934 | | − | .iter() |
| 935 | | − | .filter_map(|p| p.strip_prefix('!')) |
| 936 | | − | // tar's patterns: `*` already crosses `/`, so `**` is the same. |
| 937 | | − | .map(|p| format!("--exclude={}", quote(&p.replace("**", "*")))) |
| 938 | | − | .collect(); |
| 939 | | − | format!( |
| 940 | | − | "set -o pipefail; shopt -s globstar nullglob dotglob; found=( {} ); files=(); \ |
| 941 | | − | for f in \"${{found[@]}}\"; do if [ -e \"$f\" ]; then files+=(\"$f\"); fi; done; \ |
| 942 | | − | if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; \ |
| 943 | | − | if command -v zstd >/dev/null; then compress='zstd -T0 -3'; else compress=gzip; fi; \ |
| 944 | | − | tar{} -cPf {} -I \"$compress\" {} -- \"${{files[@]}}\"", |
| 945 | | − | includes.join(" "), |
| 946 | | − | tar_local(), |
| 947 | | − | quote(archive), |
| 948 | | − | excludes.join(" ") |
| 949 | | − | ) |
| 950 | 692 | | } |
| 951 | 693 | | |
| 952 | 694 | | /// GNU tar on Windows reads `C:...` as a host and a file; `--force-local` |
| ⋯ |
| 975 | 717 | | #[cfg(test)] |
| 976 | 718 | | mod tests { |
| 977 | 719 | | use super::*; |
| 978 | | − | |
| 979 | | − | #[test] |
| 980 | | − | fn a_cache_entrys_version_follows_its_paths_and_compression() { |
| 981 | | − | let version = cache_version("~/.cargo/registry\ntarget", "zstd"); |
| 982 | | − | assert_eq!(version.len(), 64); |
| 983 | | − | assert_eq!(version, cache_version("~/.cargo/registry\n\ntarget\n", "zstd"), "blank lines do not count"); |
| 984 | | − | assert_ne!(version, cache_version("~/.cargo/registry", "zstd")); |
| 985 | | − | assert_ne!(version, cache_version("~/.cargo/registry\ntarget", "gzip")); |
| 986 | | − | } |
| 987 | | − | |
| 988 | | − | #[test] |
| 989 | | − | fn cache_paths_take_home_globs_and_exclusions() { |
| 990 | | − | let paths = cache_patterns("~/.cargo/registry/cache\ntarget/*/release/\n!target/**/incremental\n./dist\n/abs/x\n~", "/home/node", "/w/repo/"); |
| 991 | | − | assert_eq!( |
| 992 | | − | paths, |
| 993 | | − | ["/home/node/.cargo/registry/cache", "/w/repo/target/*/release", "!/w/repo/target/**/incremental", "/w/repo/dist", "/abs/x", "/home/node"] |
| 994 | | − | ); |
| 995 | | − | assert_eq!(glob_word("/w/my repo/target/**/*.rlib"), "/w/my\\ repo/target/**/*.rlib"); |
| 996 | | − | assert_eq!(glob_word("/w/a'b"), "/w/a\\'b"); |
| 997 | | − | } |
| 998 | | − | |
| 999 | | − | #[test] |
| 1000 | | − | fn packing_keeps_windows_paths_local() { |
| 1001 | | − | let script = pack_script(&["C:/w/target".into()], "C:/t/c.tar"); |
| 1002 | | − | assert_eq!(script.contains("--force-local"), cfg!(windows), "{script}"); |
| 1003 | | − | } |
| 1004 | | − | |
| 1005 | | − | #[test] |
| 1006 | | − | fn packing_expands_globs_and_leaves_exclusions_out() { |
| 1007 | | − | let script = pack_script(&["/w/target/*/release".into(), "!/w/target/**/incremental".into()], "/t/c.tar"); |
| 1008 | | − | assert!(script.contains("found=( /w/target/*/release )"), "{script}"); |
| 1009 | | − | assert!(script.contains("--exclude='/w/target/*/incremental'"), "{script}"); |
| 1010 | | − | assert!(script.contains("zstd -T0"), "{script}"); |
| 1011 | | − | assert!(script.contains("exit 3"), "{script}"); |
| 1012 | | − | } |
| 1013 | | − | |
| 1014 | | − | /// Packs and unpacks for real, where bash and tar are (not on Windows). |
| 1015 | | − | #[test] |
| 1016 | | − | #[cfg(unix)] |
| 1017 | | − | fn a_packed_cache_unpacks_without_what_was_left_out() { |
| 1018 | | − | let dir = std::env::temp_dir().join(format!("g1t-cache-test-{}", std::process::id())); |
| 1019 | | − | let _ = std::fs::remove_dir_all(&dir); |
| 1020 | | − | for file in ["target/release/deps/a.rlib", "target/release/incremental/x.bin", "target/wasm/release/deps/b.rlib", "src/main.rs"] { |
| 1021 | | − | let path = dir.join(file); |
| 1022 | | − | std::fs::create_dir_all(path.parent().unwrap()).unwrap(); |
| 1023 | | − | std::fs::write(&path, file).unwrap(); |
| 1024 | | − | } |
| 1025 | | − | let root = dir.display().to_string(); |
| 1026 | | − | let patterns = cache_patterns("target/**/deps\ntarget/release/incremental\n!target/**/incremental", "/home/node", &root); |
| 1027 | | − | let archive = dir.join("c.tar").display().to_string(); |
| 1028 | | − | let status = Command::new("bash").args(["-c", &pack_script(&patterns, &archive)]).status().unwrap(); |
| 1029 | | − | assert!(status.success()); |
| 1030 | | − | let listed = Command::new("tar").args(["-tPf", &archive]).output().unwrap(); |
| 1031 | | − | let listed = String::from_utf8_lossy(&listed.stdout); |
| 1032 | | − | assert!(listed.contains("deps/a.rlib") && listed.contains("deps/b.rlib"), "{listed}"); |
| 1033 | | − | assert!(!listed.contains("incremental") && !listed.contains("main.rs"), "{listed}"); |
| 1034 | | − | let none = Command::new("bash").args(["-c", &pack_script(&[format!("{root}/nothing/*")], &archive)]).status().unwrap(); |
| 1035 | | − | assert_eq!(none.code(), Some(3)); |
| 1036 | | − | let _ = std::fs::remove_dir_all(&dir); |
| 1037 | | − | } |
| 1038 | 720 | | |
| 1039 | 721 | | #[test] |
| 1040 | 722 | | fn transfers_say_how_far_they_have_got_as_githubs_do() { |