| 1 | //! Finding the files an artifact's `path` names, as `actions/upload-artifact` |
| 2 | //! does with `@actions/glob`: each line a file, a folder (taken whole) or a |
| 3 | //! glob, `!` lines leaving files out, and the folder the files are stored |
| 4 | //! relative to. Done here rather than in bash so it works the same on a |
| 5 | //! self-hosted runner on Windows or macOS. |
| 6 | //! |
| 7 | //! Paths are handled as text with `/` between their parts: `/w/dist` on |
| 8 | //! Linux and macOS, `C:/w/dist` on Windows, where matching ignores case. |
| 9 | |
| 10 | use std::collections::{BTreeMap, BTreeSet}; |
| 11 | use std::path::{Path, PathBuf}; |
| 12 | |
| 13 | /// What a `path` found: the folder files are stored relative to, and each |
| 14 | /// file's name in the artifact with the file on disk, sorted by name. |
| 15 | #[derive(Debug, Default)] |
| 16 | pub(crate) struct Found { |
| 17 | pub(crate) root: String, |
| 18 | pub(crate) files: Vec<(String, PathBuf)>, |
| 19 | } |
| 20 | |
| 21 | /// The files a `path` input names. `workspace` is where relative lines |
| 22 | /// start, `home` what `~` stands for. Files and folders whose names start |
| 23 | /// with `.` are left out below the paths named, unless `hidden`. |
| 24 | pub(crate) fn find(lines: &[String], workspace: &str, home: &str, hidden: bool) -> Found { |
| 25 | let mut includes = Vec::new(); |
| 26 | let mut excludes = Vec::new(); |
| 27 | for line in lines { |
| 28 | match line.strip_prefix('!') { |
| 29 | Some(rest) => excludes.push(resolve(rest.trim(), workspace, home)), |
| 30 | None => includes.push(resolve(line, workspace, home)), |
| 31 | } |
| 32 | } |
| 33 | let mut matched = BTreeSet::new(); |
| 34 | for pattern in &includes { |
| 35 | let search = search_path(pattern); |
| 36 | let Ok(meta) = std::fs::metadata(&search) else { continue }; |
| 37 | let plain = search == *pattern; |
| 38 | if meta.is_file() { |
| 39 | if plain { |
| 40 | matched.insert(search.clone()); |
| 41 | } |
| 42 | continue; |
| 43 | } |
| 44 | let mut files = Vec::new(); |
| 45 | let mut stack = Vec::new(); |
| 46 | walk(Path::new(&search), &search, hidden, &mut stack, &mut files); |
| 47 | let depth = segments(&search).len(); |
| 48 | for file in files { |
| 49 | // A glob that matches a folder takes everything in it. |
| 50 | if plain || prefixes(&file).skip(depth).any(|p| matches(pattern, &p)) { |
| 51 | matched.insert(file); |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | matched.retain(|file| !excludes.iter().any(|ex| prefixes(file).any(|p| matches(ex, &p)))); |
| 56 | let searches: Vec<String> = includes.iter().map(|p| search_path(p)).collect(); |
| 57 | let all: Vec<String> = matched.into_iter().collect(); |
| 58 | let Some(root) = root_dir(&searches, &all) else { return Found::default() }; |
| 59 | let mut files: Vec<(String, PathBuf)> = all |
| 60 | .iter() |
| 61 | .filter_map(|file| relative(&root, file).map(|name| (name, PathBuf::from(file)))) |
| 62 | .collect(); |
| 63 | files.sort(); |
| 64 | files.dedup_by(|a, b| a.0 == b.0); |
| 65 | Found { root, files } |
| 66 | } |
| 67 | |
| 68 | /// A line made absolute with `/` separators: `~` is `home`, anything not |
| 69 | /// absolute is under `workspace`; `.` parts dropped and `..` taken back. |
| 70 | pub(crate) fn resolve(line: &str, workspace: &str, home: &str) -> String { |
| 71 | let line = separators(line.trim()); |
| 72 | let line = if line == "~" { |
| 73 | separators(home) |
| 74 | } else if let Some(rest) = line.strip_prefix("~/") { |
| 75 | format!("{}/{rest}", separators(home).trim_end_matches('/')) |
| 76 | } else if is_absolute(&line) { |
| 77 | line |
| 78 | } else { |
| 79 | format!("{}/{line}", separators(workspace).trim_end_matches('/')) |
| 80 | }; |
| 81 | let mut parts: Vec<&str> = Vec::new(); |
| 82 | for (i, part) in line.split('/').enumerate() { |
| 83 | match part { |
| 84 | "" if i > 0 => {} |
| 85 | "." => {} |
| 86 | ".." if parts.len() > 1 && !has_glob(parts[parts.len() - 1]) => { |
| 87 | parts.pop(); |
| 88 | } |
| 89 | _ => parts.push(part), |
| 90 | } |
| 91 | } |
| 92 | let joined = parts.join("/"); |
| 93 | if joined.is_empty() || (parts.len() == 1 && parts[0].is_empty()) { "/".into() } else if parts.len() == 1 { format!("{joined}/") } else { joined } |
| 94 | } |
| 95 | |
| 96 | fn separators(path: &str) -> String { |
| 97 | if cfg!(windows) { path.replace('\\', "/") } else { path.to_owned() } |
| 98 | } |
| 99 | |
| 100 | fn is_absolute(path: &str) -> bool { |
| 101 | let bytes = path.as_bytes(); |
| 102 | path.starts_with('/') || (cfg!(windows) && bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':') |
| 103 | } |
| 104 | |
| 105 | fn has_glob(segment: &str) -> bool { |
| 106 | segment.contains(['*', '?', '[']) |
| 107 | } |
| 108 | |
| 109 | fn segments(path: &str) -> Vec<&str> { |
| 110 | path.trim_end_matches('/').split('/').collect() |
| 111 | } |
| 112 | |
| 113 | /// The part of a pattern before its first glob: where looking starts. |
| 114 | /// A pattern with no glob is its own search path. |
| 115 | pub(crate) fn search_path(pattern: &str) -> String { |
| 116 | let parts = segments(pattern); |
| 117 | let literal: Vec<&str> = parts.iter().take_while(|s| !has_glob(s)).copied().collect(); |
| 118 | if literal.len() == parts.len() { |
| 119 | return pattern.trim_end_matches('/').to_owned(); |
| 120 | } |
| 121 | let joined = literal.join("/"); |
| 122 | if literal.len() <= 1 { format!("{joined}/") } else { joined } |
| 123 | } |
| 124 | |
| 125 | /// A path and each folder above it, shortest first, as text. |
| 126 | fn prefixes(path: &str) -> impl Iterator<Item = String> + '_ { |
| 127 | let parts = segments(path); |
| 128 | (1..=parts.len()).map(move |n| parts[..n].join("/")) |
| 129 | } |
| 130 | |
| 131 | /// Every file under `dir`, following links, leaving out hidden names |
| 132 | /// unless `hidden`. `stack` holds the folders being walked, so a link |
| 133 | /// back up the tree is not followed round. |
| 134 | fn walk(dir: &Path, text: &str, hidden: bool, stack: &mut Vec<PathBuf>, out: &mut Vec<String>) { |
| 135 | let real = std::fs::canonicalize(dir).unwrap_or_else(|_| dir.to_path_buf()); |
| 136 | if stack.contains(&real) { |
| 137 | return; |
| 138 | } |
| 139 | let Ok(entries) = std::fs::read_dir(dir) else { return }; |
| 140 | stack.push(real); |
| 141 | let mut names: Vec<(String, PathBuf)> = entries.flatten().map(|e| (e.file_name().to_string_lossy().into_owned(), e.path())).collect(); |
| 142 | names.sort(); |
| 143 | for (name, path) in names { |
| 144 | if !hidden && name.starts_with('.') { |
| 145 | continue; |
| 146 | } |
| 147 | let child = format!("{}/{name}", text.trim_end_matches('/')); |
| 148 | match std::fs::metadata(&path) { |
| 149 | Ok(meta) if meta.is_dir() => walk(&path, &child, hidden, stack, out), |
| 150 | Ok(meta) if meta.is_file() => out.push(child), |
| 151 | _ => {} |
| 152 | } |
| 153 | } |
| 154 | stack.pop(); |
| 155 | } |
| 156 | |
| 157 | /// The folder an artifact's files are stored relative to, as |
| 158 | /// `actions/upload-artifact` picks it: the deepest folder common to the |
| 159 | /// search paths when there are several; the folder of the one file when |
| 160 | /// the only search path is that file; otherwise the search path. |
| 161 | pub(crate) fn root_dir(searches: &[String], files: &[String]) -> Option<String> { |
| 162 | let mut unique: Vec<String> = Vec::new(); |
| 163 | for search in searches { |
| 164 | if !unique.contains(search) { |
| 165 | unique.push(search.clone()); |
| 166 | } |
| 167 | } |
| 168 | // A search path inside another one adds nothing. |
| 169 | let unique: Vec<String> = unique |
| 170 | .iter() |
| 171 | .filter(|s| !unique.iter().any(|other| other != *s && relative(other, s).is_some())) |
| 172 | .cloned() |
| 173 | .collect(); |
| 174 | match unique.as_slice() { |
| 175 | [] => None, |
| 176 | [one] if files.len() == 1 && files[0] == *one => { |
| 177 | let parts = segments(one); |
| 178 | let parent = parts[..parts.len() - 1].join("/"); |
| 179 | Some(if parts.len() <= 2 { format!("{parent}/") } else { parent }) |
| 180 | } |
| 181 | [one] => Some(one.clone()), |
| 182 | many => { |
| 183 | let first = segments(&many[0]); |
| 184 | let mut common = first.len(); |
| 185 | for other in &many[1..] { |
| 186 | let other = segments(other); |
| 187 | common = common.min(first.iter().zip(&other).take_while(|(a, b)| same(a, b)).count()); |
| 188 | } |
| 189 | if common == 0 { |
| 190 | return None; |
| 191 | } |
| 192 | let joined = first[..common].join("/"); |
| 193 | Some(if common == 1 { format!("{joined}/") } else { joined }) |
| 194 | } |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | fn same(a: &str, b: &str) -> bool { |
| 199 | if cfg!(windows) { a.eq_ignore_ascii_case(b) } else { a == b } |
| 200 | } |
| 201 | |
| 202 | /// `path` relative to the folder `root`, with `/`, when it is inside it. |
| 203 | fn relative(root: &str, path: &str) -> Option<String> { |
| 204 | let root = segments(root); |
| 205 | let path = segments(path); |
| 206 | let root: Vec<&str> = root.into_iter().filter(|s| !s.is_empty()).collect(); |
| 207 | let rest: Vec<&str> = path.iter().copied().filter(|s| !s.is_empty()).collect(); |
| 208 | if rest.len() <= root.len() || !root.iter().zip(&rest).all(|(a, b)| same(a, b)) { |
| 209 | return None; |
| 210 | } |
| 211 | Some(rest[root.len()..].join("/")) |
| 212 | } |
| 213 | |
| 214 | /// Whether `path` matches `pattern`, both absolute with `/`: `*` and `?` |
| 215 | /// within a part, `**` as a whole part for any number of parts, `[...]` |
| 216 | /// for one of a set. Leading dots match like any character. |
| 217 | pub(crate) fn matches(pattern: &str, path: &str) -> bool { |
| 218 | let pattern = segments(pattern); |
| 219 | let path = segments(path); |
| 220 | match_parts(&pattern, &path) |
| 221 | } |
| 222 | |
| 223 | fn match_parts(pattern: &[&str], path: &[&str]) -> bool { |
| 224 | match pattern.split_first() { |
| 225 | None => path.is_empty(), |
| 226 | Some((&"**", rest)) => (0..=path.len()).any(|skip| match_parts(rest, &path[skip..])), |
| 227 | Some((first, rest)) => match path.split_first() { |
| 228 | Some((part, others)) => matches_part(first, part) && match_parts(rest, others), |
| 229 | None => false, |
| 230 | }, |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | /// Whether one name matches one glob part (`**` within a part is `*`). |
| 235 | pub(crate) fn matches_part(pattern: &str, name: &str) -> bool { |
| 236 | let (pattern, name): (Vec<char>, Vec<char>) = if cfg!(windows) { |
| 237 | (pattern.to_lowercase().chars().collect(), name.to_lowercase().chars().collect()) |
| 238 | } else { |
| 239 | (pattern.chars().collect(), name.chars().collect()) |
| 240 | }; |
| 241 | let mut memo = BTreeMap::new(); |
| 242 | glob_chars(&pattern, &name, 0, 0, &mut memo) |
| 243 | } |
| 244 | |
| 245 | fn glob_chars(p: &[char], n: &[char], i: usize, j: usize, memo: &mut BTreeMap<(usize, usize), bool>) -> bool { |
| 246 | if let Some(known) = memo.get(&(i, j)) { |
| 247 | return *known; |
| 248 | } |
| 249 | let result = if i == p.len() { |
| 250 | j == n.len() |
| 251 | } else { |
| 252 | match p[i] { |
| 253 | '*' => glob_chars(p, n, i + 1, j, memo) || (j < n.len() && glob_chars(p, n, i, j + 1, memo)), |
| 254 | '?' => j < n.len() && glob_chars(p, n, i + 1, j + 1, memo), |
| 255 | '[' => match class(p, i) { |
| 256 | Some((end, set)) => j < n.len() && set(n[j]) && glob_chars(p, n, end, j + 1, memo), |
| 257 | None => j < n.len() && n[j] == '[' && glob_chars(p, n, i + 1, j + 1, memo), |
| 258 | }, |
| 259 | c => j < n.len() && n[j] == c && glob_chars(p, n, i + 1, j + 1, memo), |
| 260 | } |
| 261 | }; |
| 262 | memo.insert((i, j), result); |
| 263 | result |
| 264 | } |
| 265 | |
| 266 | /// The set `[...]` starting at `p[start]`: where it ends, and what it |
| 267 | /// takes. `None` when it never closes, so the `[` is a plain character. |
| 268 | #[allow(clippy::type_complexity)] |
| 269 | fn class(p: &[char], start: usize) -> Option<(usize, Box<dyn Fn(char) -> bool>)> { |
| 270 | let mut i = start + 1; |
| 271 | let negated = i < p.len() && (p[i] == '!' || p[i] == '^'); |
| 272 | if negated { |
| 273 | i += 1; |
| 274 | } |
| 275 | let mut ranges = Vec::new(); |
| 276 | let mut first = true; |
| 277 | while i < p.len() { |
| 278 | if p[i] == ']' && !first { |
| 279 | return Some((i + 1, Box::new(move |c| ranges.iter().any(|&(lo, hi)| lo <= c && c <= hi) != negated))); |
| 280 | } |
| 281 | first = false; |
| 282 | if i + 2 < p.len() && p[i + 1] == '-' && p[i + 2] != ']' { |
| 283 | ranges.push((p[i], p[i + 2])); |
| 284 | i += 3; |
| 285 | } else { |
| 286 | ranges.push((p[i], p[i])); |
| 287 | i += 1; |
| 288 | } |
| 289 | } |
| 290 | None |
| 291 | } |
| 292 | |
| 293 | #[cfg(test)] |
| 294 | mod tests { |
| 295 | use super::*; |
| 296 | |
| 297 | fn tree(files: &[&str]) -> (PathBuf, String) { |
| 298 | let dir = std::env::temp_dir().join(format!("g1t-glob-test-{}-{}", std::process::id(), super::super::rand_id())); |
| 299 | for file in files { |
| 300 | let path = dir.join(file); |
| 301 | std::fs::create_dir_all(path.parent().unwrap()).unwrap(); |
| 302 | std::fs::write(&path, file).unwrap(); |
| 303 | } |
| 304 | let text = separators(&dir.display().to_string()); |
| 305 | (dir, text) |
| 306 | } |
| 307 | |
| 308 | fn names(found: &Found) -> Vec<&str> { |
| 309 | found.files.iter().map(|(name, _)| name.as_str()).collect() |
| 310 | } |
| 311 | |
| 312 | fn lines(text: &str) -> Vec<String> { |
| 313 | text.lines().map(str::to_owned).collect() |
| 314 | } |
| 315 | |
| 316 | #[test] |
| 317 | fn parts_match_stars_marks_and_sets() { |
| 318 | assert!(matches_part("*.txt", "a.txt")); |
| 319 | assert!(matches_part("*.txt", ".hidden.txt")); |
| 320 | assert!(!matches_part("*.txt", "a.txt.bak")); |
| 321 | assert!(matches_part("a?c", "abc")); |
| 322 | assert!(!matches_part("a?c", "ac")); |
| 323 | assert!(matches_part("[abc]x", "bx")); |
| 324 | assert!(!matches_part("[!abc]x", "bx")); |
| 325 | assert!(matches_part("[^abc]x", "dx")); |
| 326 | assert!(matches_part("v[0-9]", "v7")); |
| 327 | assert!(!matches_part("v[0-9]", "vx")); |
| 328 | assert!(matches_part("[]]", "]")); |
| 329 | assert!(matches_part("a[b", "a[b")); |
| 330 | assert!(matches_part("a**b", "axyzb")); |
| 331 | } |
| 332 | |
| 333 | #[test] |
| 334 | fn globstar_reaches_any_depth() { |
| 335 | assert!(matches("/w/**/*.rs", "/w/main.rs")); |
| 336 | assert!(matches("/w/**/*.rs", "/w/a/b/c.rs")); |
| 337 | assert!(!matches("/w/**/*.rs", "/x/a.rs")); |
| 338 | assert!(matches("/w/*/x", "/w/a/x")); |
| 339 | assert!(!matches("/w/*/x", "/w/a/b/x")); |
| 340 | assert!(matches("/w/**", "/w/a/b")); |
| 341 | } |
| 342 | |
| 343 | #[test] |
| 344 | fn lines_resolve_against_the_workspace_and_home() { |
| 345 | assert_eq!(resolve("dist/", "/w/repo", "/home/u"), "/w/repo/dist"); |
| 346 | assert_eq!(resolve("./a/../b/*.txt", "/w/repo/", "/home/u"), "/w/repo/b/*.txt"); |
| 347 | assert_eq!(resolve("~/out", "/w", "/home/u"), "/home/u/out"); |
| 348 | assert_eq!(resolve("~", "/w", "/home/u"), "/home/u"); |
| 349 | assert_eq!(resolve("/abs/x", "/w", "/home/u"), "/abs/x"); |
| 350 | assert_eq!(resolve("../other", "/w/repo", "/h"), "/w/other"); |
| 351 | } |
| 352 | |
| 353 | #[test] |
| 354 | #[cfg(windows)] |
| 355 | fn windows_paths_take_either_separator_and_any_case() { |
| 356 | assert_eq!(resolve("dist\\out", "C:\\w\\repo", "C:\\Users\\u"), "C:/w/repo/dist/out"); |
| 357 | assert_eq!(resolve("D:\\x\\*.txt", "C:\\w", "C:\\h"), "D:/x/*.txt"); |
| 358 | assert!(matches("C:/w/*.TXT", "c:/W/a.txt")); |
| 359 | assert_eq!(search_path("C:/w/**/*.txt"), "C:/w"); |
| 360 | } |
| 361 | |
| 362 | #[test] |
| 363 | fn search_paths_stop_before_the_first_glob() { |
| 364 | assert_eq!(search_path("/w/dist/**/*.js"), "/w/dist"); |
| 365 | assert_eq!(search_path("/w/dist/a.txt"), "/w/dist/a.txt"); |
| 366 | assert_eq!(search_path("/w/d?st/a"), "/w"); |
| 367 | assert_eq!(search_path("/*.txt"), "/"); |
| 368 | } |
| 369 | |
| 370 | #[test] |
| 371 | fn the_root_is_picked_as_upload_artifact_does() { |
| 372 | let s = |v: &[&str]| v.iter().map(|x| x.to_string()).collect::<Vec<_>>(); |
| 373 | // One file named: its folder. |
| 374 | assert_eq!(root_dir(&s(&["/w/dist/a.txt"]), &s(&["/w/dist/a.txt"])).unwrap(), "/w/dist"); |
| 375 | // One folder, or one glob: the search path. |
| 376 | assert_eq!(root_dir(&s(&["/w/dist"]), &s(&["/w/dist/a.txt"])).unwrap(), "/w/dist"); |
| 377 | assert_eq!(root_dir(&s(&["/w/dist"]), &s(&["/w/dist/a", "/w/dist/b"])).unwrap(), "/w/dist"); |
| 378 | // Several: what they have in common. |
| 379 | assert_eq!(root_dir(&s(&["/w/a/x.txt", "/w/b/c"]), &s(&["/w/a/x.txt", "/w/b/c/y"])).unwrap(), "/w"); |
| 380 | assert_eq!(root_dir(&s(&["/w/a", "/w/a/sub"]), &[]).unwrap(), "/w/a"); |
| 381 | assert_eq!(root_dir(&s(&["/a", "/b"]), &[]).unwrap(), "/"); |
| 382 | assert!(root_dir(&[], &[]).is_none()); |
| 383 | } |
| 384 | |
| 385 | #[test] |
| 386 | fn finding_files_takes_folders_globs_and_exclusions() { |
| 387 | let (dir, w) = tree(&["dist/a.js", "dist/sub/b.js", "dist/sub/c.map", "dist/.env", "dist/.cache/x", "src/main.rs", "README.md", ".hidden/y"]); |
| 388 | let found = find(&lines("dist"), &w, "/h", false); |
| 389 | assert_eq!(found.root, format!("{w}/dist")); |
| 390 | assert_eq!(names(&found), ["a.js", "sub/b.js", "sub/c.map"]); |
| 391 | |
| 392 | let found = find(&lines("dist\n!dist/**/*.map"), &w, "/h", false); |
| 393 | assert_eq!(names(&found), ["a.js", "sub/b.js"]); |
| 394 | // Leaving a folder out leaves out what is in it. |
| 395 | let found = find(&lines("dist\n!dist/sub"), &w, "/h", false); |
| 396 | assert_eq!(names(&found), ["a.js"]); |
| 397 | |
| 398 | let found = find(&lines("dist"), &w, "/h", true); |
| 399 | assert!(names(&found).contains(&".env") && names(&found).contains(&".cache/x")); |
| 400 | |
| 401 | let found = find(&lines("**/*.js"), &w, "/h", false); |
| 402 | assert_eq!(found.root, w); |
| 403 | assert_eq!(names(&found), ["dist/a.js", "dist/sub/b.js"]); |
| 404 | |
| 405 | // Stored relative to the search path, the part before the glob. |
| 406 | let found = find(&lines("dist/s?b/[bc].*"), &w, "/h", false); |
| 407 | assert_eq!(found.root, format!("{w}/dist")); |
| 408 | assert_eq!(names(&found), ["sub/b.js", "sub/c.map"]); |
| 409 | |
| 410 | // A glob matching a folder takes what is in it. |
| 411 | let found = find(&lines("d*"), &w, "/h", false); |
| 412 | assert_eq!(names(&found), ["dist/a.js", "dist/sub/b.js", "dist/sub/c.map"]); |
| 413 | |
| 414 | let found = find(&lines("README.md"), &w, "/h", false); |
| 415 | assert_eq!(found.root, w); |
| 416 | assert_eq!(names(&found), ["README.md"]); |
| 417 | |
| 418 | let found = find(&lines("README.md\nsrc/"), &w, "/h", false); |
| 419 | assert_eq!(names(&found), ["README.md", "src/main.rs"]); |
| 420 | |
| 421 | // A hidden path named outright is taken. |
| 422 | let found = find(&lines(".hidden"), &w, "/h", false); |
| 423 | assert_eq!(names(&found), ["y"]); |
| 424 | let found = find(&lines("dist/.env"), &w, "/h", false); |
| 425 | assert_eq!(names(&found), [".env"]); |
| 426 | |
| 427 | assert!(find(&lines("nothing/*"), &w, "/h", false).files.is_empty()); |
| 428 | let _ = std::fs::remove_dir_all(&dir); |
| 429 | } |
| 430 | } |