Sign in

pulls/pr_01m47d24b0e6n91zwymwxg0vpx

public

pr_01m47d24b0e6n91zwymwxg0vpx/crates/runner/src/actions/blobs.rs

295 lines13,878 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

A repository has its own sidebar, as settings do1//! Artifacts and the cache: `actions/upload-artifact`,
2//! `actions/download-artifact` and `actions/cache`, done natively against
3//! g1t, which keeps them in R2. Artifacts belong to the run; cache entries
4//! to the repository, found by exact key or by the newest under a
5//! `restore-keys` prefix.
6
7use std::collections::BTreeMap;
8use std::io::Read;
9use std::path::Path;
10use std::process::Command;
11use std::time::Duration;
12
13use super::process::{self, Commands, Ended};
14use super::{Job, Post, PostRun};
15
16/// The largest upload g1t takes, as the platform limits a request.
17const MAX_UPLOAD: u64 = 60 * 1024 * 1024;
18
19fn lines(text: &str) -> Vec<String> {
20 text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect()
21}
22
23fn quote(text: &str) -> String {
24 format!("'{}'", text.replace('\'', "'\\''"))
25}
26
27fn safe_name(name: &str) -> bool {
28 !name.is_empty() && name.len() <= 200 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ' ')) && !name.starts_with('.')
29}
30
31impl Job {
32 fn url(&self, rest: &str) -> String {
33 format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job)
34 }
35
36 fn auth(&self) -> String {
37 format!("Bearer {}", self.log.api.token)
38 }
39
40 /// Runs a shell line, logging its output; whether it succeeded.
41 fn shell(&mut self, script: &str) -> bool {
42 let mut command = Command::new("bash");
43 command.args(["-c", script]).current_dir(&self.workspace);
44 let mut commands = Commands::default();
45 matches!(process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
46 }
47
48 fn upload(&mut self, rest: &str, file: &Path) -> Result<u64, String> {
49 let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len();
50 if size > MAX_UPLOAD {
51 return Err(format!("it is {} MB, more than g1t takes at once ({} MB)", size / 1_048_576, MAX_UPLOAD / 1_048_576));
52 }
53 let bytes = std::fs::read(file).map_err(|e| e.to_string())?;
54 ureq::put(&self.url(rest))
55 .set("authorization", &self.auth())
56 .set("content-type", "application/gzip")
57 .timeout(Duration::from_secs(600))
58 .send_bytes(&bytes)
59 .map_err(|e| e.to_string())?;
60 Ok(size)
61 }
62
63 /// Downloads into `file`; `Ok(None)` when there is nothing there.
64 fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> {
65 let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).call() {
66 Ok(response) => response,
67 Err(ureq::Error::Status(404, _)) => return Ok(None),
68 Err(error) => return Err(error.to_string()),
69 };
70 let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default();
71 let mut bytes = Vec::new();
72 response.into_reader().take(MAX_UPLOAD * 2).read_to_end(&mut bytes).map_err(|e| e.to_string())?;
73 std::fs::write(file, bytes).map_err(|e| e.to_string())?;
74 Ok(Some(matched))
75 }
76
77 /// `actions/upload-artifact`.
78 pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
79 let name = with.get("name").filter(|n| !n.is_empty()).cloned().unwrap_or_else(|| "artifact".into());
80 if !safe_name(&name) {
81 self.log.line(&format!("##[error]`{name}` is not an artifact name g1t takes: letters, digits, spaces, `-`, `_` and `.`."));
82 return (false, BTreeMap::new());
83 }
84 let paths = lines(with.get("path").map(String::as_str).unwrap_or_default());
85 let missing = with.get("if-no-files-found").map(String::as_str).unwrap_or("warn").to_owned();
86 let archive = self.temp.join(format!("artifact-{name}.tgz"));
87 // As on GitHub: one folder uploads its contents; otherwise paths
88 // are kept relative to the workspace.
89 let single_dir = paths.len() == 1 && self.workspace.join(&paths[0]).is_dir();
90 let script = if single_dir {
91 format!("tar -czf {} -C {} .", quote(&archive.display().to_string()), quote(&paths[0]))
92 } else {
93 let patterns: Vec<String> = paths.iter().filter(|p| !p.starts_with('!')).map(|p| p.replace('\'', "")).collect();
94 format!(
95 "shopt -s globstar nullglob dotglob; files=( {} ); if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; tar -czf {} -- \"${{files[@]}}\"",
96 patterns.join(" "),
97 quote(&archive.display().to_string())
98 )
99 };
100 let mut command = Command::new("bash");
101 command.args(["-c", &script]).current_dir(&self.workspace);
102 let mut commands = Commands::default();
103 match process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands) {
104 Ok(Ended::Exited(0)) => {}
105 Ok(Ended::Exited(3)) => {
106 let message = format!("No files were found at {}.", paths.join(", "));
107 return match missing.as_str() {
108 "error" => {
109 self.log.line(&format!("##[error]{message}"));
110 (false, BTreeMap::new())
111 }
112 "ignore" => (true, BTreeMap::new()),
113 _ => {
114 self.log.line(&format!("##[warning]{message} Nothing was uploaded."));
115 (true, BTreeMap::new())
116 }
117 };
118 }
119 _ => {
120 self.log.line("##[error]The files could not be packed.");
121 return (false, BTreeMap::new());
122 }
123 }
124 match self.upload(&format!("artifacts/{name}"), &archive) {
125 Ok(size) => {
126 self.log.line(&format!("Uploaded artifact {name} ({} KB). It is kept with the run for 14 days.", size.div_ceil(1024)));
127 let mut outputs = BTreeMap::new();
128 outputs.insert("artifact-id".into(), name.clone());
129 (true, outputs)
130 }
131 Err(error) => {
132 self.log.line(&format!("##[error]The artifact could not be uploaded: {error}"));
133 (false, BTreeMap::new())
134 }
135 }
136 }
137
138 /// `actions/download-artifact`: one by name, or every artifact of the
139 /// run, each into a folder of its name.
140 pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
141 let dest = with.get("path").filter(|p| !p.is_empty()).map_or(self.workspace.clone(), |p| self.workspace.join(p));
142 let names: Vec<String> = match with.get("name").filter(|n| !n.is_empty()) {
143 Some(name) => vec![name.clone()],
144 None => {
145 let listed = ureq::get(&self.url("artifacts")).set("authorization", &self.auth()).call().ok().and_then(|r| r.into_json::<Vec<serde_json::Value>>().ok()).unwrap_or_default();
146 listed.iter().filter_map(|a| a["name"].as_str().map(str::to_owned)).collect()
147 }
148 };
149 let merge = with.get("merge-multiple").is_some_and(|m| m == "true");
150 let single = with.get("name").is_some_and(|n| !n.is_empty());
151 for name in &names {
152 let archive = self.temp.join(format!("download-{name}.tgz"));
153 match self.download(&format!("artifacts/{name}"), &archive) {
154 Ok(Some(_)) => {}
155 Ok(None) => {
156 self.log.line(&format!("##[error]This run has no artifact called {name}."));
157 return (false, BTreeMap::new());
158 }
159 Err(error) => {
160 self.log.line(&format!("##[error]The artifact {name} could not be downloaded: {error}"));
161 return (false, BTreeMap::new());
162 }
163 }
164 let target = if single || merge { dest.clone() } else { dest.join(name) };
165 let _ = std::fs::create_dir_all(&target);
166 if !self.shell(&format!("tar -xzf {} -C {}", quote(&archive.display().to_string()), quote(&target.display().to_string()))) {
167 return (false, BTreeMap::new());
168 }
169 self.log.line(&format!("Downloaded artifact {name} into {}", target.display()));
170 }
171 let mut outputs = BTreeMap::new();
172 outputs.insert("download-path".into(), dest.display().to_string());
173 (true, outputs)
174 }
175
176 fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> {
177 let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into());
178 lines(with.get("path").map(String::as_str).unwrap_or_default())
179 .into_iter()
180 .map(|p| {
181 let p = if let Some(rest) = p.strip_prefix("~/") { format!("{home}/{rest}") } else { p };
182 if p.starts_with('/') { p } else { self.workspace.join(p).display().to_string() }
183 })
184 .collect()
185 }
186
187 /// `actions/cache` and `actions/cache/restore`: restores what it can,
188 /// and for `actions/cache`, saves at the end of the job on a miss.
189 pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) {
190 let key = with.get("key").cloned().unwrap_or_default();
191 if key.is_empty() {
192 self.log.line("##[error]The cache needs a `key`.");
193 return (false, BTreeMap::new());
194 }
195 let paths = self.cache_paths(with);
196 let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default());
197 let query = format!(
198 "cache?key={}&restore={}",
199 urlencode(&key),
200 urlencode(&restore.join("\n"))
201 );
202 let archive = self.temp.join("cache-restore.tgz");
203 let mut outputs = BTreeMap::new();
204 let exact = match self.download(&query, &archive) {
205 Ok(Some(matched)) => {
206 let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true");
207 if !lookup_only && !self.shell(&format!("tar -xzPf {}", quote(&archive.display().to_string()))) {
208 self.log.line("##[warning]The cache was found but could not be unpacked.");
209 }
210 self.log.line(&format!("Cache restored from key: {matched}"));
211 outputs.insert("cache-matched-key".into(), matched.clone());
212 matched == key
213 }
214 Ok(None) => {
215 self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", ")));
216 if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") {
217 self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set.");
218 return (false, outputs);
219 }
220 false
221 }
222 Err(error) => {
223 self.log.line(&format!("##[warning]The cache could not be read: {error}"));
224 false
225 }
226 };
227 outputs.insert("cache-hit".into(), exact.to_string());
228 outputs.insert("cache-primary-key".into(), key.clone());
229 if save_after && !exact {
230 self.posts.push(Post {
231 name: format!("Post {title}"),
232 condition: "success()".into(),
233 env: BTreeMap::new(),
234 run: PostRun::CacheSave { key, paths },
235 });
236 }
237 (true, outputs)
238 }
239
240 /// Saves paths under a key, unless the key is taken.
241 pub(crate) fn cache_save(&mut self, key: &str, paths: &[String]) -> bool {
242 if paths.is_empty() {
243 self.log.line("##[warning]Nothing to cache: no `path`.");
244 return true;
245 }
246 let archive = self.temp.join("cache-save.tgz");
247 let list: Vec<String> = paths.iter().filter(|p| Path::new(p).exists()).map(|p| quote(p)).collect();
248 if list.is_empty() {
249 self.log.line("##[warning]None of the cache's paths exist; nothing was saved.");
250 return true;
251 }
252 if !self.shell(&format!("tar -czPf {} {}", quote(&archive.display().to_string()), list.join(" "))) {
253 self.log.line("##[warning]The cache could not be packed; nothing was saved.");
254 return true;
255 }
256 match self.upload(&format!("cache?key={}", urlencode(key)), &archive) {
257 Ok(size) => self.log.line(&format!("Cache saved with key: {key} ({} KB)", size.div_ceil(1024))),
258 // A cache that cannot be saved does not fail the job, as on GitHub.
259 Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")),
260 }
261 true
262 }
263
264 /// `actions/cache/save`.
265 pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
266 let key = with.get("key").cloned().unwrap_or_default();
267 let paths = self.cache_paths(with);
268 (self.cache_save(&key, &paths), BTreeMap::new())
269 }
270}
271
272fn urlencode(text: &str) -> String {
273 let mut out = String::new();
274 for byte in text.bytes() {
275 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
276 out.push(byte as char);
277 } else {
278 out.push_str(&format!("%{byte:02X}"));
279 }
280 }
281 out
282}
283
284#[cfg(test)]
285mod tests {
286 use super::*;
287
288 #[test]
289 fn keys_are_encoded_and_names_checked() {
290 assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202");
291 assert!(safe_name("coverage report"));
292 assert!(!safe_name("../etc"));
293 assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]);
294 }
295}