Skip to content
Sign in

flagon-io/g1t

public
0
1,055 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R21//! Artifacts and the cache: `actions/upload-artifact`, its `merge`,
A repository has its own sidebar, as settings do2//! `actions/download-artifact` and `actions/cache`, done natively against
Fast pages, required checks on the branch, self-hosted runners, honest incidents3//! g1t. Artifacts belong to the run; cache entries to the repository, found
4//! by exact key or by the newest under a `restore-keys` prefix.
5//!
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R26//! An artifact is a ZIP file, as GitHub's v4 actions make it (zip.rs), of
7//! the files its `path` finds (glob.rs): uploaded in parts with its SHA-256,
8//! and downloaded straight to a file before it is unpacked.
9//!
Fast pages, required checks on the branch, self-hosted runners, honest incidents10//! A cache entry is a tar archive, compressed with zstd where the machine
11//! has it (gzip otherwise), of up to 2 GB: uploaded in parts, and
12//! downloaded straight to a file. Its `path` takes globs (`**` included)
Merge branch 'worktree-agent-a3abfcce648e87dca'13//! and `!` patterns that leave paths out, as `actions/cache` does. Each is
14//! saved and found under a version, the hash of its `path` and compression,
15//! and g1t keeps it under the ref whose run saved it.
A repository has its own sidebar, as settings do16
17use std::collections::BTreeMap;
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R218use std::io::{Read, Write};
A repository has its own sidebar, as settings do19use std::path::Path;
20use std::process::Command;
Fast pages, required checks on the branch, self-hosted runners, honest incidents21use std::time::{Duration, Instant};
A repository has its own sidebar, as settings do22
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R223use sha2::{Digest, Sha256};
24
A repository has its own sidebar, as settings do25use super::process::{self, Commands, Ended};
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R226use super::{Job, Post, PostRun, glob, zip};
A repository has its own sidebar, as settings do27
Fast pages, required checks on the branch, self-hosted runners, honest incidents28/// The largest cache entry, compressed (`g1t_contracts::actions::CACHE_MAX_ENTRY_BYTES`).
29const MAX_CACHE_ENTRY: u64 = 2 * 1024 * 1024 * 1024;
A repository has its own sidebar, as settings do30
31fn lines(text: &str) -> Vec<String> {
32 text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect()
33}
34
35fn quote(text: &str) -> String {
36 format!("'{}'", text.replace('\'', "'\\''"))
37}
38
39impl Job {
40 fn url(&self, rest: &str) -> String {
41 format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job)
42 }
43
44 fn auth(&self) -> String {
45 format!("Bearer {}", self.log.api.token)
46 }
47
48 /// Runs a shell line, logging its output; whether it succeeded.
49 fn shell(&mut self, script: &str) -> bool {
50 let mut command = Command::new("bash");
51 command.args(["-c", script]).current_dir(&self.workspace);
52 let mut commands = Commands::default();
Merge Actions: cross-repo workflows and actions, release and deployment triggers, step timeouts53 matches!(process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
A repository has its own sidebar, as settings do54 }
55
Fast pages, required checks on the branch, self-hosted runners, honest incidents56 /// Downloads into `file`, as it comes; `Ok(None)` when there is
57 /// nothing there.
A repository has its own sidebar, as settings do58 fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> {
Fast pages, required checks on the branch, self-hosted runners, honest incidents59 let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(1800)).call() {
A repository has its own sidebar, as settings do60 Ok(response) => response,
61 Err(ureq::Error::Status(404, _)) => return Ok(None),
62 Err(error) => return Err(error.to_string()),
63 };
64 let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default();
Fast pages, required checks on the branch, self-hosted runners, honest incidents65 let mut out = std::fs::File::create(file).map_err(|e| e.to_string())?;
66 std::io::copy(&mut response.into_reader().take(MAX_CACHE_ENTRY + 1024), &mut out).map_err(|e| e.to_string())?;
A repository has its own sidebar, as settings do67 Ok(Some(matched))
68 }
69
Fast pages, required checks on the branch, self-hosted runners, honest incidents70 /// Saves `file` as the cache entry `key`, in parts. `Ok(false)` when
71 /// the key is already cached.
Merge branch 'worktree-agent-a3abfcce648e87dca'72 fn upload_cache(&mut self, key: &str, version: &str, file: &Path) -> Result<bool, String> {
Fast pages, required checks on the branch, self-hosted runners, honest incidents73 let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len();
74 if size > MAX_CACHE_ENTRY {
75 return Err(format!("it is {} MB, more than a cache entry may be ({} MB)", size / 1_048_576, MAX_CACHE_ENTRY / 1_048_576));
76 }
Merge branch 'worktree-agent-a3abfcce648e87dca'77 let started = match ureq::post(&self.url(&format!("cache/uploads?key={}&size={size}&version={}", urlencode(key), urlencode(version))))
Fast pages, required checks on the branch, self-hosted runners, honest incidents78 .set("authorization", &self.auth())
79 .timeout(Duration::from_secs(60))
80 .call()
81 {
82 Ok(response) => response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?,
83 Err(ureq::Error::Status(409, _)) => return Ok(false),
84 Err(ureq::Error::Status(_, response)) => return Err(refusal(response)),
85 Err(error) => return Err(error.to_string()),
86 };
87 let id = started["id"].as_str().unwrap_or_default().to_owned();
88 let upload = started["upload"].as_str().unwrap_or_default().to_owned();
89 let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024);
90 let base = format!("cache/uploads/{}", urlencode(&id));
91 let sent = self.send_parts(&base, &upload, file, part_bytes);
92 let parts = match sent {
93 Ok(parts) => parts,
94 Err(error) => {
95 let _ = ureq::delete(&self.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &self.auth()).call();
96 return Err(error);
97 }
98 };
99 ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload))))
100 .set("authorization", &self.auth())
101 .timeout(Duration::from_secs(120))
102 .send_json(serde_json::json!({ "size": size, "parts": parts }))
103 .map_err(|e| match e {
104 ureq::Error::Status(_, response) => refusal(response),
105 other => other.to_string(),
106 })?;
107 Ok(true)
108 }
109
110 /// Sends `file` in parts of `part_bytes`; each part's number and etag.
111 fn send_parts(&mut self, base: &str, upload: &str, file: &Path, part_bytes: u64) -> Result<Vec<serde_json::Value>, String> {
112 let mut reader = std::fs::File::open(file).map_err(|e| e.to_string())?;
113 let mut parts = Vec::new();
114 let mut buffer = vec![0u8; part_bytes as usize];
115 for number in 1u32.. {
116 let mut filled = 0;
117 while filled < buffer.len() {
118 let read = reader.read(&mut buffer[filled..]).map_err(|e| e.to_string())?;
119 if read == 0 {
120 break;
121 }
122 filled += read;
123 }
124 if filled == 0 && number > 1 {
125 break;
126 }
127 let url = self.url(&format!("{base}/{number}?upload={}", urlencode(upload)));
128 // A part that fails is sent again, twice at most.
129 let mut tries = 0;
130 let answer = loop {
131 tries += 1;
132 match ureq::put(&url).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).send_bytes(&buffer[..filled]) {
133 Ok(response) => break response.into_json::<serde_json::Value>().map_err(|e| e.to_string())?,
134 Err(ureq::Error::Status(status, response)) if status < 500 => return Err(refusal(response)),
135 Err(error) if tries >= 3 => return Err(error.to_string()),
136 Err(_) => std::thread::sleep(Duration::from_secs(2 * tries)),
137 }
138 };
139 parts.push(serde_json::json!({ "part": number, "etag": answer["etag"] }));
140 if filled < buffer.len() {
141 break;
142 }
143 }
144 Ok(parts)
145 }
146
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2147 /// A value of the `github` context, as text.
148 fn github_value(&self, key: &str) -> String {
149 match self.contexts.get("github").and_then(|github| github.get(key)) {
150 Some(serde_json::Value::String(text)) => text.clone(),
151 Some(serde_json::Value::Null) | None => String::new(),
152 Some(other) => other.to_string(),
153 }
154 }
155
156 fn home(&self) -> String {
157 self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into())
158 }
159
160 /// `actions/upload-artifact`: the files `path` names, packed into one
161 /// ZIP and uploaded in parts.
A repository has its own sidebar, as settings do162 pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2163 let name = input(with, "name").unwrap_or("artifact").to_owned();
164 let missing = input(with, "if-no-files-found").unwrap_or("warn").to_ascii_lowercase();
165 let checked = check_name(&name)
166 .and_then(|()| keep(with, true))
167 .and_then(|keep| if matches!(missing.as_str(), "warn" | "error" | "ignore") { Ok(keep) } else { Err(format!("`if-no-files-found` is `{missing}`; it takes warn, error or ignore.")) });
168 let keep = match checked {
169 Ok(keep) => keep,
170 Err(message) => {
171 self.log.line(&format!("##[error]{message}"));
172 return (false, BTreeMap::new());
A repository has its own sidebar, as settings do173 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2174 };
175 let paths = lines(input(with, "path").unwrap_or_default());
176 let found = glob::find(&paths, &self.workspace.display().to_string(), &self.home(), keep.hidden);
177 if found.files.is_empty() {
178 let message = format!("No files were found with the provided path: {}. No artifacts will be uploaded.", paths.join(", "));
179 return match missing.as_str() {
180 "error" => {
181 self.log.line(&format!("##[error]{message}"));
182 (false, BTreeMap::new())
183 }
184 "ignore" => {
185 self.log.line(&message);
186 (true, BTreeMap::new())
187 }
188 _ => {
189 self.log.line(&format!("##[warning]{message}"));
190 (true, BTreeMap::new())
191 }
192 };
193 }
194 self.log.line(&format!("Found {} with the provided path, stored relative to {}.", count(found.files.len(), "file"), found.root));
195 self.send_artifact(&name, &found.files, &keep)
196 }
197
198 /// Packs `files` into a ZIP and uploads it as the artifact `name`;
199 /// whether it worked, and the outputs `upload-artifact` sets.
200 fn send_artifact(&mut self, name: &str, files: &[(String, std::path::PathBuf)], keep: &Keep) -> (bool, BTreeMap<String, String>) {
201 let archive = self.temp.join(format!("artifact-{}.zip", super::rand_id()));
202 let sent = zip::write(&archive, files, keep.level)
203 .map_err(|e| format!("The files could not be packed: {e}"))
204 .and_then(|packed| self.post_artifact(name, &archive, keep).map(|done| (packed, done)));
205 let _ = std::fs::remove_file(&archive);
206 let (packed, done) = match sent {
207 Ok(sent) => sent,
208 Err(error) => {
209 self.log.line(&format!("##[error]{error}"));
A repository has its own sidebar, as settings do210 return (false, BTreeMap::new());
211 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2212 };
213 let kept = done.retention.map(|days| format!(" It is kept for {}.", count(days as usize, "day"))).unwrap_or_default();
214 self.log.line(&format!("Uploaded artifact {name} ({}, {}).{kept}", megabytes(packed.size), count(packed.files, "file")));
215 let url = format!(
216 "{}/{}/actions/runs/{}/artifacts/{}",
217 self.github_value("server_url").trim_end_matches('/'),
218 self.github_value("repository"),
219 self.github_value("run_id"),
220 done.id
221 );
222 self.log.line(&format!("Artifact download URL: {url}"));
223 let mut outputs = BTreeMap::new();
224 outputs.insert("artifact-id".into(), done.id.to_string());
225 outputs.insert("artifact-url".into(), url);
226 outputs.insert("artifact-digest".into(), done.digest);
227 (true, outputs)
228 }
229
230 /// Uploads the ZIP file `archive` as the artifact `name`, in parts,
231 /// giving the upload up when a part or the end fails.
232 fn post_artifact(&mut self, name: &str, archive: &Path, keep: &Keep) -> Result<Sent, String> {
233 let size = std::fs::metadata(archive).map_err(|e| e.to_string())?.len();
234 let digest = sha256_file(archive).map_err(|e| e.to_string())?;
235 let query = format!(
236 "artifacts/uploads?name={}&size={size}&retention_days={}&overwrite={}&format=zip",
237 urlencode(name),
238 keep.retention,
239 keep.overwrite
240 );
241 let failed = |e: ureq::Error| match e {
242 ureq::Error::Status(_, response) => format!("The artifact could not be uploaded: {}", refusal(response)),
243 other => format!("The artifact could not be uploaded: {other}"),
244 };
245 let started = ureq::post(&self.url(&query))
246 .set("authorization", &self.auth())
247 .timeout(Duration::from_secs(60))
248 .call()
249 .map_err(failed)?
250 .into_json::<serde_json::Value>()
251 .map_err(|e| e.to_string())?;
252 let id = number(&started["id"]).ok_or("g1t did not say which artifact the upload is")?;
253 let upload = started["upload"].as_str().unwrap_or_default().to_owned();
254 let part_bytes = started["part_bytes"].as_u64().filter(|n| *n > 0).unwrap_or(32 * 1024 * 1024);
255 let retention = started["retention_days"].as_u64();
256 if let Some(applied) = retention
257 && keep.retention != 0
258 && applied != u64::from(keep.retention)
259 {
260 self.log.line(&format!(
261 "##[notice]The artifact is kept for {}, not the {} asked for: the most this repository keeps artifacts.",
262 count(applied as usize, "day"),
263 keep.retention
264 ));
A repository has its own sidebar, as settings do265 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2266 let base = format!("artifacts/uploads/{id}");
267 let give_up = |job: &Job| {
268 let _ = ureq::delete(&job.url(&format!("{base}?upload={}", urlencode(&upload)))).set("authorization", &job.auth()).call();
269 };
270 let parts = match self.send_parts(&base, &upload, archive, part_bytes) {
271 Ok(parts) => parts,
272 Err(error) => {
273 give_up(self);
274 return Err(format!("The artifact could not be uploaded: {error}"));
A repository has its own sidebar, as settings do275 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2276 };
277 let done = ureq::post(&self.url(&format!("{base}/complete?upload={}", urlencode(&upload))))
278 .set("authorization", &self.auth())
279 .timeout(Duration::from_secs(120))
280 .send_json(serde_json::json!({ "size": size, "parts": parts, "digest": format!("sha256:{digest}") }));
281 let done = match done {
282 Ok(response) => response.into_json::<serde_json::Value>().unwrap_or_default(),
A repository has its own sidebar, as settings do283 Err(error) => {
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2284 give_up(self);
285 return Err(failed(error));
286 }
287 };
288 Ok(Sent { id: number(&done["id"]).unwrap_or(id), digest, retention: retention.map(|d| d as u32).or((keep.retention != 0).then_some(keep.retention)) })
289 }
290
291 /// The artifacts of this run, or of the run `run_id` of this
292 /// repository; one per name, the newest.
293 fn list_artifacts(&mut self, run_id: Option<&str>) -> Result<Vec<Listed>, String> {
294 let rest = match run_id {
295 Some(run) => format!("artifacts?run_id={}", urlencode(run)),
296 None => "artifacts".to_owned(),
297 };
298 let listed = ureq::get(&self.url(&rest))
299 .set("authorization", &self.auth())
300 .timeout(Duration::from_secs(60))
301 .call()
302 .map_err(|e| match e {
303 ureq::Error::Status(_, response) => refusal(response),
304 other => other.to_string(),
305 })?
306 .into_json::<Vec<serde_json::Value>>()
307 .map_err(|e| e.to_string())?;
308 Ok(newest(listed.iter().filter_map(Listed::from_json).collect()))
309 }
310
311 /// Downloads an artifact to a file, as it comes, and unpacks it into
312 /// `into`.
313 fn fetch_artifact(&mut self, artifact: &Listed, into: &Path) -> Result<(), String> {
314 let file = self.temp.join(format!("download-{}.zip", super::rand_id()));
315 let fetched = self.fetch_to(artifact, &file).and_then(|format| {
316 if format == "tgz" {
317 std::fs::create_dir_all(into).map_err(|e| e.to_string())?;
318 let script = format!("tar -xzf {} -C {}", quote(&file.display().to_string()), quote(&into.display().to_string()));
319 if self.shell(&script) { Ok(()) } else { Err("it could not be unpacked".into()) }
320 } else {
321 zip::extract(&file, into).map(|_| ()).map_err(|e| format!("it could not be unpacked: {e}"))
322 }
323 });
324 let _ = std::fs::remove_file(&file);
325 fetched
326 }
327
328 /// Downloads an artifact into `file`, checking its digest; how it is
329 /// packed, `zip` or `tgz`.
330 fn fetch_to(&mut self, artifact: &Listed, file: &Path) -> Result<String, String> {
331 let response = match ureq::get(&self.url(&format!("artifacts/{}/download", artifact.id)))
332 .set("authorization", &self.auth())
333 .timeout(Duration::from_secs(4 * 3600))
334 .call()
335 {
336 Ok(response) => response,
337 Err(ureq::Error::Status(404, _)) => return Err("it is gone: it expired or was deleted".into()),
338 Err(ureq::Error::Status(_, response)) => return Err(refusal(response)),
339 Err(error) => return Err(error.to_string()),
340 };
341 let format = response.header("x-g1t-format").map(str::to_owned).unwrap_or_else(|| artifact.format.clone());
342 let mut reader = response.into_reader();
343 let mut out = std::io::BufWriter::new(std::fs::File::create(file).map_err(|e| e.to_string())?);
344 let mut hasher = Sha256::new();
345 let mut buffer = vec![0u8; 256 * 1024];
346 loop {
347 let n = reader.read(&mut buffer).map_err(|e| e.to_string())?;
348 if n == 0 {
349 break;
A repository has its own sidebar, as settings do350 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2351 hasher.update(&buffer[..n]);
352 out.write_all(&buffer[..n]).map_err(|e| e.to_string())?;
A repository has its own sidebar, as settings do353 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2354 out.flush().map_err(|e| e.to_string())?;
355 let got = hex::encode(hasher.finalize());
356 if let Some(expected) = artifact.digest.as_deref().map(|d| d.trim_start_matches("sha256:"))
357 && !expected.is_empty()
358 && !expected.eq_ignore_ascii_case(&got)
359 {
360 self.log.line(&format!("##[warning]Artifact {} does not match its digest (sha256:{got}, not sha256:{expected}): it may have been changed since it was uploaded.", artifact.name));
361 }
362 Ok(format)
A repository has its own sidebar, as settings do363 }
364
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2365 /// Where artifacts are downloaded: `path`, under the workspace unless
366 /// absolute, `~` the home folder.
367 fn download_dir(&self, path: Option<&str>) -> std::path::PathBuf {
368 match path {
369 None => self.workspace.clone(),
370 Some(path) => {
371 let home = self.home();
372 let path = if path == "~" {
373 home
374 } else if let Some(rest) = path.strip_prefix("~/") {
375 format!("{}/{rest}", home.trim_end_matches('/'))
376 } else {
377 path.to_owned()
378 };
379 self.workspace.join(path)
A repository has its own sidebar, as settings do380 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2381 }
382 }
383
384 /// `actions/download-artifact`: one artifact by name straight into
385 /// `path`, or the run's artifacts (by `pattern` or `artifact-ids`) each
386 /// into a folder of its name, or all into `path` with `merge-multiple`.
387 pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
388 let fail = |job: &mut Job, message: &str| {
389 job.log.line(&format!("##[error]{message}"));
390 (false, BTreeMap::new())
391 };
392 let name = input(with, "name");
393 let ids = match artifact_ids(input(with, "artifact-ids").unwrap_or_default()) {
394 Ok(ids) => ids,
395 Err(message) => return fail(self, &message),
396 };
397 if name.is_some() && !ids.is_empty() {
398 return fail(self, "Inputs 'name' and 'artifact-ids' cannot be used together. Please specify only one.");
399 }
400 let merge = match flag(with, "merge-multiple", false) {
401 Ok(merge) => merge,
402 Err(message) => return fail(self, &message),
A repository has its own sidebar, as settings do403 };
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2404 let dest = self.download_dir(input(with, "path"));
405 // Another run's artifacts, as v4 has it: with a token and a run id.
406 let run_id = match (input(with, "github-token"), input(with, "run-id")) {
407 (Some(_), Some(run)) => {
408 let own = self.github_value("repository");
409 let repository = input(with, "repository").unwrap_or(&own);
410 if !repository.eq_ignore_ascii_case(&own) {
411 return fail(self, &format!("g1t downloads artifacts from other runs of the same repository only; {repository} is not this run's repository, {own}."));
A repository has its own sidebar, as settings do412 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2413 Some(run.to_owned())
A repository has its own sidebar, as settings do414 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2415 _ => None,
416 };
417 let listed = match self.list_artifacts(run_id.as_deref()) {
418 Ok(listed) => listed,
419 Err(error) => return fail(self, &format!("The artifacts could not be listed: {error}")),
420 };
421 let chosen: Vec<Listed> = if let Some(name) = name {
422 match listed.into_iter().find(|a| a.name == name) {
423 Some(artifact) => vec![artifact],
424 None => return fail(self, &format!("Unable to download artifact(s): Artifact not found for name: {name}")),
A repository has its own sidebar, as settings do425 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2426 } else if !ids.is_empty() {
427 let chosen: Vec<Listed> = listed.into_iter().filter(|a| ids.contains(&a.id)).collect();
428 if chosen.is_empty() {
429 return fail(self, "Unable to download artifact(s): None of the provided artifact IDs were found.");
430 }
431 if chosen.len() < ids.len() {
432 self.log.line(&format!("##[warning]Could only find {} of {} artifact IDs.", chosen.len(), ids.len()));
433 }
434 chosen
435 } else {
436 match input(with, "pattern") {
437 Some(pattern) => listed.into_iter().filter(|a| glob::matches_part(pattern, &a.name)).collect(),
438 None => listed,
439 }
440 };
A repository has its own sidebar, as settings do441 let mut outputs = BTreeMap::new();
442 outputs.insert("download-path".into(), dest.display().to_string());
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2443 if chosen.is_empty() {
444 match input(with, "pattern") {
445 Some(pattern) => self.log.line(&format!("No artifacts matched the pattern {pattern}; nothing was downloaded.")),
446 None => self.log.line("The run has no artifacts; nothing was downloaded."),
447 }
448 return (true, outputs);
449 }
450 let targets = download_targets(&dest, &chosen, name.is_some(), !ids.is_empty(), merge);
451 for (artifact, target) in chosen.iter().zip(targets) {
452 let Some(target) = target else {
453 return fail(self, &format!("The artifact {} cannot be downloaded into a folder of its name.", artifact.name));
454 };
455 if let Err(error) = self.fetch_artifact(artifact, &target) {
456 return fail(self, &format!("The artifact {} could not be downloaded: {error}", artifact.name));
457 }
458 self.log.line(&format!("Downloaded artifact {} ({}) into {}", artifact.name, megabytes(artifact.size), target.display()));
459 }
460 if chosen.len() > 1 {
461 self.log.line(&format!("Downloaded {}.", count(chosen.len(), "artifact")));
462 }
A repository has its own sidebar, as settings do463 (true, outputs)
464 }
465
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2466 /// `actions/upload-artifact/merge`: the run's artifacts matching
467 /// `pattern`, downloaded together and uploaded again as one.
468 pub(crate) fn merge_artifacts(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
469 let fail = |job: &mut Job, message: &str| {
470 job.log.line(&format!("##[error]{message}"));
471 (false, BTreeMap::new())
472 };
473 let name = input(with, "name").unwrap_or("merged-artifacts").to_owned();
474 let pattern = input(with, "pattern").unwrap_or("*").to_owned();
475 let options = check_name(&name).and_then(|()| {
476 let keep = keep(with, false)?;
477 Ok((keep, flag(with, "separate-directories", false)?, flag(with, "delete-merged", false)?))
478 });
479 let (keep, separate, delete) = match options {
480 Ok(options) => options,
481 Err(message) => return fail(self, &message),
482 };
483 let listed = match self.list_artifacts(None) {
484 Ok(listed) => listed,
485 Err(error) => return fail(self, &format!("The artifacts could not be listed: {error}")),
486 };
487 let chosen: Vec<Listed> = listed.into_iter().filter(|a| glob::matches_part(&pattern, &a.name)).collect();
488 if chosen.is_empty() {
489 return fail(self, &format!("No artifacts found matching pattern '{pattern}'."));
490 }
491 self.log.line(&format!("Merging {}: {}", count(chosen.len(), "artifact"), chosen.iter().map(|a| a.name.as_str()).collect::<Vec<_>>().join(", ")));
492 let folder = self.temp.join(format!("merge-{}", super::rand_id()));
493 let targets = download_targets(&folder, &chosen, false, false, !separate);
494 let mut result = None;
495 for (artifact, target) in chosen.iter().zip(targets) {
496 let Some(target) = target else {
497 result = Some(fail(self, &format!("The artifact {} cannot be merged into a folder of its name.", artifact.name)));
498 break;
499 };
500 if let Err(error) = self.fetch_artifact(artifact, &target) {
501 result = Some(fail(self, &format!("The artifact {} could not be downloaded: {error}", artifact.name)));
502 break;
503 }
504 }
505 let result = result.unwrap_or_else(|| {
506 let root = folder.display().to_string();
507 let found = glob::find(std::slice::from_ref(&root), &root, &self.home(), keep.hidden);
508 if found.files.is_empty() {
509 fail(self, "The artifacts matched hold no files to merge.")
510 } else {
511 self.send_artifact(&name, &found.files, &keep)
512 }
513 });
514 let _ = std::fs::remove_dir_all(&folder);
515 if result.0 && delete {
516 for artifact in &chosen {
517 match ureq::delete(&self.url(&format!("artifacts/{}", artifact.id))).set("authorization", &self.auth()).timeout(Duration::from_secs(60)).call() {
518 Ok(_) => self.log.line(&format!("Deleted artifact {}.", artifact.name)),
519 Err(ureq::Error::Status(_, response)) => self.log.line(&format!("##[warning]Artifact {} could not be deleted: {}", artifact.name, refusal(response))),
520 Err(error) => self.log.line(&format!("##[warning]Artifact {} could not be deleted: {error}", artifact.name)),
521 }
522 }
523 }
524 result
525 }
526
Fast pages, required checks on the branch, self-hosted runners, honest incidents527 /// The cache's `path`, each made absolute (`~/` is the home folder,
528 /// anything else is under the workspace), `!` patterns kept as they
529 /// came, with their `!`.
A repository has its own sidebar, as settings do530 fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> {
531 let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into());
Fast pages, required checks on the branch, self-hosted runners, honest incidents532 let workspace = self.workspace.display().to_string();
533 cache_patterns(with.get("path").map(String::as_str).unwrap_or_default(), &home, &workspace)
A repository has its own sidebar, as settings do534 }
535
536 /// `actions/cache` and `actions/cache/restore`: restores what it can,
537 /// and for `actions/cache`, saves at the end of the job on a miss.
538 pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) {
539 let key = with.get("key").cloned().unwrap_or_default();
540 if key.is_empty() {
541 self.log.line("##[error]The cache needs a `key`.");
542 return (false, BTreeMap::new());
543 }
544 let paths = self.cache_paths(with);
Merge branch 'worktree-agent-a3abfcce648e87dca'545 let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression());
A repository has its own sidebar, as settings do546 let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default());
547 let query = format!(
Merge branch 'worktree-agent-a3abfcce648e87dca'548 "cache?key={}&restore={}&version={}",
A repository has its own sidebar, as settings do549 urlencode(&key),
Merge branch 'worktree-agent-a3abfcce648e87dca'550 urlencode(&restore.join("\n")),
551 urlencode(&version)
A repository has its own sidebar, as settings do552 );
Fast pages, required checks on the branch, self-hosted runners, honest incidents553 let archive = self.temp.join("cache-restore.tar");
554 let started = Instant::now();
A repository has its own sidebar, as settings do555 let mut outputs = BTreeMap::new();
556 let exact = match self.download(&query, &archive) {
557 Ok(Some(matched)) => {
558 let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true");
Fast pages, required checks on the branch, self-hosted runners, honest incidents559 let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0);
560 // tar finds out from the archive whether it is zstd or gzip.
561 if !lookup_only && !self.shell(&format!("tar -xPf {}", quote(&archive.display().to_string()))) {
A repository has its own sidebar, as settings do562 self.log.line("##[warning]The cache was found but could not be unpacked.");
563 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents564 let _ = std::fs::remove_file(&archive);
565 self.log.line(&format!("Cache restored from key: {matched} ({} in {:.1}s)", megabytes(size), started.elapsed().as_secs_f64()));
A repository has its own sidebar, as settings do566 outputs.insert("cache-matched-key".into(), matched.clone());
567 matched == key
568 }
569 Ok(None) => {
570 self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", ")));
571 if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") {
572 self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set.");
573 return (false, outputs);
574 }
575 false
576 }
577 Err(error) => {
578 self.log.line(&format!("##[warning]The cache could not be read: {error}"));
579 false
580 }
581 };
582 outputs.insert("cache-hit".into(), exact.to_string());
583 outputs.insert("cache-primary-key".into(), key.clone());
584 if save_after && !exact {
585 self.posts.push(Post {
586 name: format!("Post {title}"),
587 condition: "success()".into(),
588 env: BTreeMap::new(),
Merge branch 'worktree-agent-a3abfcce648e87dca'589 run: PostRun::CacheSave { key, paths, version },
A repository has its own sidebar, as settings do590 });
591 }
592 (true, outputs)
593 }
594
595 /// Saves paths under a key, unless the key is taken.
Merge branch 'worktree-agent-a3abfcce648e87dca'596 pub(crate) fn cache_save(&mut self, key: &str, paths: &[String], version: &str) -> bool {
Fast pages, required checks on the branch, self-hosted runners, honest incidents597 if paths.iter().all(|p| p.starts_with('!')) {
A repository has its own sidebar, as settings do598 self.log.line("##[warning]Nothing to cache: no `path`.");
599 return true;
600 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents601 let archive = self.temp.join("cache-save.tar");
602 let started = Instant::now();
603 match self.run_shell(&pack_script(paths, &archive.display().to_string())) {
604 Some(0) => {}
605 Some(3) => {
606 self.log.line("##[warning]None of the cache's paths exist; nothing was saved.");
607 return true;
608 }
609 _ => {
610 self.log.line("##[warning]The cache could not be packed; nothing was saved.");
611 return true;
612 }
A repository has its own sidebar, as settings do613 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents614 let size = std::fs::metadata(&archive).map(|m| m.len()).unwrap_or(0);
615 let packed = started.elapsed().as_secs_f64();
Merge branch 'worktree-agent-a3abfcce648e87dca'616 match self.upload_cache(key, version, &archive) {
Fast pages, required checks on the branch, self-hosted runners, honest incidents617 Ok(true) => self.log.line(&format!(
618 "Cache saved with key: {key} ({}, packed in {packed:.1}s, sent in {:.1}s)",
619 megabytes(size),
620 started.elapsed().as_secs_f64() - packed
621 )),
622 Ok(false) => self.log.line(&format!("Cache not saved: {key} is already cached.")),
A repository has its own sidebar, as settings do623 // A cache that cannot be saved does not fail the job, as on GitHub.
624 Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")),
625 }
Fast pages, required checks on the branch, self-hosted runners, honest incidents626 let _ = std::fs::remove_file(&archive);
A repository has its own sidebar, as settings do627 true
628 }
629
Fast pages, required checks on the branch, self-hosted runners, honest incidents630 /// Runs a shell line, logging its output; its exit code.
631 fn run_shell(&mut self, script: &str) -> Option<i32> {
632 let mut command = Command::new("bash");
633 command.args(["-c", script]).current_dir(&self.workspace);
634 let mut commands = Commands::default();
Merge Actions: cross-repo workflows and actions, release and deployment triggers, step timeouts635 match process::run(command, Duration::from_secs(1800).min(self.remaining_time()), &mut self.log, &mut commands) {
Fast pages, required checks on the branch, self-hosted runners, honest incidents636 Ok(Ended::Exited(code)) => Some(code),
637 _ => None,
638 }
639 }
640
A repository has its own sidebar, as settings do641 /// `actions/cache/save`.
642 pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
643 let key = with.get("key").cloned().unwrap_or_default();
644 let paths = self.cache_paths(with);
Merge branch 'worktree-agent-a3abfcce648e87dca'645 let version = cache_version(with.get("path").map(String::as_str).unwrap_or_default(), compression());
646 (self.cache_save(&key, &paths, &version), BTreeMap::new())
A repository has its own sidebar, as settings do647 }
648}
649
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2650/// An input, trimmed; `None` when empty.
651fn input<'a>(with: &'a BTreeMap<String, String>, key: &str) -> Option<&'a str> {
652 with.get(key).map(|v| v.trim()).filter(|v| !v.is_empty())
653}
654
655/// A true-or-false input, as `core.getBooleanInput` reads it.
656fn flag(with: &BTreeMap<String, String>, key: &str, default: bool) -> Result<bool, String> {
657 match input(with, key) {
658 None => Ok(default),
659 Some("true" | "True" | "TRUE") => Ok(true),
660 Some("false" | "False" | "FALSE") => Ok(false),
661 Some(other) => Err(format!("`{key}` is `{other}`; it takes true or false.")),
662 }
663}
664
665/// `1 file`, `3 files`.
666fn count(n: usize, what: &str) -> String {
667 if n == 1 { format!("1 {what}") } else { format!("{n} {what}s") }
668}
669
670/// An id that came as a number or as text.
671fn number(value: &serde_json::Value) -> Option<u64> {
672 value.as_u64().or_else(|| value.as_str().and_then(|s| s.parse().ok()))
673}
674
675/// Whether a name is one GitHub takes for an artifact: not empty, at most
676/// 256 characters, none of `" : < > | * ? \ /` or a line break.
677fn check_name(name: &str) -> Result<(), String> {
678 if name.is_empty() {
679 return Err("The artifact needs a name.".into());
680 }
681 if name.chars().count() > 256 {
682 return Err(format!("The artifact name `{name}` is longer than 256 characters."));
683 }
684 if let Some(bad) = name.chars().find(|c| matches!(c, '"' | ':' | '<' | '>' | '|' | '*' | '?' | '\r' | '\n' | '\\' | '/')) {
685 let shown = match bad {
686 '\r' => "a carriage return".to_owned(),
687 '\n' => "a line break".to_owned(),
688 c => format!("`{c}`"),
689 };
690 return Err(format!("The artifact name `{name}` is not valid: it contains {shown}. A name may not contain \" : < > | * ? \\ / or line breaks."));
691 }
692 Ok(())
693}
694
695/// How an artifact is packed and kept, from the inputs `upload-artifact`
696/// and its merge share.
697#[derive(Debug, PartialEq)]
698struct Keep {
699 /// Days; 0 for the repository's own setting.
700 retention: u32,
701 /// 0 (stored) to 9.
702 level: u32,
703 overwrite: bool,
704 /// Whether files and folders whose names start with `.` are taken.
705 hidden: bool,
706}
707
708fn keep(with: &BTreeMap<String, String>, takes_overwrite: bool) -> Result<Keep, String> {
709 let retention = match input(with, "retention-days") {
710 None => 0,
711 Some(text) => match text.parse::<u32>() {
712 Ok(days @ 0..=90) => days,
713 _ => return Err(format!("`retention-days` is `{text}`; it takes a number of days from 1 to 90, or nothing for the repository's setting.")),
714 },
715 };
716 let level = match input(with, "compression-level") {
717 None => 6,
718 Some(text) => match text.parse::<u32>() {
719 Ok(level @ 0..=9) => level,
720 _ => return Err(format!("`compression-level` is `{text}`; it takes 0 (no compression) to 9.")),
721 },
722 };
723 Ok(Keep {
724 retention,
725 level,
726 overwrite: takes_overwrite && flag(with, "overwrite", false)?,
727 hidden: flag(with, "include-hidden-files", false)?,
728 })
729}
730
731/// What finishing an upload gave: the artifact's id, the ZIP's SHA-256 in
732/// hex, and the days it is kept, when known.
733struct Sent {
734 id: u64,
735 digest: String,
736 retention: Option<u32>,
737}
738
739/// An artifact as g1t lists it.
740#[derive(Debug, Clone, PartialEq)]
741struct Listed {
742 id: u64,
743 name: String,
744 size: u64,
745 digest: Option<String>,
746 /// `zip`, or `tgz` for an artifact an older runner stored.
747 format: String,
748}
749
750impl Listed {
751 fn from_json(value: &serde_json::Value) -> Option<Listed> {
752 Some(Listed {
753 id: number(&value["id"])?,
754 name: value["name"].as_str()?.to_owned(),
755 size: value["size"].as_u64().unwrap_or(0),
756 digest: value["digest"].as_str().map(str::to_owned),
757 format: value["format"].as_str().unwrap_or("zip").to_owned(),
758 })
759 }
760}
761
762/// One artifact per name, the newest (highest id), in name order.
763fn newest(listed: Vec<Listed>) -> Vec<Listed> {
764 let mut by_name: BTreeMap<String, Listed> = BTreeMap::new();
765 for artifact in listed {
766 if by_name.get(&artifact.name).is_none_or(|kept| kept.id < artifact.id) {
767 by_name.insert(artifact.name.clone(), artifact);
768 }
769 }
770 by_name.into_values().collect()
771}
772
773/// `artifact-ids`: numbers, separated by commas.
774fn artifact_ids(text: &str) -> Result<Vec<u64>, String> {
775 text.split(',')
776 .map(str::trim)
777 .filter(|id| !id.is_empty())
778 .map(|id| id.parse::<u64>().map_err(|_| format!("`artifact-ids` takes artifact ids, numbers separated by commas; `{id}` is not one.")))
779 .collect()
780}
781
782/// The folder each artifact is unpacked into: `dest` itself for one
783/// artifact by name, for `merge-multiple`, and for a single artifact by
784/// id; otherwise a folder of the artifact's name in `dest`. `None` for a
785/// name that cannot be a folder.
786fn download_targets(dest: &Path, chosen: &[Listed], by_name: bool, by_ids: bool, merge: bool) -> Vec<Option<std::path::PathBuf>> {
787 let straight = by_name || merge || (by_ids && chosen.len() == 1);
788 chosen
789 .iter()
790 .map(|artifact| {
791 if straight {
792 Some(dest.to_path_buf())
793 } else if artifact.name.is_empty() || artifact.name == "." || artifact.name == ".." || artifact.name.contains(['/', '\\']) {
794 None
795 } else {
796 Some(dest.join(&artifact.name))
797 }
798 })
799 .collect()
800}
801
802/// The SHA-256 of a file, in hex, read as it comes.
803fn sha256_file(path: &Path) -> std::io::Result<String> {
804 let mut file = std::fs::File::open(path)?;
805 let mut hasher = Sha256::new();
806 let mut buffer = vec![0u8; 256 * 1024];
807 loop {
808 let n = file.read(&mut buffer)?;
809 if n == 0 {
810 break;
811 }
812 hasher.update(&buffer[..n]);
813 }
814 Ok(hex::encode(hasher.finalize()))
815}
816
Fast pages, required checks on the branch, self-hosted runners, honest incidents817/// The message of a refused request, from its JSON body.
818fn refusal(response: ureq::Response) -> String {
819 let status = response.status();
820 let body: serde_json::Value = response.into_json().unwrap_or_default();
821 body["error"]["message"].as_str().map_or_else(|| format!("g1t answered {status}"), str::to_owned)
822}
823
824fn megabytes(bytes: u64) -> String {
825 if bytes < 1_048_576 { format!("{} KB", bytes.div_ceil(1024)) } else { format!("{:.1} MB", bytes as f64 / 1_048_576.0) }
826}
827
Merge branch 'worktree-agent-a3abfcce648e87dca'828/// How this machine compresses cache entries: zstd where it has it, else
829/// gzip, as `pack_script` decides.
830fn compression() -> &'static str {
831 let zstd = Command::new("sh").args(["-c", "command -v zstd"]).output().is_ok_and(|out| out.status.success());
832 if zstd { "zstd" } else { "gzip" }
833}
834
835/// An entry's version: the hash of its `path` lines, as written, and its
836/// compression, as `actions/cache` makes one. The same key saved for other
837/// paths, or packed another way, is another entry.
838fn cache_version(path: &str, compression: &str) -> String {
839 use sha2::{Digest, Sha256};
840 let mut parts = lines(path);
841 parts.push(compression.to_owned());
842 hex::encode(Sha256::digest(parts.join("|").as_bytes()))
843}
844
Fast pages, required checks on the branch, self-hosted runners, honest incidents845/// The lines of a cache's `path`, absolute: `~/` is `home`, a relative
846/// path is under `workspace`. A `!` pattern keeps its `!`.
847fn cache_patterns(path: &str, home: &str, workspace: &str) -> Vec<String> {
848 lines(path)
849 .into_iter()
850 .map(|line| {
851 let (bang, p) = match line.strip_prefix('!') {
852 Some(rest) => ("!", rest.trim().to_owned()),
853 None => ("", line),
854 };
855 let p = if p == "~" {
856 home.to_owned()
857 } else if let Some(rest) = p.strip_prefix("~/") {
858 format!("{home}/{rest}")
859 } else {
860 p
861 };
862 let p = if p.starts_with('/') { p } else { format!("{}/{}", workspace.trim_end_matches('/'), p.trim_start_matches("./")) };
863 format!("{bang}{}", p.trim_end_matches('/'))
864 })
865 .collect()
866}
867
868/// A path pattern as a word bash expands as a glob: everything but `*`,
869/// `?` and `[...]` escaped, so spaces and quotes stay literal.
870fn glob_word(pattern: &str) -> String {
871 let mut out = String::new();
872 for c in pattern.chars() {
873 if c.is_ascii_alphanumeric() || matches!(c, '*' | '?' | '[' | ']' | '/' | '.' | '-' | '_' | '~' | '+' | ',' | '=' | '@' | ':') {
874 out.push(c);
875 } else {
876 out.push('\\');
877 out.push(c);
878 }
879 }
880 out
881}
882
883/// The script that packs a cache: its patterns expanded (`**` reaching
884/// any depth), `!` patterns left out, into a tar archive compressed with
885/// zstd where there is one, else gzip. Exits 3 when nothing matched.
886fn pack_script(patterns: &[String], archive: &str) -> String {
887 let includes: Vec<String> = patterns.iter().filter(|p| !p.starts_with('!')).map(|p| glob_word(p)).collect();
888 let excludes: Vec<String> = patterns
889 .iter()
890 .filter_map(|p| p.strip_prefix('!'))
891 // tar's patterns: `*` already crosses `/`, so `**` is the same.
892 .map(|p| format!("--exclude={}", quote(&p.replace("**", "*"))))
893 .collect();
894 format!(
895 "set -o pipefail; shopt -s globstar nullglob dotglob; found=( {} ); files=(); \
896 for f in \"${{found[@]}}\"; do if [ -e \"$f\" ]; then files+=(\"$f\"); fi; done; \
897 if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; \
898 if command -v zstd >/dev/null; then compress='zstd -T0 -3'; else compress=gzip; fi; \
899 tar -cPf {} -I \"$compress\" {} -- \"${{files[@]}}\"",
900 includes.join(" "),
901 quote(archive),
902 excludes.join(" ")
903 )
904}
905
A repository has its own sidebar, as settings do906fn urlencode(text: &str) -> String {
907 let mut out = String::new();
908 for byte in text.bytes() {
909 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
910 out.push(byte as char);
911 } else {
912 out.push_str(&format!("%{byte:02X}"));
913 }
914 }
915 out
916}
917
918#[cfg(test)]
919mod tests {
920 use super::*;
921
922 #[test]
Merge branch 'worktree-agent-a3abfcce648e87dca'923 fn a_cache_entrys_version_follows_its_paths_and_compression() {
924 let version = cache_version("~/.cargo/registry\ntarget", "zstd");
925 assert_eq!(version.len(), 64);
926 assert_eq!(version, cache_version("~/.cargo/registry\n\ntarget\n", "zstd"), "blank lines do not count");
927 assert_ne!(version, cache_version("~/.cargo/registry", "zstd"));
928 assert_ne!(version, cache_version("~/.cargo/registry\ntarget", "gzip"));
929 }
930
931 #[test]
Fast pages, required checks on the branch, self-hosted runners, honest incidents932 fn cache_paths_take_home_globs_and_exclusions() {
933 let paths = cache_patterns("~/.cargo/registry/cache\ntarget/*/release/\n!target/**/incremental\n./dist\n/abs/x\n~", "/home/node", "/w/repo/");
934 assert_eq!(
935 paths,
936 ["/home/node/.cargo/registry/cache", "/w/repo/target/*/release", "!/w/repo/target/**/incremental", "/w/repo/dist", "/abs/x", "/home/node"]
937 );
938 assert_eq!(glob_word("/w/my repo/target/**/*.rlib"), "/w/my\\ repo/target/**/*.rlib");
939 assert_eq!(glob_word("/w/a'b"), "/w/a\\'b");
940 }
941
942 #[test]
943 fn packing_expands_globs_and_leaves_exclusions_out() {
944 let script = pack_script(&["/w/target/*/release".into(), "!/w/target/**/incremental".into()], "/t/c.tar");
945 assert!(script.contains("found=( /w/target/*/release )"), "{script}");
946 assert!(script.contains("--exclude='/w/target/*/incremental'"), "{script}");
947 assert!(script.contains("zstd -T0"), "{script}");
948 assert!(script.contains("exit 3"), "{script}");
949 }
950
951 /// Packs and unpacks for real, where bash and tar are (not on Windows).
952 #[test]
953 #[cfg(unix)]
954 fn a_packed_cache_unpacks_without_what_was_left_out() {
955 let dir = std::env::temp_dir().join(format!("g1t-cache-test-{}", std::process::id()));
956 let _ = std::fs::remove_dir_all(&dir);
957 for file in ["target/release/deps/a.rlib", "target/release/incremental/x.bin", "target/wasm/release/deps/b.rlib", "src/main.rs"] {
958 let path = dir.join(file);
959 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
960 std::fs::write(&path, file).unwrap();
961 }
962 let root = dir.display().to_string();
963 let patterns = cache_patterns("target/**/deps\ntarget/release/incremental\n!target/**/incremental", "/home/node", &root);
964 let archive = dir.join("c.tar").display().to_string();
965 let status = Command::new("bash").args(["-c", &pack_script(&patterns, &archive)]).status().unwrap();
966 assert!(status.success());
967 let listed = Command::new("tar").args(["-tPf", &archive]).output().unwrap();
968 let listed = String::from_utf8_lossy(&listed.stdout);
969 assert!(listed.contains("deps/a.rlib") && listed.contains("deps/b.rlib"), "{listed}");
970 assert!(!listed.contains("incremental") && !listed.contains("main.rs"), "{listed}");
971 let none = Command::new("bash").args(["-c", &pack_script(&[format!("{root}/nothing/*")], &archive)]).status().unwrap();
972 assert_eq!(none.code(), Some(3));
973 let _ = std::fs::remove_dir_all(&dir);
974 }
975
976 #[test]
A repository has its own sidebar, as settings do977 fn keys_are_encoded_and_names_checked() {
978 assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202");
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2979 assert!(check_name("coverage report").is_ok());
980 assert!(check_name("dist-linux_x64.1 (debug)").is_ok());
981 assert!(check_name("ünïcødé").is_ok());
982 assert!(check_name(&"a".repeat(256)).is_ok());
983 assert!(check_name(&"a".repeat(257)).is_err());
984 assert!(check_name("").is_err());
985 for bad in ["a/b", "a\\b", "a:b", "a*b", "a?b", "a\"b", "a<b", "a>b", "a|b", "a\nb", "a\rb"] {
986 assert!(check_name(bad).is_err(), "{bad}");
987 }
A repository has its own sidebar, as settings do988 assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]);
989 }
Actions: OIDC tokens, the toolkit's cache and artifact services, and artifacts in R2990
991 fn with(pairs: &[(&str, &str)]) -> BTreeMap<String, String> {
992 pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect()
993 }
994
995 #[test]
996 fn upload_inputs_are_read_as_v4_reads_them() {
997 assert_eq!(keep(&with(&[]), true).unwrap(), Keep { retention: 0, level: 6, overwrite: false, hidden: false });
998 let set = with(&[("retention-days", "14"), ("compression-level", "0"), ("overwrite", "true"), ("include-hidden-files", "True")]);
999 assert_eq!(keep(&set, true).unwrap(), Keep { retention: 14, level: 0, overwrite: true, hidden: true });
1000 // The merge takes no `overwrite`.
1001 assert!(!keep(&set, false).unwrap().overwrite);
1002 assert_eq!(keep(&with(&[("retention-days", "0")]), true).unwrap().retention, 0);
1003 assert!(keep(&with(&[("retention-days", "91")]), true).is_err());
1004 assert!(keep(&with(&[("retention-days", "-1")]), true).is_err());
1005 assert!(keep(&with(&[("retention-days", "two")]), true).is_err());
1006 assert!(keep(&with(&[("compression-level", "10")]), true).is_err());
1007 assert!(keep(&with(&[("overwrite", "yes")]), true).is_err());
1008 assert_eq!(artifact_ids(" 12, 34 ,,").unwrap(), [12, 34]);
1009 assert!(artifact_ids("12,abc").is_err());
1010 assert_eq!(count(1, "file"), "1 file");
1011 assert_eq!(count(37, "file"), "37 files");
1012 }
1013
1014 fn listed(id: u64, name: &str) -> Listed {
1015 Listed { id, name: name.into(), size: 0, digest: None, format: "zip".into() }
1016 }
1017
1018 #[test]
1019 fn listings_read_and_keep_the_newest_of_a_name() {
1020 let json = serde_json::json!([
1021 { "id": 1, "name": "dist", "size": 10, "digest": null, "format": "tgz", "created_at": "x", "expires_at": "y" },
1022 { "id": "3", "name": "dist", "size": 30, "digest": "sha256:ab", "format": "zip" },
1023 { "id": 2, "name": "logs", "size": 5 },
1024 { "name": "no id" }
1025 ]);
1026 let all: Vec<Listed> = json.as_array().unwrap().iter().filter_map(Listed::from_json).collect();
1027 assert_eq!(all.len(), 3);
1028 assert_eq!(all[0].format, "tgz");
1029 let kept = newest(all);
1030 assert_eq!(kept.iter().map(|a| (a.id, a.name.as_str())).collect::<Vec<_>>(), [(3, "dist"), (2, "logs")]);
1031 assert_eq!(kept[0].digest.as_deref(), Some("sha256:ab"));
1032 }
1033
1034 #[test]
1035 fn downloads_land_where_v4_puts_them() {
1036 let dest = Path::new("/w/out");
1037 let one = [listed(1, "dist")];
1038 let two = [listed(1, "dist"), listed(2, "logs")];
1039 let at = |targets: Vec<Option<std::path::PathBuf>>| targets.into_iter().map(|t| t.unwrap()).collect::<Vec<_>>();
1040 // By name: straight into `path`.
1041 assert_eq!(at(download_targets(dest, &one, true, false, false)), [dest.to_path_buf()]);
1042 // Every artifact, or by pattern: a folder each.
1043 assert_eq!(at(download_targets(dest, &two, false, false, false)), [dest.join("dist"), dest.join("logs")]);
1044 assert_eq!(at(download_targets(dest, &one, false, false, false)), [dest.join("dist")]);
1045 // merge-multiple: all into `path`.
1046 assert_eq!(at(download_targets(dest, &two, false, false, true)), [dest.to_path_buf(), dest.to_path_buf()]);
1047 // By id: one straight into `path`, several a folder each.
1048 assert_eq!(at(download_targets(dest, &one, false, true, false)), [dest.to_path_buf()]);
1049 assert_eq!(at(download_targets(dest, &two, false, true, false)), [dest.join("dist"), dest.join("logs")]);
1050 // A name that is no folder is refused.
1051 assert!(download_targets(dest, &[listed(1, ".."), listed(2, "x")], false, false, false)[0].is_none());
1052 assert!(glob::matches_part("dist-*", "dist-linux"));
1053 assert!(!glob::matches_part("dist-*", "logs"));
1054 }
A repository has its own sidebar, as settings do1055}

This file's history is long; its oldest lines are credited to the oldest commit read.