Skip to content
Sign up

flagon-io/g1t

public
0
908 linesCodeBlameRaw
1//! `uses:` steps: `actions/checkout` done natively against g1t, actions
2//! fetched from another repository on g1t (or else GitHub, at the commit
3//! g1t resolved its ref to, from the archive g1t keeps of it when it has
4//! one) and run as they are (JavaScript, composite and Docker),
5//! `docker://` images, and a few of GitHub's own whose services g1t does
6//! not have yet. Every action a job's steps name is downloaded before the
7//! first step runs, as GitHub's runner does.
8
9use std::collections::BTreeMap;
10use std::path::{Path, PathBuf};
11use std::process::Command;
12use std::time::Duration;
13
14use base64::Engine;
15use base64::engine::general_purpose::STANDARD;
16use g1t_actions::expr;
17use g1t_actions::workflow::yaml_to_json;
18use serde_json::{Map, Value, json};
19
20use super::containers::{self, DockerRun};
21use super::files::StepFiles;
22use super::process::{self, Commands, Ended};
23use super::{Frame, Job, Post, PostRun};
24
25const ACTIONS_DIR: &str = "/home/runner/_actions";
26
27/// Where an action comes from.
28enum Source {
29 Local(PathBuf),
30 /// Another repository: on g1t when g1t has it and this one may use
31 /// it, otherwise on GitHub.
32 GitHub { owner: String, repo: String, path: String, git_ref: String },
33}
34
35fn safe(part: &str) -> bool {
36 !part.is_empty() && part.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/')) && !part.contains("..")
37}
38
39fn is_sha(text: &str) -> bool {
40 text.len() == 40 && text.chars().all(|c| c.is_ascii_hexdigit())
41}
42
43/// Whether the runner does `name` (lowercased, without its ref) itself
44/// rather than fetching it.
45fn native(name: &str, docker_hosted: bool) -> bool {
46 matches!(
47 name,
48 "actions/checkout"
49 | "actions/upload-artifact"
50 | "actions/upload-artifact/merge"
51 | "actions/download-artifact"
52 | "actions/cache"
53 | "actions/cache/restore"
54 | "actions/cache/save"
55 ) || (docker_hosted && name == "docker/setup-buildx-action")
56}
57
58/// The repository and ref of an action `uses` fetches from another
59/// repository: (owner, repo, ref). None for one that is local, a Docker
60/// image, done natively, or not a name at all.
61fn remote_action(uses: &str, docker_hosted: bool) -> Option<(String, String, String)> {
62 let uses = uses.trim();
63 if uses.starts_with("./") || uses.starts_with("docker://") {
64 return None;
65 }
66 let (name, git_ref) = uses.split_once('@')?;
67 if git_ref.is_empty() || native(&name.to_ascii_lowercase(), docker_hosted) {
68 return None;
69 }
70 let mut parts = name.splitn(3, '/');
71 let (owner, repo) = (parts.next()?, parts.next()?);
72 (!owner.is_empty() && !repo.is_empty()).then(|| (owner.to_owned(), repo.to_owned(), git_ref.to_owned()))
73}
74
75impl Job {
76 /// Runs a git command, logging it; the credential header is never logged.
77 fn git(&mut self, dir: &Path, args: &[&str], auth: Option<&str>) -> bool {
78 let shown: Vec<&str> = args.to_vec();
79 self.log.line(&format!("[command]git {}", shown.join(" ")));
80 let mut command = Command::new("git");
81 command.current_dir(dir);
82 if let Some(header) = auth {
83 command.args(["-c", &format!("http.extraheader={header}")]);
84 }
85 command.args(args).env("GIT_TERMINAL_PROMPT", "0");
86 let mut commands = Commands::default();
87 matches!(process::run(command, Duration::from_secs(600).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
88 }
89
90 /// A fetch, tried again after a short wait when it fails: a transfer
91 /// cut short on the way ("transfer closed with N bytes remaining") is
92 /// over by the next try. Three tries in all, as actions/checkout does.
93 fn fetch_retrying(&mut self, dir: &Path, args: &[&str], auth: Option<&str>) -> bool {
94 for (attempt, wait) in [0u64, 2, 5].into_iter().enumerate() {
95 if attempt > 0 {
96 self.log.line(&format!("The fetch failed; trying again in {wait} s ({} of 3).", attempt + 1));
97 std::thread::sleep(Duration::from_secs(wait));
98 }
99 if self.git(dir, args, auth) {
100 return true;
101 }
102 }
103 false
104 }
105
106 /// `actions/checkout`, against g1t.
107 fn checkout(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
108 let checkout = self.spec["checkout"].clone();
109 let own = checkout["repository"].as_str().unwrap_or_default().to_owned();
110 let server = self.contexts["github"]["server_url"].as_str().unwrap_or("https://g1t.sh").to_owned();
111 let repository = with.get("repository").filter(|r| !r.is_empty()).cloned().unwrap_or(own.clone());
112 let same = repository.eq_ignore_ascii_case(&own);
113 let token = with.get("token").filter(|t| !t.is_empty()).cloned().or_else(|| checkout["token"].as_str().map(str::to_owned)).unwrap_or_default();
114 let url = if same { checkout["url"].as_str().unwrap_or_default().to_owned() } else { format!("{server}/{repository}.git") };
115 let path = with.get("path").filter(|p| !p.is_empty()).map_or(self.workspace.clone(), |p| self.workspace.join(p));
116 let depth: u32 = with.get("fetch-depth").and_then(|d| d.parse().ok()).unwrap_or(1);
117 let wanted_ref = with.get("ref").filter(|r| !r.is_empty()).cloned();
118 let auth = (!token.is_empty()).then(|| format!("AUTHORIZATION: basic {}", STANDARD.encode(format!("x-access-token:{token}"))));
119
120 self.log.line(&format!("Checking out {repository} into {}", path.display()));
121 if path.exists() {
122 let _ = std::fs::remove_dir_all(&path);
123 }
124 if let Err(error) = std::fs::create_dir_all(&path) {
125 self.log.line(&format!("##[error]Could not make {}: {error}", path.display()));
126 return (false, BTreeMap::new());
127 }
128 let _ = Command::new("git").args(["config", "--global", "--add", "safe.directory", "*"]).status();
129 if !self.git(&path, &["init", "--quiet"], None) || !self.git(&path, &["remote", "add", "origin", &url], None) {
130 return (false, BTreeMap::new());
131 }
132
133 // What to fetch, and which commit to end up on.
134 let run_ref = checkout["ref"].as_str().unwrap_or_default().to_owned();
135 let run_sha = checkout["sha"].as_str().unwrap_or_default().to_owned();
136 let is_sha = |r: &str| r.len() == 40 && r.chars().all(|c| c.is_ascii_hexdigit());
137 let (fetch, sha, branch): (String, Option<String>, Option<String>) = match &wanted_ref {
138 Some(r) if is_sha(r) => ("HEAD".into(), Some(r.clone()), None),
139 Some(r) if r.starts_with("refs/") => (r.clone(), None, r.strip_prefix("refs/heads/").map(str::to_owned)),
140 Some(r) => (r.clone(), None, Some(r.clone())),
141 // A pull request's merge ref, or no ref at all (a deployment of
142 // a bare commit): the commit itself.
143 None if same && (run_ref.starts_with("refs/pull/") || run_ref.is_empty()) => ("HEAD".into(), Some(run_sha.clone()), None),
144 None if same => (run_ref.clone(), Some(run_sha.clone()), run_ref.strip_prefix("refs/heads/").map(str::to_owned)),
145 None => ("HEAD".into(), None, None),
146 };
147 let depth_arg = format!("--depth={depth}");
148 let mut args = vec!["fetch", "--no-tags", "--prune", "--quiet"];
149 if depth > 0 {
150 args.push(&depth_arg);
151 }
152 if with.get("fetch-tags").is_some_and(|t| t == "true") {
153 args.retain(|a| *a != "--no-tags");
154 }
155 args.push("origin");
156 args.push(&fetch);
157 if !self.fetch_retrying(&path, &args, auth.as_deref()) {
158 self.log.line(&format!("##[error]Could not fetch {fetch} from {repository}."));
159 return (false, BTreeMap::new());
160 }
161 let target = sha.clone().unwrap_or_else(|| "FETCH_HEAD".into());
162 // The commit may be further back than a shallow fetch reaches: the
163 // branch moved on after the run began, say. Ask for the commit
164 // itself, and failing that the whole history; a plain fetch never
165 // reaches past a shallow boundary.
166 let has = |target: &str| Command::new("git").current_dir(&path).args(["cat-file", "-e", &format!("{target}^{{commit}}")]).status().is_ok_and(|s| s.success());
167 if !has(&target) {
168 let by_sha = sha.as_deref().is_some_and(|sha| {
169 let mut args = vec!["fetch", "--no-tags", "--quiet"];
170 if depth > 0 {
171 args.push(&depth_arg);
172 }
173 args.extend(["origin", sha]);
174 self.git(&path, &args, auth.as_deref()) && has(sha)
175 });
176 let shallow = path.join(".git").join("shallow").exists();
177 let deepen: &[&str] = if shallow { &["fetch", "--no-tags", "--quiet", "--unshallow", "origin"] } else { &["fetch", "--no-tags", "--quiet", "origin"] };
178 if !by_sha && !self.fetch_retrying(&path, deepen, auth.as_deref()) {
179 return (false, BTreeMap::new());
180 }
181 }
182 let checked_out = match &branch {
183 Some(branch) => self.git(&path, &["checkout", "--quiet", "--force", "-B", branch, &target], None),
184 None => self.git(&path, &["checkout", "--quiet", "--force", "--detach", &target], None),
185 };
186 if !checked_out {
187 return (false, BTreeMap::new());
188 }
189 if with.get("persist-credentials").is_none_or(|p| p != "false")
190 && let Some(header) = &auth
191 {
192 let key = format!("http.{server}/.extraheader");
193 let _ = Command::new("git").current_dir(&path).args(["config", "--local", &key, header]).status();
194 }
195 if let Some(submodules) = with.get("submodules").filter(|s| *s == "true" || *s == "recursive") {
196 let mut args = vec!["submodule", "update", "--init", "--quiet"];
197 if submodules == "recursive" {
198 args.push("--recursive");
199 }
200 if !self.git(&path, &args, auth.as_deref()) {
201 self.log.line("##[warning]Submodules could not all be checked out; only those hosted on g1t can be.");
202 }
203 }
204 if with.get("lfs").is_some_and(|l| l == "true") {
205 self.log.line("##[warning]Git LFS files are not fetched on g1t yet.");
206 }
207 let commit = Command::new("git").current_dir(&path).args(["rev-parse", "HEAD"]).output().ok().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_owned()).unwrap_or_default();
208 self.log.line(&format!("Checked out {commit}"));
209 let mut outputs = BTreeMap::new();
210 outputs.insert("ref".into(), wanted_ref.unwrap_or(run_ref));
211 outputs.insert("commit".into(), commit);
212 (true, outputs)
213 }
214
215 /// Fetches another repository's action, once per job: from g1t when
216 /// g1t has the repository and this one may use it, otherwise from
217 /// GitHub. A private repository on g1t that may not be used here fails
218 /// the step, saying why, rather than fetching something else by its
219 /// name.
220 fn fetch_remote_action(&mut self, owner: &str, repo: &str, git_ref: &str) -> Option<PathBuf> {
221 let on_g1t = super::paths::under_home(ACTIONS_DIR).join("_g1t").join(owner).join(repo).join(git_ref);
222 let on_github = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
223 for dir in [&on_g1t, &on_github] {
224 if dir.join(".g1t-fetched").exists() {
225 return Some(dir.clone());
226 }
227 }
228 if !(safe(owner) && safe(repo) && safe(git_ref)) {
229 self.log.line(&format!("##[error]`{owner}/{repo}@{git_ref}` is not a name g1t can fetch."));
230 return None;
231 }
232 match self.log.api.action(&format!("{owner}/{repo}"), git_ref) {
233 Ok(found) if found["source"] == "g1t" => self.fetch_g1t_action(&on_g1t, owner, repo, git_ref, &found),
234 Ok(found) => {
235 if let Some(note) = found["note"].as_str() {
236 self.log.line(&format!("##[warning]{note}"));
237 }
238 let sha = found["sha"].as_str().filter(|sha| is_sha(sha)).map(str::to_owned);
239 if let Some(sha) = &sha
240 && found["archive"] == true
241 && let Some(dir) = self.fetch_archived_action(owner, repo, git_ref, sha)
242 {
243 return Some(dir);
244 }
245 self.fetch_action(owner, repo, git_ref, sha.as_deref())
246 }
247 Err((true, why)) => {
248 self.log.line(&format!("##[error]{why}"));
249 None
250 }
251 Err((false, why)) => {
252 self.log.line(&format!("##[warning]g1t could not say where {owner}/{repo} is ({why}); fetching it from GitHub."));
253 self.fetch_action(owner, repo, git_ref, None)
254 }
255 }
256 }
257
258 /// Fetches an action from a repository on g1t, at its ref, with the
259 /// read-only token g1t gave for it when it is private.
260 fn fetch_g1t_action(&mut self, dir: &Path, owner: &str, repo: &str, git_ref: &str, found: &Value) -> Option<PathBuf> {
261 let url = found["url"].as_str().unwrap_or_default().to_owned();
262 let token = found["token"].as_str().filter(|token| !token.is_empty()).map(str::to_owned);
263 if let Some(token) = &token {
264 self.log.add_mask(token);
265 }
266 let auth = token.map(|token| format!("AUTHORIZATION: basic {}", STANDARD.encode(format!("x-access-token:{token}"))));
267 self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' from g1t"));
268 let _ = std::fs::remove_dir_all(dir);
269 if std::fs::create_dir_all(dir).is_err() || !self.git(dir, &["init", "--quiet"], None) || !self.git(dir, &["remote", "add", "origin", &url], None) {
270 return None;
271 }
272 // A branch or tag at its tip; a commit may need the history.
273 let shallow = self.fetch_retrying(dir, &["fetch", "--depth=1", "--no-tags", "--quiet", "origin", git_ref], auth.as_deref());
274 let checked_out = if shallow {
275 self.git(dir, &["checkout", "--quiet", "--force", "--detach", "FETCH_HEAD"], None)
276 } else {
277 self.fetch_retrying(dir, &["fetch", "--quiet", "--tags", "origin", "+refs/heads/*:refs/remotes/origin/*"], auth.as_deref())
278 && self.git(dir, &["checkout", "--quiet", "--force", "--detach", git_ref], None)
279 };
280 if !checked_out {
281 self.log.line(&format!("##[error]Could not fetch {owner}/{repo}@{git_ref} from g1t: is {git_ref} a branch, tag or commit there?"));
282 let _ = std::fs::remove_dir_all(dir);
283 return None;
284 }
285 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
286 Some(dir.to_path_buf())
287 }
288
289 /// Downloads a GitHub action's archive that g1t keeps, at the commit
290 /// g1t resolved its ref to. None when it could not, and the job fetches
291 /// it from GitHub instead.
292 fn fetch_archived_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: &str) -> Option<PathBuf> {
293 let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
294 self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha}) from g1t"));
295 let archive = self.temp.join(format!("action{}.tar.gz", super::rand_id()));
296 let downloaded = self.log.api.action_archive(&format!("{owner}/{repo}"), sha, &archive);
297 let unpacked = match downloaded {
298 Ok(_) => {
299 let _ = std::fs::remove_dir_all(&dir);
300 let _ = std::fs::create_dir_all(&dir);
301 let mut command = Command::new("tar");
302 command.arg("-xzf").arg(&archive).arg("-C").arg(&dir).arg("--strip-components=1");
303 if cfg!(windows) {
304 // GNU tar on Windows reads `C:...` as a host and a file.
305 command.arg("--force-local");
306 }
307 let mut commands = Commands::default();
308 matches!(process::run(command, Duration::from_secs(300).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
309 }
310 Err(why) => {
311 self.log.line(&format!("g1t could not send it ({why}); fetching it from GitHub."));
312 false
313 }
314 };
315 let _ = std::fs::remove_file(&archive);
316 if !unpacked {
317 let _ = std::fs::remove_dir_all(&dir);
318 return None;
319 }
320 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
321 Some(dir)
322 }
323
324 /// Fetches an action from GitHub, once per job: at `sha` when g1t
325 /// resolved its ref to one, so it is the commit g1t named.
326 fn fetch_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: Option<&str>) -> Option<PathBuf> {
327 let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
328 if dir.join(".g1t-fetched").exists() {
329 return Some(dir);
330 }
331 if !(safe(owner) && safe(repo) && safe(git_ref)) {
332 self.log.line(&format!("##[error]`{owner}/{repo}@{git_ref}` is not a name g1t can fetch."));
333 return None;
334 }
335 match sha {
336 Some(sha) => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha})")),
337 None => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}'")),
338 }
339 let _ = std::fs::create_dir_all(&dir);
340 let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{}", sha.unwrap_or(git_ref));
341 let script = format!("set -o pipefail; curl -fsSL --retry 3 '{url}' | tar{} -xz -C '{}' --strip-components=1", super::blobs::tar_local(), dir.display());
342 let mut command = Command::new("bash");
343 command.args(["-c", &script]);
344 let mut commands = Commands::default();
345 match process::run(command, Duration::from_secs(300).min(self.remaining_time()), &mut self.log, &mut commands) {
346 Ok(Ended::Exited(0)) => {
347 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
348 Some(dir)
349 }
350 _ => {
351 self.log.line(&format!("##[error]Could not download {owner}/{repo}@{git_ref} from GitHub."));
352 let _ = std::fs::remove_dir_all(&dir);
353 None
354 }
355 }
356 }
357
358 /// Downloads every action the job's steps fetch from other
359 /// repositories, before the first step runs, as GitHub's runner does in
360 /// "Set up job". False when one could not be: the job fails before
361 /// anything runs. Actions a composite action uses are fetched as it
362 /// runs.
363 pub(crate) fn prepare_actions(&mut self, steps: &[Map<String, Value>]) -> bool {
364 let mut wanted: Vec<(String, String, String)> = Vec::new();
365 for step in steps {
366 if let Some(found) = step.get("uses").and_then(Value::as_str).and_then(|uses| remote_action(uses, self.docker_hosted))
367 && !wanted.contains(&found)
368 {
369 wanted.push(found);
370 }
371 }
372 if wanted.is_empty() {
373 return true;
374 }
375 self.log.line("Prepare all required actions");
376 let mut ready = true;
377 for (owner, repo, git_ref) in &wanted {
378 if self.fetch_remote_action(owner, repo, git_ref).is_none() {
379 ready = false;
380 }
381 }
382 if !ready {
383 self.log.line("##[error]Not every action this job uses could be downloaded, so no step was run.");
384 }
385 ready
386 }
387
388 /// Runs a JavaScript file of an action with Node: the one its
389 /// `runs.using` names where the machine has it (`node_program`).
390 pub(crate) fn run_node(&mut self, action_dir: &Path, script: &str, env: &BTreeMap<String, String>, using: &str) -> bool {
391 let id = format!("node{}", super::rand_id());
392 let Ok(files) = StepFiles::new(&self.temp, &id) else { return false };
393 let full = self.process_env(env, &files);
394 let script_path = action_dir.join(script);
395 // In a job container whose image runs the runner's Node, the action
396 // runs there, as on GitHub.
397 let in_container = self.container.as_ref().filter(|c| c.node).map(|c| c.path.clone()).and_then(|image_path| {
398 let inside = self.container_env(env, full.clone(), &image_path);
399 self.in_container(containers::CONTAINER_NODE, &[script_path.display().to_string()], &self.workspace, inside)
400 });
401 let command = match in_container {
402 Some(command) => command,
403 None => {
404 let node = node_program(using, |name| std::env::var(name).ok(), |path| path.is_file());
405 let mut command = Command::new(node);
406 command.arg(&script_path).current_dir(&self.workspace).env_clear().envs(full);
407 command
408 }
409 };
410 let mut commands = Commands {
411 debug: self.debug,
412 ..Commands::default()
413 };
414 let ended = process::run(command, Duration::from_secs(6 * 3600).min(self.deadline_left()), &mut self.log, &mut commands);
415 let ok = matches!(ended, Ok(Ended::Exited(0)));
416 match ended {
417 Ok(Ended::Exited(code)) if code != 0 => self.log.line(&format!("##[error]The action exited with code {code}.")),
418 Ok(Ended::TimedOut) => self.log.line("##[error]The action ran past its time limit and was stopped."),
419 _ => {}
420 }
421 let (outputs, state) = self.absorb(&files, &commands);
422 self.last_node_outputs = outputs;
423 self.last_node_state = state;
424 ok
425 }
426
427 fn deadline_left(&self) -> Duration {
428 self.remaining_time()
429 }
430
431 /// Runs a `uses:` step. Returns whether it succeeded, and its outputs.
432 #[allow(clippy::too_many_arguments)]
433 pub(crate) fn uses(
434 &mut self,
435 uses: &str,
436 with: &BTreeMap<String, String>,
437 env: &BTreeMap<String, String>,
438 frame: &Frame,
439 title: &str,
440 action_name: &str,
441 _timeout: Duration,
442 ) -> (bool, BTreeMap<String, String>) {
443 let uses = uses.trim();
444 if let Some(image) = uses.strip_prefix("docker://") {
445 // `with.args` and `with.entrypoint` are the container's; every
446 // input is also an `INPUT_` variable, as on GitHub.
447 let mut step_env = env.clone();
448 step_env.insert("GITHUB_ACTION".into(), action_name.to_owned());
449 for (input, value) in with {
450 step_env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
451 }
452 let run = DockerRun {
453 image: image.to_owned(),
454 entrypoint: with.get("entrypoint").filter(|e| !e.is_empty()).cloned(),
455 args: with.get("args").map(|a| containers::split_words(a)).unwrap_or_default(),
456 env: step_env,
457 };
458 let (ok, outputs, _) = self.run_docker(&run);
459 return (ok, outputs);
460 }
461 let (name, git_ref) = uses.split_once('@').unwrap_or((uses, ""));
462 let lower = name.to_ascii_lowercase();
463 if lower == "docker/setup-buildx-action" && native(&lower, self.docker_hosted) {
464 return self.setup_buildx(with);
465 }
466 match lower.as_str() {
467 "actions/checkout" => return self.checkout(with),
468 "actions/upload-artifact" => return self.upload_artifact(with),
469 "actions/upload-artifact/merge" => return self.merge_artifacts(with),
470 "actions/download-artifact" => return self.download_artifact(with),
471 "actions/cache" => return self.cache(with, true, title),
472 "actions/cache/restore" => return self.cache(with, false, title),
473 "actions/cache/save" => return self.cache_save_now(with),
474 _ => {}
475 }
476 let Some(found) = self.resolve_action(uses) else {
477 return (false, BTreeMap::new());
478 };
479 let Action { dir, local, manifest: action, .. } = &found;
480
481 // Inputs: what the step gives, else the action's defaults.
482 let env_context = env.clone();
483 let contexts = self.contexts_for(frame, &env_context);
484 let inputs = self.action_inputs(action, with, &contexts, true);
485
486 let runs = action.get("runs").cloned().unwrap_or(Value::Null);
487 let using = runs.get("using").map(expr::to_text).unwrap_or_default().to_ascii_lowercase();
488 let mut step_env = action_env(env, action_name, &found, git_ref);
489
490 if using.starts_with("node") {
491 for (input, value) in &inputs {
492 step_env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
493 }
494 let condition_of = |key: &str| runs.get(key).map(expr::to_text).unwrap_or_else(|| "always()".into());
495 // State an action saves, `STATE_<name>` for its later stages.
496 let mut state = BTreeMap::new();
497 if let Some(pre) = runs.get("pre").map(expr::to_text) {
498 if frame.action_path.is_some() {
499 // An action inside a composite action: its `pre` right
500 // before its `main`.
501 let run_pre = self.with_scope(&contexts, |scope| expr::condition(&condition_of("pre-if"), scope)).unwrap_or(true);
502 if run_pre {
503 if !self.run_node(dir, &pre, &step_env, &using) {
504 return (false, BTreeMap::new());
505 }
506 state = std::mem::take(&mut self.last_node_state);
507 }
508 } else if *local {
509 self.log.line(&format!("##[warning]`pre` execution is not supported for local action from '{uses}'"));
510 } else {
511 // Run before the job's first step (`pre_steps`).
512 state = self.pre_state.remove(action_name).unwrap_or_default();
513 }
514 }
515 let Some(main) = runs.get("main").map(expr::to_text) else {
516 self.log.line(&format!("##[error]`{uses}` has no `runs.main`."));
517 return (false, BTreeMap::new());
518 };
519 let mut main_env = step_env.clone();
520 for (name, value) in &state {
521 main_env.insert(format!("STATE_{name}"), value.clone());
522 }
523 let ok = self.run_node(dir, &main, &main_env, &using);
524 let outputs = std::mem::take(&mut self.last_node_outputs);
525 state.extend(std::mem::take(&mut self.last_node_state));
526 if let Some(post) = runs.get("post").map(expr::to_text) {
527 let mut post_env = step_env.clone();
528 for (name, value) in state {
529 post_env.insert(format!("STATE_{name}"), value);
530 }
531 self.posts.push(Post {
532 name: format!("Post {title}"),
533 condition: condition_of("post-if"),
534 env: post_env,
535 run: PostRun::Node { action_dir: dir.clone(), script: post, using: using.clone() },
536 });
537 }
538 return (ok, outputs);
539 }
540 if using == "composite" {
541 let mut inner = Frame {
542 steps: Map::new(),
543 inputs: Some(Value::Object(inputs.iter().map(|(k, v)| (k.clone(), json!(v))).collect())),
544 action_path: Some(dir.display().to_string()),
545 env: env.clone(),
546 };
547 let mut steps: Vec<Map<String, Value>> = runs.get("steps").and_then(Value::as_array).map(|s| s.iter().filter_map(|s| s.as_object().cloned()).collect()).unwrap_or_default();
548 super::name_steps(&mut steps);
549 let was_failed = self.failed;
550 // A composite's steps see their own success, not the job's.
551 self.failed = false;
552 let mut ok = true;
553 for step in &steps {
554 if !self.step(&mut inner, step, 0, false, &Map::new()) {
555 ok = false;
556 }
557 }
558 let contexts = self.contexts_for(&inner, &inner.env.clone());
559 let mut outputs = BTreeMap::new();
560 if let Some(Value::Object(declared)) = action.get("outputs") {
561 for (name, spec) in declared {
562 if let Some(value) = spec.get("value") {
563 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
564 outputs.insert(name.clone(), expr::to_text(&value));
565 }
566 }
567 }
568 self.failed = was_failed;
569 return (ok, outputs);
570 }
571 if using == "docker" {
572 return self.docker_action(uses, dir, &runs, &inputs, &step_env, frame, title);
573 }
574 self.log.line(&format!("##[error]`{uses}` runs with `{using}`, which g1t does not know."));
575 (false, BTreeMap::new())
576 }
577
578 /// A Docker action: its image built from its Dockerfile (or pulled,
579 /// for `docker://`), then run with its `args`, `entrypoint` and `env`,
580 /// its inputs as `INPUT_` variables, and `pre-entrypoint` and
581 /// `post-entrypoint` around it.
582 #[allow(clippy::too_many_arguments)]
583 fn docker_action(
584 &mut self,
585 uses: &str,
586 dir: &Path,
587 runs: &Value,
588 inputs: &BTreeMap<String, String>,
589 step_env: &BTreeMap<String, String>,
590 frame: &Frame,
591 title: &str,
592 ) -> (bool, BTreeMap<String, String>) {
593 let image = runs.get("image").map(expr::to_text).unwrap_or_default();
594 let image = if let Some(pulled) = image.strip_prefix("docker://") {
595 pulled.to_owned()
596 } else if image.is_empty() {
597 self.log.line(&format!("##[error]`{uses}` has no `runs.image`."));
598 return (false, BTreeMap::new());
599 } else {
600 match self.build_action_image(dir, &image, uses) {
601 Some(tag) => tag,
602 None => return (false, BTreeMap::new()),
603 }
604 };
605 // `args` and `env` read with the action's own inputs.
606 let mut env = step_env.clone();
607 for (input, value) in inputs {
608 env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
609 }
610 let mut scope_frame = frame.clone();
611 scope_frame.inputs = Some(Value::Object(inputs.iter().map(|(k, v)| (k.clone(), json!(v))).collect()));
612 let contexts = self.contexts_for(&scope_frame, &env);
613 if let Some(Value::Object(own)) = runs.get("env") {
614 for (name, value) in own {
615 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
616 env.insert(name.clone(), expr::to_text(&value));
617 }
618 }
619 let args: Vec<String> = match runs.get("args") {
620 Some(Value::Array(items)) => items
621 .iter()
622 .map(|item| {
623 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(item, scope)).unwrap_or(Value::Null);
624 expr::to_text(&value)
625 })
626 .collect(),
627 _ => Vec::new(),
628 };
629 let entry = |key: &str| runs.get(key).map(expr::to_text).filter(|e| !e.is_empty());
630 if let Some(pre) = entry("pre-entrypoint") {
631 let run = DockerRun { image: image.clone(), entrypoint: Some(pre), args: Vec::new(), env: env.clone() };
632 if !self.run_docker(&run).0 {
633 return (false, BTreeMap::new());
634 }
635 }
636 let run = DockerRun { image: image.clone(), entrypoint: entry("entrypoint"), args, env: env.clone() };
637 let (ok, outputs, state) = self.run_docker(&run);
638 if let Some(post) = entry("post-entrypoint") {
639 let mut post_env = env;
640 for (name, value) in state {
641 post_env.insert(format!("STATE_{name}"), value);
642 }
643 self.posts.push(Post {
644 name: format!("Post {title}"),
645 condition: runs.get("post-if").map(expr::to_text).unwrap_or_else(|| "always()".into()),
646 env: BTreeMap::new(),
647 run: PostRun::Docker(DockerRun { image, entrypoint: Some(post), args: Vec::new(), env: post_env }),
648 });
649 }
650 (ok, outputs)
651 }
652}
653
654/// An action, read from its action.yml.
655pub(crate) struct Action {
656 pub(crate) dir: PathBuf,
657 /// `owner/repo`, or empty for one in the repository.
658 pub(crate) repository: String,
659 pub(crate) local: bool,
660 pub(crate) manifest: Value,
661}
662
663/// An action's `pre`, run before the job's first step.
664pub(crate) struct PreStep {
665 /// Its step's `GITHUB_ACTION`, by which the step finds the state it saved.
666 pub(crate) key: String,
667 /// `pre-if`, `always()` when it has none.
668 pub(crate) condition: String,
669 pub(crate) action_dir: PathBuf,
670 pub(crate) script: String,
671 pub(crate) using: String,
672 /// The `env` context its condition reads.
673 pub(crate) context_env: BTreeMap<String, String>,
674 /// What its process is given: the step's variables and inputs.
675 pub(crate) env: BTreeMap<String, String>,
676}
677
678impl Job {
679 /// Finds `uses` (fetching it when it is another repository's) and reads
680 /// its action.yml. A problem is told in the log.
681 fn resolve_action(&mut self, uses: &str) -> Option<Action> {
682 let (name, git_ref) = uses.split_once('@').unwrap_or((uses, ""));
683 let source = if let Some(local) = name.strip_prefix("./") {
684 // A repository moved from GitHub renamed `.github` to `.g1t`, but
685 // its workflows still say `./.github/actions/…`.
686 let mut dir = self.workspace.join(local);
687 if let Some(rest) = local.strip_prefix(".github/")
688 && !dir.exists()
689 {
690 dir = self.workspace.join(".g1t").join(rest);
691 }
692 Source::Local(dir)
693 } else {
694 let mut parts = name.splitn(3, '/');
695 let (Some(owner), Some(repo)) = (parts.next(), parts.next()) else {
696 self.log.line(&format!("##[error]`{uses}` is not an action: use owner/repo@ref, owner/repo/path@ref, or ./path."));
697 return None;
698 };
699 if git_ref.is_empty() {
700 self.log.line(&format!("##[error]`{uses}` needs a version, such as @v4."));
701 return None;
702 }
703 Source::GitHub {
704 owner: owner.to_owned(),
705 repo: repo.to_owned(),
706 path: parts.next().unwrap_or_default().to_owned(),
707 git_ref: git_ref.to_owned(),
708 }
709 };
710 let (dir, repository) = match &source {
711 Source::Local(dir) => (dir.clone(), String::new()),
712 Source::GitHub { owner, repo, path, git_ref } => {
713 let root = self.fetch_remote_action(owner, repo, git_ref)?;
714 (if path.is_empty() { root } else { root.join(path) }, format!("{owner}/{repo}"))
715 }
716 };
717 let Some(manifest) = ["action.yml", "action.yaml"].iter().map(|f| dir.join(f)).find(|p| p.exists()) else {
718 self.log.line(&format!("##[error]`{uses}` has no action.yml."));
719 return None;
720 };
721 let Some(yaml) = std::fs::read_to_string(&manifest).ok().and_then(|text| serde_yaml::from_str::<serde_yaml::Value>(&text).ok()) else {
722 self.log.line(&format!("##[error]`{uses}`: its action.yml does not read."));
723 return None;
724 };
725 Some(Action { dir, repository, local: matches!(source, Source::Local(_)), manifest: yaml_to_json(&yaml) })
726 }
727
728 /// An action's inputs: what the step gives, else the action's defaults,
729 /// read with `contexts`. With `warn`, inputs the action does not declare
730 /// are warned of, as GitHub's runner does.
731 fn action_inputs(&mut self, action: &Value, with: &BTreeMap<String, String>, contexts: &Map<String, Value>, warn: bool) -> BTreeMap<String, String> {
732 let mut inputs: BTreeMap<String, String> = BTreeMap::new();
733 let mut declared_names = Vec::new();
734 if let Some(Value::Object(declared)) = action.get("inputs") {
735 for (input, spec) in declared {
736 declared_names.push(input.clone());
737 let given = with.iter().find(|(k, _)| k.eq_ignore_ascii_case(input)).map(|(_, v)| v.clone());
738 let value = match given {
739 Some(value) => value,
740 None => match spec.get("default") {
741 Some(default) => {
742 let default = self.with_scope(contexts, |scope| expr::interpolate_value(default, scope)).unwrap_or(Value::Null);
743 expr::to_text(&default)
744 }
745 None => String::new(),
746 },
747 };
748 inputs.insert(input.clone(), value);
749 }
750 }
751 for (key, value) in with {
752 if !inputs.keys().any(|k| k.eq_ignore_ascii_case(key)) {
753 inputs.insert(key.clone(), value.clone());
754 }
755 }
756 let docker = action["runs"]["using"].as_str().is_some_and(|u| u.eq_ignore_ascii_case("docker"));
757 if warn && let Some(warning) = unexpected_inputs(&declared_names, &with.keys().cloned().collect::<Vec<_>>(), docker) {
758 self.log.line(&format!("##[warning]{warning}"));
759 }
760 inputs
761 }
762
763 /// The `pre`s of the job's steps, in step order, each with its step's
764 /// index: those of JavaScript actions from other repositories, fetched
765 /// now, as GitHub's runner fetches every action while it sets up the
766 /// job. An action in the repository is not there yet, so its `pre` is
767 /// not run, as on GitHub.
768 pub(crate) fn pre_steps(&mut self, steps: &[Map<String, Value>]) -> Vec<(usize, PreStep)> {
769 let frame = Frame::default();
770 let mut pres = Vec::new();
771 for (index, step) in steps.iter().enumerate() {
772 let Some(uses) = step.get("uses").map(|u| expr::to_text(u).trim().to_owned()) else { continue };
773 // Only another repository's action, which `prepare_actions` has
774 // already downloaded.
775 if remote_action(&uses, self.docker_hosted).is_none() {
776 continue;
777 }
778 let Some(found) = self.resolve_action(&uses) else { continue };
779 let runs = found.manifest.get("runs").cloned().unwrap_or(Value::Null);
780 let using = runs.get("using").map(expr::to_text).unwrap_or_default().to_ascii_lowercase();
781 let Some(script) = runs.get("pre").map(expr::to_text).filter(|_| using.starts_with("node")) else { continue };
782 let env_before = self.env_context(&frame);
783 let contexts = self.contexts_for(&frame, &env_before);
784 let with = self.step_with(step, &contexts);
785 let context_env = self.step_env(step, &contexts, env_before);
786 let contexts = self.contexts_for(&frame, &context_env);
787 let inputs = self.action_inputs(&found.manifest, &with, &contexts, false);
788 let key = step.get(super::ACTION_NAME).map(expr::to_text).unwrap_or_default();
789 let git_ref = uses.split_once('@').map_or("", |(_, r)| r);
790 let mut env = action_env(&context_env, &key, &found, git_ref);
791 for (input, value) in &inputs {
792 env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
793 }
794 let condition = runs.get("pre-if").map(expr::to_text).unwrap_or_else(|| "always()".into());
795 pres.push((index, PreStep { key, condition, action_dir: found.dir, script, using, context_env, env }));
796 }
797 pres
798 }
799
800 /// Whether a `pre`'s `pre-if` holds, read against the job's status.
801 pub(crate) fn pre_runs(&self, pre: &PreStep) -> bool {
802 let contexts = self.contexts_for(&Frame::default(), &pre.context_env);
803 self.with_scope(&contexts, |scope| expr::condition(&pre.condition, scope)).unwrap_or(true)
804 }
805
806 /// Runs a `pre`, keeping the state it saves for its step's main and post.
807 pub(crate) fn run_pre(&mut self, pre: &PreStep) -> bool {
808 let ok = self.run_node(&pre.action_dir, &pre.script, &pre.env, &pre.using);
809 let state = std::mem::take(&mut self.last_node_state);
810 self.last_node_outputs.clear();
811 self.pre_state.insert(pre.key.clone(), state);
812 ok
813 }
814}
815
816/// The variables a step running an action gets over its own: which
817/// action it is and where.
818fn action_env(env: &BTreeMap<String, String>, action_name: &str, action: &Action, git_ref: &str) -> BTreeMap<String, String> {
819 let mut step_env = env.clone();
820 step_env.insert("GITHUB_ACTION".into(), action_name.to_owned());
821 step_env.insert("GITHUB_ACTION_REPOSITORY".into(), action.repository.clone());
822 step_env.insert("GITHUB_ACTION_REF".into(), git_ref.to_owned());
823 step_env.insert("GITHUB_ACTION_PATH".into(), action.dir.display().to_string());
824 step_env
825}
826
827/// GitHub's warning for inputs a step gives that its action does not
828/// declare (a Docker action also takes `entryPoint` and `args`), if any.
829fn unexpected_inputs(declared: &[String], given: &[String], docker: bool) -> Option<String> {
830 let mut valid: Vec<String> = declared.to_vec();
831 if docker {
832 valid.extend(["entryPoint".to_owned(), "args".to_owned()]);
833 }
834 let unexpected: Vec<&str> = given.iter().filter(|g| !valid.iter().any(|v| v.eq_ignore_ascii_case(g))).map(String::as_str).collect();
835 if unexpected.is_empty() {
836 return None;
837 }
838 Some(format!("Unexpected input(s) '{}', valid inputs are ['{}']", unexpected.join("', '"), valid.join("', '")))
839}
840
841/// The Node an action's `runs.using` asks for: for `node20` (or any
842/// `nodeNN`), the binary `G1T_NODE20` names or `/opt/node20/bin/node`,
843/// when there is one; otherwise, and for anything else, the `node` on
844/// `PATH`, the runner's own (Node 24). Nothing is logged either way.
845pub(crate) fn node_program(using: &str, var: impl Fn(&str) -> Option<String>, exists: impl Fn(&Path) -> bool) -> String {
846 let using = using.trim().to_ascii_lowercase();
847 let versioned = using.strip_prefix("node").is_some_and(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit()));
848 if versioned {
849 let named = var(&format!("G1T_{}", using.to_ascii_uppercase())).filter(|p| !p.is_empty() && exists(Path::new(p)));
850 if let Some(path) = named {
851 return path;
852 }
853 let standard = format!("/opt/{using}/bin/node");
854 if exists(Path::new(&standard)) {
855 return standard;
856 }
857 }
858 "node".into()
859}
860
861#[cfg(test)]
862mod tests {
863 use super::*;
864
865 #[test]
866 fn the_actions_fetched_before_the_first_step_are_other_repositories() {
867 let found = |uses: &str| remote_action(uses, true);
868 assert_eq!(found("actions/setup-node@v4"), Some(("actions".into(), "setup-node".into(), "v4".into())));
869 assert_eq!(found(" github/codeql-action/init@v3 "), Some(("github".into(), "codeql-action".into(), "v3".into())));
870 // Done by the runner itself, local, an image, or no version.
871 assert_eq!(found("actions/checkout@v4"), None);
872 assert_eq!(found("Actions/Cache/Restore@v4"), None);
873 assert_eq!(found("docker/setup-buildx-action@v3"), None);
874 assert!(remote_action("docker/setup-buildx-action@v3", false).is_some());
875 assert_eq!(found("./.g1t/actions/setup"), None);
876 assert_eq!(found("docker://alpine:3"), None);
877 assert_eq!(found("actions/setup-node"), None);
878 assert_eq!(found("setup-node@v4"), None);
879 }
880
881 #[test]
882 fn unexpected_inputs_are_named_with_the_valid_ones() {
883 let declared = vec!["token".to_owned(), "path".to_owned()];
884 assert_eq!(unexpected_inputs(&declared, &["Token".into()], false), None);
885 assert_eq!(
886 unexpected_inputs(&declared, &["token".into(), "foo".into(), "bar".into()], false).as_deref(),
887 Some("Unexpected input(s) 'foo', 'bar', valid inputs are ['token', 'path']")
888 );
889 assert_eq!(unexpected_inputs(&[], &["x".into()], false).as_deref(), Some("Unexpected input(s) 'x', valid inputs are ['']"));
890 // A Docker action takes `entrypoint` and `args` too.
891 assert_eq!(unexpected_inputs(&declared, &["entrypoint".into(), "args".into()], true), None);
892 assert!(unexpected_inputs(&declared, &["args".into()], false).is_some());
893 }
894
895 #[test]
896 fn node20_runs_with_a_node20_where_there_is_one() {
897 let none = |_: &str| None;
898 assert_eq!(node_program("node20", none, |_| false), "node");
899 assert_eq!(node_program("node20", none, |p| p == Path::new("/opt/node20/bin/node")), "/opt/node20/bin/node");
900 let named = |name: &str| (name == "G1T_NODE20").then(|| "/usr/local/n20/bin/node".to_owned());
901 assert_eq!(node_program("Node20", named, |_| true), "/usr/local/n20/bin/node");
902 // A named Node that is not there is passed over.
903 assert_eq!(node_program("node20", named, |p| p == Path::new("/opt/node20/bin/node")), "/opt/node20/bin/node");
904 assert_eq!(node_program("node24", none, |_| false), "node");
905 assert_eq!(node_program("composite", none, |_| true), "node");
906 assert_eq!(node_program("node", none, |_| true), "node");
907 }
908}