Skip to content
Sign up

flagon-io/g1t

public
0
828 linesCodeBlameRaw
1//! `uses:` steps: `actions/checkout` done natively against g1t, actions
2//! fetched from another repository on g1t (or else GitHub, at the commit
3//! g1t resolved its ref to, from the archive g1t keeps of it when it has
4//! one) and run as they are (JavaScript, composite and Docker),
5//! `docker://` images, and a few of GitHub's own whose services g1t does
6//! not have yet. Every action a job's steps name is downloaded before the
7//! first step runs, as GitHub's runner does.
8
9use std::collections::BTreeMap;
10use std::path::{Path, PathBuf};
11use std::process::Command;
12use std::time::Duration;
13
14use base64::Engine;
15use base64::engine::general_purpose::STANDARD;
16use g1t_actions::expr;
17use g1t_actions::workflow::yaml_to_json;
18use serde_json::{Map, Value, json};
19
20use super::containers::{self, DockerRun};
21use super::files::StepFiles;
22use super::process::{self, Commands, Ended};
23use super::{Frame, Job, Post, PostRun};
24
25const ACTIONS_DIR: &str = "/home/runner/_actions";
26/// How often a wait on g1t for an action says it is still waiting, as
27/// transfers say how far they have got.
28const WAITING_EVERY: Duration = Duration::from_secs(5);
29
30/// Where an action comes from.
31enum Source {
32 Local(PathBuf),
33 /// Another repository: on g1t when g1t has it and this one may use
34 /// it, otherwise on GitHub.
35 GitHub { owner: String, repo: String, path: String, git_ref: String },
36}
37
38fn safe(part: &str) -> bool {
39 !part.is_empty() && part.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/')) && !part.contains("..")
40}
41
42fn is_sha(text: &str) -> bool {
43 text.len() == 40 && text.chars().all(|c| c.is_ascii_hexdigit())
44}
45
46/// Whether the runner does `name` (lowercased, without its ref) itself
47/// rather than fetching it.
48fn native(name: &str, docker_hosted: bool) -> bool {
49 matches!(
50 name,
51 "actions/upload-artifact"
52 | "actions/upload-artifact/merge"
53 | "actions/download-artifact"
54 ) || (docker_hosted && name == "docker/setup-buildx-action")
55}
56
57/// The repository and ref of an action `uses` fetches from another
58/// repository: (owner, repo, ref). None for one that is local, a Docker
59/// image, done natively, or not a name at all.
60fn remote_action(uses: &str, docker_hosted: bool) -> Option<(String, String, String)> {
61 let uses = uses.trim();
62 if uses.starts_with("./") || uses.starts_with("docker://") {
63 return None;
64 }
65 let (name, git_ref) = uses.split_once('@')?;
66 if git_ref.is_empty() || native(&name.to_ascii_lowercase(), docker_hosted) {
67 return None;
68 }
69 let mut parts = name.splitn(3, '/');
70 let (owner, repo) = (parts.next()?, parts.next()?);
71 (!owner.is_empty() && !repo.is_empty()).then(|| (owner.to_owned(), repo.to_owned(), git_ref.to_owned()))
72}
73
74impl Job {
75 /// Runs a git command, logging it; the credential header is never logged.
76 fn git(&mut self, dir: &Path, args: &[&str], auth: Option<&str>) -> bool {
77 let shown: Vec<&str> = args.to_vec();
78 self.log.line(&format!("[command]git {}", shown.join(" ")));
79 let mut command = Command::new("git");
80 command.current_dir(dir);
81 if let Some(header) = auth {
82 command.args(["-c", &format!("http.extraheader={header}")]);
83 }
84 command.args(args).env("GIT_TERMINAL_PROMPT", "0");
85 let mut commands = Commands::default();
86 matches!(process::run(command, Duration::from_secs(600).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
87 }
88
89 /// A fetch, tried again after a short wait when it fails: a transfer
90 /// cut short on the way ("transfer closed with N bytes remaining") is
91 /// over by the next try. Three tries in all, as actions/checkout does.
92 fn fetch_retrying(&mut self, dir: &Path, args: &[&str], auth: Option<&str>) -> bool {
93 for (attempt, wait) in [0u64, 2, 5].into_iter().enumerate() {
94 if attempt > 0 {
95 self.log.line(&format!("The fetch failed; trying again in {wait} s ({} of 3).", attempt + 1));
96 std::thread::sleep(Duration::from_secs(wait));
97 }
98 if self.git(dir, args, auth) {
99 return true;
100 }
101 }
102 false
103 }
104
105 /// Fetches another repository's action, once per job: from g1t when
106 /// g1t has the repository and this one may use it, otherwise from
107 /// GitHub. A private repository on g1t that may not be used here fails
108 /// the step, saying why, rather than fetching something else by its
109 /// name.
110 fn fetch_remote_action(&mut self, owner: &str, repo: &str, git_ref: &str) -> Option<PathBuf> {
111 let on_g1t = super::paths::under_home(ACTIONS_DIR).join("_g1t").join(owner).join(repo).join(git_ref);
112 let on_github = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
113 for dir in [&on_g1t, &on_github] {
114 if dir.join(".g1t-fetched").exists() {
115 return Some(dir.clone());
116 }
117 }
118 if !(safe(owner) && safe(repo) && safe(git_ref)) {
119 self.log.line(&format!("##[error]`{owner}/{repo}@{git_ref}` is not a name g1t can fetch."));
120 return None;
121 }
122 // The first job to use an action at a commit waits while g1t
123 // fetches its archive from GitHub, up to two minutes: the log says
124 // what it is waiting for, and that it still is, as GitHub's does.
125 self.log.line(&format!("Getting action download info for '{owner}/{repo}@{git_ref}'"));
126 let repository = format!("{owner}/{repo}");
127 let asked = (repository.clone(), git_ref.to_owned());
128 let found = self.log.waiting(
129 WAITING_EVERY,
130 move |api| api.action(&asked.0, &asked.1),
131 |seconds| format!("Waiting for g1t to fetch '{repository}@{git_ref}' ({seconds}s)"),
132 );
133 match found {
134 Ok(found) if found["source"] == "g1t" => self.fetch_g1t_action(&on_g1t, owner, repo, git_ref, &found),
135 Ok(found) => {
136 if let Some(note) = found["note"].as_str() {
137 self.log.line(&format!("##[warning]{note}"));
138 }
139 let sha = found["sha"].as_str().filter(|sha| is_sha(sha)).map(str::to_owned);
140 if let Some(sha) = &sha
141 && found["archive"] == true
142 && let Some(dir) = self.fetch_archived_action(owner, repo, git_ref, sha)
143 {
144 return Some(dir);
145 }
146 self.fetch_action(owner, repo, git_ref, sha.as_deref())
147 }
148 Err((true, why)) => {
149 self.log.line(&format!("##[error]{why}"));
150 None
151 }
152 Err((false, why)) => {
153 self.log.line(&format!("##[warning]g1t could not say where {owner}/{repo} is ({why}); fetching it from GitHub."));
154 self.fetch_action(owner, repo, git_ref, None)
155 }
156 }
157 }
158
159 /// Fetches an action from a repository on g1t, at its ref, with the
160 /// read-only token g1t gave for it when it is private.
161 fn fetch_g1t_action(&mut self, dir: &Path, owner: &str, repo: &str, git_ref: &str, found: &Value) -> Option<PathBuf> {
162 let url = found["url"].as_str().unwrap_or_default().to_owned();
163 let token = found["token"].as_str().filter(|token| !token.is_empty()).map(str::to_owned);
164 if let Some(token) = &token {
165 self.log.add_mask(token);
166 }
167 let auth = token.map(|token| format!("AUTHORIZATION: basic {}", STANDARD.encode(format!("x-access-token:{token}"))));
168 self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' from g1t"));
169 let _ = std::fs::remove_dir_all(dir);
170 if std::fs::create_dir_all(dir).is_err() || !self.git(dir, &["init", "--quiet"], None) || !self.git(dir, &["remote", "add", "origin", &url], None) {
171 return None;
172 }
173 // A branch or tag at its tip; a commit may need the history.
174 let shallow = self.fetch_retrying(dir, &["fetch", "--depth=1", "--no-tags", "--quiet", "origin", git_ref], auth.as_deref());
175 let checked_out = if shallow {
176 self.git(dir, &["checkout", "--quiet", "--force", "--detach", "FETCH_HEAD"], None)
177 } else {
178 self.fetch_retrying(dir, &["fetch", "--quiet", "--tags", "origin", "+refs/heads/*:refs/remotes/origin/*"], auth.as_deref())
179 && self.git(dir, &["checkout", "--quiet", "--force", "--detach", git_ref], None)
180 };
181 if !checked_out {
182 self.log.line(&format!("##[error]Could not fetch {owner}/{repo}@{git_ref} from g1t: is {git_ref} a branch, tag or commit there?"));
183 let _ = std::fs::remove_dir_all(dir);
184 return None;
185 }
186 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
187 Some(dir.to_path_buf())
188 }
189
190 /// Downloads a GitHub action's archive that g1t keeps, at the commit
191 /// g1t resolved its ref to. None when it could not, and the job fetches
192 /// it from GitHub instead.
193 fn fetch_archived_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: &str) -> Option<PathBuf> {
194 let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
195 self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha}) from g1t"));
196 let archive = self.temp.join(format!("action{}.tar.gz", super::rand_id()));
197 let asked = (format!("{owner}/{repo}"), sha.to_owned(), archive.clone());
198 let downloaded = self.log.waiting(
199 WAITING_EVERY,
200 move |api| api.action_archive(&asked.0, &asked.1, &asked.2),
201 |seconds| format!("Downloading '{owner}/{repo}@{git_ref}' from g1t ({seconds}s)"),
202 );
203 let unpacked = match downloaded {
204 Ok(_) => {
205 let _ = std::fs::remove_dir_all(&dir);
206 let _ = std::fs::create_dir_all(&dir);
207 let mut command = Command::new("tar");
208 command.arg("-xzf").arg(&archive).arg("-C").arg(&dir).arg("--strip-components=1");
209 if cfg!(windows) {
210 // GNU tar on Windows reads `C:...` as a host and a file.
211 command.arg("--force-local");
212 }
213 let mut commands = Commands::default();
214 matches!(process::run(command, Duration::from_secs(300).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
215 }
216 Err(why) => {
217 self.log.line(&format!("g1t could not send it ({why}); fetching it from GitHub."));
218 false
219 }
220 };
221 let _ = std::fs::remove_file(&archive);
222 if !unpacked {
223 let _ = std::fs::remove_dir_all(&dir);
224 return None;
225 }
226 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
227 Some(dir)
228 }
229
230 /// Fetches an action from GitHub, once per job: at `sha` when g1t
231 /// resolved its ref to one, so it is the commit g1t named.
232 fn fetch_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: Option<&str>) -> Option<PathBuf> {
233 let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
234 if dir.join(".g1t-fetched").exists() {
235 return Some(dir);
236 }
237 if !(safe(owner) && safe(repo) && safe(git_ref)) {
238 self.log.line(&format!("##[error]`{owner}/{repo}@{git_ref}` is not a name g1t can fetch."));
239 return None;
240 }
241 match sha {
242 Some(sha) => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha})")),
243 None => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}'")),
244 }
245 let _ = std::fs::create_dir_all(&dir);
246 let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{}", sha.unwrap_or(git_ref));
247 let script = format!("set -o pipefail; curl -fsSL --retry 3 '{url}' | tar{} -xz -C '{}' --strip-components=1", super::blobs::tar_local(), dir.display());
248 let mut command = Command::new("bash");
249 command.args(["-c", &script]);
250 let mut commands = Commands::default();
251 match process::run(command, Duration::from_secs(300).min(self.remaining_time()), &mut self.log, &mut commands) {
252 Ok(Ended::Exited(0)) => {
253 let _ = std::fs::write(dir.join(".g1t-fetched"), "");
254 Some(dir)
255 }
256 _ => {
257 self.log.line(&format!("##[error]Could not download {owner}/{repo}@{git_ref} from GitHub."));
258 let _ = std::fs::remove_dir_all(&dir);
259 None
260 }
261 }
262 }
263
264 /// Downloads every action the job's steps fetch from other
265 /// repositories, before the first step runs, as GitHub's runner does in
266 /// "Set up job". False when one could not be: the job fails before
267 /// anything runs. Actions a composite action uses are fetched as it
268 /// runs.
269 pub(crate) fn prepare_actions(&mut self, steps: &[Map<String, Value>]) -> bool {
270 let mut wanted: Vec<(String, String, String)> = Vec::new();
271 for step in steps {
272 if let Some(found) = step.get("uses").and_then(Value::as_str).and_then(|uses| remote_action(uses, self.docker_hosted))
273 && !wanted.contains(&found)
274 {
275 wanted.push(found);
276 }
277 }
278 if wanted.is_empty() {
279 return true;
280 }
281 self.log.line("Prepare all required actions");
282 self.log.line("Getting action download info");
283 let mut ready = true;
284 for (owner, repo, git_ref) in &wanted {
285 if self.fetch_remote_action(owner, repo, git_ref).is_none() {
286 ready = false;
287 }
288 }
289 if !ready {
290 self.log.line("##[error]Not every action this job uses could be downloaded, so no step was run.");
291 }
292 ready
293 }
294
295 /// Runs a JavaScript file of an action with Node: the one its
296 /// `runs.using` names where the machine has it (`node_program`).
297 pub(crate) fn run_node(&mut self, action_dir: &Path, script: &str, env: &BTreeMap<String, String>, using: &str) -> bool {
298 let id = format!("node{}", super::rand_id());
299 let Ok(files) = StepFiles::new(&self.temp, &id) else { return false };
300 let full = self.process_env(env, &files);
301 let script_path = action_dir.join(script);
302 // In a job container whose image runs the runner's Node, the action
303 // runs there, as on GitHub.
304 let in_container = self.container.as_ref().filter(|c| c.node).map(|c| c.path.clone()).and_then(|image_path| {
305 let inside = self.container_env(env, full.clone(), &image_path);
306 self.in_container(containers::CONTAINER_NODE, &[script_path.display().to_string()], &self.workspace, inside)
307 });
308 let command = match in_container {
309 Some(command) => command,
310 None => {
311 let node = node_program(using, |name| std::env::var(name).ok(), |path| path.is_file());
312 let mut command = Command::new(node);
313 command.arg(&script_path).current_dir(&self.workspace).env_clear().envs(full);
314 command
315 }
316 };
317 let mut commands = Commands {
318 debug: self.debug,
319 ..Commands::default()
320 };
321 let ended = process::run(command, Duration::from_secs(6 * 3600).min(self.deadline_left()), &mut self.log, &mut commands);
322 let ok = matches!(ended, Ok(Ended::Exited(0)));
323 match ended {
324 Ok(Ended::Exited(code)) if code != 0 => self.log.line(&format!("##[error]The action exited with code {code}.")),
325 Ok(Ended::TimedOut) => self.log.line("##[error]The action ran past its time limit and was stopped."),
326 _ => {}
327 }
328 let (outputs, state) = self.absorb(&files, &commands);
329 self.last_node_outputs = outputs;
330 self.last_node_state = state;
331 ok
332 }
333
334 fn log_header(&mut self, uses: &str, with: &BTreeMap<String, String>, env: &BTreeMap<String, String>) {
335 for line in super::uses_header(uses, with, env) {
336 self.log.line(&line);
337 }
338 }
339
340 fn deadline_left(&self) -> Duration {
341 self.remaining_time()
342 }
343
344 /// Runs a `uses:` step. Returns whether it succeeded, and its outputs.
345 #[allow(clippy::too_many_arguments)]
346 pub(crate) fn uses(
347 &mut self,
348 uses: &str,
349 with: &BTreeMap<String, String>,
350 env: &BTreeMap<String, String>,
351 frame: &Frame,
352 title: &str,
353 action_name: &str,
354 _timeout: Duration,
355 ) -> (bool, BTreeMap<String, String>) {
356 let uses = uses.trim();
357 let (name, git_ref) = uses.split_once('@').unwrap_or((uses, ""));
358 let lower = name.to_ascii_lowercase();
359 // The step opens as GitHub's do: `Run` and the action, its inputs
360 // and variables, folded. An action fetched from a repository lists
361 // its inputs as they are read, its defaults among them.
362 let fetched = !uses.starts_with("docker://") && !native(&lower, self.docker_hosted);
363 if !fetched {
364 self.log_header(uses, with, env);
365 }
366 if let Some(image) = uses.strip_prefix("docker://") {
367 // `with.args` and `with.entrypoint` are the container's; every
368 // input is also an `INPUT_` variable, as on GitHub.
369 let mut step_env = env.clone();
370 step_env.insert("GITHUB_ACTION".into(), action_name.to_owned());
371 for (input, value) in with {
372 step_env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
373 }
374 let run = DockerRun {
375 image: image.to_owned(),
376 entrypoint: with.get("entrypoint").filter(|e| !e.is_empty()).cloned(),
377 args: with.get("args").map(|a| containers::split_words(a)).unwrap_or_default(),
378 env: step_env,
379 };
380 let (ok, outputs, _) = self.run_docker(&run);
381 return (ok, outputs);
382 }
383 if lower == "docker/setup-buildx-action" && native(&lower, self.docker_hosted) {
384 return self.setup_buildx(with);
385 }
386 match lower.as_str() {
387 "actions/upload-artifact" => return self.upload_artifact(with),
388 "actions/upload-artifact/merge" => return self.merge_artifacts(with),
389 "actions/download-artifact" => return self.download_artifact(with),
390 _ => {}
391 }
392 let Some(found) = self.resolve_action(uses) else {
393 self.log_header(uses, with, env);
394 return (false, BTreeMap::new());
395 };
396 let Action { dir, local, manifest: action, .. } = &found;
397
398 // Inputs: what the step gives, else the action's defaults.
399 let env_context = env.clone();
400 let contexts = self.contexts_for(frame, &env_context);
401 let inputs = self.action_inputs(action, with, &contexts, true);
402 self.log_header(uses, &inputs, env);
403
404 let runs = action.get("runs").cloned().unwrap_or(Value::Null);
405 let using = runs.get("using").map(expr::to_text).unwrap_or_default().to_ascii_lowercase();
406 let mut step_env = action_env(env, action_name, &found, git_ref);
407
408 if using.starts_with("node") {
409 for (input, value) in &inputs {
410 step_env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
411 }
412 let condition_of = |key: &str| runs.get(key).map(expr::to_text).unwrap_or_else(|| "always()".into());
413 // State an action saves, `STATE_<name>` for its later stages.
414 let mut state = BTreeMap::new();
415 if let Some(pre) = runs.get("pre").map(expr::to_text) {
416 if frame.action_path.is_some() {
417 // An action inside a composite action: its `pre` right
418 // before its `main`.
419 let run_pre = self.with_scope(&contexts, |scope| expr::condition(&condition_of("pre-if"), scope)).unwrap_or(true);
420 if run_pre {
421 if !self.run_node(dir, &pre, &step_env, &using) {
422 return (false, BTreeMap::new());
423 }
424 state = std::mem::take(&mut self.last_node_state);
425 }
426 } else if *local {
427 self.log.line(&format!("##[warning]`pre` execution is not supported for local action from '{uses}'"));
428 } else {
429 // Run before the job's first step (`pre_steps`).
430 state = self.pre_state.remove(action_name).unwrap_or_default();
431 }
432 }
433 let Some(main) = runs.get("main").map(expr::to_text) else {
434 self.log.line(&format!("##[error]`{uses}` has no `runs.main`."));
435 return (false, BTreeMap::new());
436 };
437 let mut main_env = step_env.clone();
438 for (name, value) in &state {
439 main_env.insert(format!("STATE_{name}"), value.clone());
440 }
441 let ok = self.run_node(dir, &main, &main_env, &using);
442 let outputs = std::mem::take(&mut self.last_node_outputs);
443 state.extend(std::mem::take(&mut self.last_node_state));
444 if let Some(post) = runs.get("post").map(expr::to_text) {
445 let mut post_env = step_env.clone();
446 for (name, value) in state {
447 post_env.insert(format!("STATE_{name}"), value);
448 }
449 self.posts.push(Post {
450 name: format!("Post {title}"),
451 condition: condition_of("post-if"),
452 env: post_env,
453 run: PostRun::Node { action_dir: dir.clone(), script: post, using: using.clone() },
454 });
455 }
456 return (ok, outputs);
457 }
458 if using == "composite" {
459 let mut inner = Frame {
460 steps: Map::new(),
461 inputs: Some(Value::Object(inputs.iter().map(|(k, v)| (k.clone(), json!(v))).collect())),
462 action_path: Some(dir.display().to_string()),
463 env: env.clone(),
464 };
465 let mut steps: Vec<Map<String, Value>> = runs.get("steps").and_then(Value::as_array).map(|s| s.iter().filter_map(|s| s.as_object().cloned()).collect()).unwrap_or_default();
466 super::name_steps(&mut steps);
467 let was_failed = self.failed;
468 // A composite's steps see their own success, not the job's.
469 self.failed = false;
470 let mut ok = true;
471 for step in &steps {
472 if !self.step(&mut inner, step, 0, false, &Map::new()) {
473 ok = false;
474 }
475 }
476 let contexts = self.contexts_for(&inner, &inner.env.clone());
477 let mut outputs = BTreeMap::new();
478 if let Some(Value::Object(declared)) = action.get("outputs") {
479 for (name, spec) in declared {
480 if let Some(value) = spec.get("value") {
481 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
482 outputs.insert(name.clone(), expr::to_text(&value));
483 }
484 }
485 }
486 self.failed = was_failed;
487 return (ok, outputs);
488 }
489 if using == "docker" {
490 return self.docker_action(uses, dir, &runs, &inputs, &step_env, frame, title);
491 }
492 self.log.line(&format!("##[error]`{uses}` runs with `{using}`, which g1t does not know."));
493 (false, BTreeMap::new())
494 }
495
496 /// A Docker action: its image built from its Dockerfile (or pulled,
497 /// for `docker://`), then run with its `args`, `entrypoint` and `env`,
498 /// its inputs as `INPUT_` variables, and `pre-entrypoint` and
499 /// `post-entrypoint` around it.
500 #[allow(clippy::too_many_arguments)]
501 fn docker_action(
502 &mut self,
503 uses: &str,
504 dir: &Path,
505 runs: &Value,
506 inputs: &BTreeMap<String, String>,
507 step_env: &BTreeMap<String, String>,
508 frame: &Frame,
509 title: &str,
510 ) -> (bool, BTreeMap<String, String>) {
511 let image = runs.get("image").map(expr::to_text).unwrap_or_default();
512 let image = if let Some(pulled) = image.strip_prefix("docker://") {
513 pulled.to_owned()
514 } else if image.is_empty() {
515 self.log.line(&format!("##[error]`{uses}` has no `runs.image`."));
516 return (false, BTreeMap::new());
517 } else {
518 match self.build_action_image(dir, &image, uses) {
519 Some(tag) => tag,
520 None => return (false, BTreeMap::new()),
521 }
522 };
523 // `args` and `env` read with the action's own inputs.
524 let mut env = step_env.clone();
525 for (input, value) in inputs {
526 env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
527 }
528 let mut scope_frame = frame.clone();
529 scope_frame.inputs = Some(Value::Object(inputs.iter().map(|(k, v)| (k.clone(), json!(v))).collect()));
530 let contexts = self.contexts_for(&scope_frame, &env);
531 if let Some(Value::Object(own)) = runs.get("env") {
532 for (name, value) in own {
533 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
534 env.insert(name.clone(), expr::to_text(&value));
535 }
536 }
537 let args: Vec<String> = match runs.get("args") {
538 Some(Value::Array(items)) => items
539 .iter()
540 .map(|item| {
541 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(item, scope)).unwrap_or(Value::Null);
542 expr::to_text(&value)
543 })
544 .collect(),
545 _ => Vec::new(),
546 };
547 let entry = |key: &str| runs.get(key).map(expr::to_text).filter(|e| !e.is_empty());
548 if let Some(pre) = entry("pre-entrypoint") {
549 let run = DockerRun { image: image.clone(), entrypoint: Some(pre), args: Vec::new(), env: env.clone() };
550 if !self.run_docker(&run).0 {
551 return (false, BTreeMap::new());
552 }
553 }
554 let run = DockerRun { image: image.clone(), entrypoint: entry("entrypoint"), args, env: env.clone() };
555 let (ok, outputs, state) = self.run_docker(&run);
556 if let Some(post) = entry("post-entrypoint") {
557 let mut post_env = env;
558 for (name, value) in state {
559 post_env.insert(format!("STATE_{name}"), value);
560 }
561 self.posts.push(Post {
562 name: format!("Post {title}"),
563 condition: runs.get("post-if").map(expr::to_text).unwrap_or_else(|| "always()".into()),
564 env: BTreeMap::new(),
565 run: PostRun::Docker(DockerRun { image, entrypoint: Some(post), args: Vec::new(), env: post_env }),
566 });
567 }
568 (ok, outputs)
569 }
570}
571
572/// An action, read from its action.yml.
573pub(crate) struct Action {
574 pub(crate) dir: PathBuf,
575 /// `owner/repo`, or empty for one in the repository.
576 pub(crate) repository: String,
577 pub(crate) local: bool,
578 pub(crate) manifest: Value,
579}
580
581/// An action's `pre`, run before the job's first step.
582pub(crate) struct PreStep {
583 /// Its step's `GITHUB_ACTION`, by which the step finds the state it saved.
584 pub(crate) key: String,
585 /// `pre-if`, `always()` when it has none.
586 pub(crate) condition: String,
587 pub(crate) action_dir: PathBuf,
588 pub(crate) script: String,
589 pub(crate) using: String,
590 /// The `env` context its condition reads.
591 pub(crate) context_env: BTreeMap<String, String>,
592 /// What its process is given: the step's variables and inputs.
593 pub(crate) env: BTreeMap<String, String>,
594}
595
596impl Job {
597 /// Finds `uses` (fetching it when it is another repository's) and reads
598 /// its action.yml. A problem is told in the log.
599 fn resolve_action(&mut self, uses: &str) -> Option<Action> {
600 let (name, git_ref) = uses.split_once('@').unwrap_or((uses, ""));
601 let source = if let Some(local) = name.strip_prefix("./") {
602 // A repository moved from GitHub renamed `.github` to `.g1t`, but
603 // its workflows still say `./.github/actions/…`.
604 let mut dir = self.workspace.join(local);
605 if let Some(rest) = local.strip_prefix(".github/")
606 && !dir.exists()
607 {
608 dir = self.workspace.join(".g1t").join(rest);
609 }
610 Source::Local(dir)
611 } else {
612 let mut parts = name.splitn(3, '/');
613 let (Some(owner), Some(repo)) = (parts.next(), parts.next()) else {
614 self.log.line(&format!("##[error]`{uses}` is not an action: use owner/repo@ref, owner/repo/path@ref, or ./path."));
615 return None;
616 };
617 if git_ref.is_empty() {
618 self.log.line(&format!("##[error]`{uses}` needs a version, such as @v4."));
619 return None;
620 }
621 Source::GitHub {
622 owner: owner.to_owned(),
623 repo: repo.to_owned(),
624 path: parts.next().unwrap_or_default().to_owned(),
625 git_ref: git_ref.to_owned(),
626 }
627 };
628 let (dir, repository) = match &source {
629 Source::Local(dir) => (dir.clone(), String::new()),
630 Source::GitHub { owner, repo, path, git_ref } => {
631 let root = self.fetch_remote_action(owner, repo, git_ref)?;
632 (if path.is_empty() { root } else { root.join(path) }, format!("{owner}/{repo}"))
633 }
634 };
635 let Some(manifest) = ["action.yml", "action.yaml"].iter().map(|f| dir.join(f)).find(|p| p.exists()) else {
636 self.log.line(&format!("##[error]`{uses}` has no action.yml."));
637 return None;
638 };
639 let Some(yaml) = std::fs::read_to_string(&manifest).ok().and_then(|text| serde_yaml::from_str::<serde_yaml::Value>(&text).ok()) else {
640 self.log.line(&format!("##[error]`{uses}`: its action.yml does not read."));
641 return None;
642 };
643 Some(Action { dir, repository, local: matches!(source, Source::Local(_)), manifest: yaml_to_json(&yaml) })
644 }
645
646 /// An action's inputs: what the step gives, else the action's defaults,
647 /// read with `contexts`. With `warn`, inputs the action does not declare
648 /// are warned of, as GitHub's runner does.
649 fn action_inputs(&mut self, action: &Value, with: &BTreeMap<String, String>, contexts: &Map<String, Value>, warn: bool) -> BTreeMap<String, String> {
650 let mut inputs: BTreeMap<String, String> = BTreeMap::new();
651 let mut declared_names = Vec::new();
652 if let Some(Value::Object(declared)) = action.get("inputs") {
653 for (input, spec) in declared {
654 declared_names.push(input.clone());
655 let given = with.iter().find(|(k, _)| k.eq_ignore_ascii_case(input)).map(|(_, v)| v.clone());
656 let value = match given {
657 Some(value) => value,
658 None => match spec.get("default") {
659 Some(default) => {
660 let default = self.with_scope(contexts, |scope| expr::interpolate_value(default, scope)).unwrap_or(Value::Null);
661 expr::to_text(&default)
662 }
663 None => String::new(),
664 },
665 };
666 inputs.insert(input.clone(), value);
667 }
668 }
669 for (key, value) in with {
670 if !inputs.keys().any(|k| k.eq_ignore_ascii_case(key)) {
671 inputs.insert(key.clone(), value.clone());
672 }
673 }
674 let docker = action["runs"]["using"].as_str().is_some_and(|u| u.eq_ignore_ascii_case("docker"));
675 if warn && let Some(warning) = unexpected_inputs(&declared_names, &with.keys().cloned().collect::<Vec<_>>(), docker) {
676 self.log.line(&format!("##[warning]{warning}"));
677 }
678 inputs
679 }
680
681 /// The `pre`s of the job's steps, in step order, each with its step's
682 /// index: those of JavaScript actions from other repositories, fetched
683 /// now, as GitHub's runner fetches every action while it sets up the
684 /// job. An action in the repository is not there yet, so its `pre` is
685 /// not run, as on GitHub.
686 pub(crate) fn pre_steps(&mut self, steps: &[Map<String, Value>]) -> Vec<(usize, PreStep)> {
687 let frame = Frame::default();
688 let mut pres = Vec::new();
689 for (index, step) in steps.iter().enumerate() {
690 let Some(uses) = step.get("uses").map(|u| expr::to_text(u).trim().to_owned()) else { continue };
691 // Only another repository's action, which `prepare_actions` has
692 // already downloaded.
693 if remote_action(&uses, self.docker_hosted).is_none() {
694 continue;
695 }
696 let Some(found) = self.resolve_action(&uses) else { continue };
697 let runs = found.manifest.get("runs").cloned().unwrap_or(Value::Null);
698 let using = runs.get("using").map(expr::to_text).unwrap_or_default().to_ascii_lowercase();
699 let Some(script) = runs.get("pre").map(expr::to_text).filter(|_| using.starts_with("node")) else { continue };
700 let env_before = self.env_context(&frame);
701 let contexts = self.contexts_for(&frame, &env_before);
702 let with = self.step_with(step, &contexts);
703 let context_env = self.step_env(step, &contexts, env_before);
704 let contexts = self.contexts_for(&frame, &context_env);
705 let inputs = self.action_inputs(&found.manifest, &with, &contexts, false);
706 let key = step.get(super::ACTION_NAME).map(expr::to_text).unwrap_or_default();
707 let git_ref = uses.split_once('@').map_or("", |(_, r)| r);
708 let mut env = action_env(&context_env, &key, &found, git_ref);
709 for (input, value) in &inputs {
710 env.insert(format!("INPUT_{}", input.replace(' ', "_").to_ascii_uppercase()), value.clone());
711 }
712 let condition = runs.get("pre-if").map(expr::to_text).unwrap_or_else(|| "always()".into());
713 pres.push((index, PreStep { key, condition, action_dir: found.dir, script, using, context_env, env }));
714 }
715 pres
716 }
717
718 /// Whether a `pre`'s `pre-if` holds, read against the job's status.
719 pub(crate) fn pre_runs(&self, pre: &PreStep) -> bool {
720 let contexts = self.contexts_for(&Frame::default(), &pre.context_env);
721 self.with_scope(&contexts, |scope| expr::condition(&pre.condition, scope)).unwrap_or(true)
722 }
723
724 /// Runs a `pre`, keeping the state it saves for its step's main and post.
725 pub(crate) fn run_pre(&mut self, pre: &PreStep) -> bool {
726 let ok = self.run_node(&pre.action_dir, &pre.script, &pre.env, &pre.using);
727 let state = std::mem::take(&mut self.last_node_state);
728 self.last_node_outputs.clear();
729 self.pre_state.insert(pre.key.clone(), state);
730 ok
731 }
732}
733
734/// The variables a step running an action gets over its own: which
735/// action it is and where.
736fn action_env(env: &BTreeMap<String, String>, action_name: &str, action: &Action, git_ref: &str) -> BTreeMap<String, String> {
737 let mut step_env = env.clone();
738 step_env.insert("GITHUB_ACTION".into(), action_name.to_owned());
739 step_env.insert("GITHUB_ACTION_REPOSITORY".into(), action.repository.clone());
740 step_env.insert("GITHUB_ACTION_REF".into(), git_ref.to_owned());
741 step_env.insert("GITHUB_ACTION_PATH".into(), action.dir.display().to_string());
742 step_env
743}
744
745/// GitHub's warning for inputs a step gives that its action does not
746/// declare (a Docker action also takes `entryPoint` and `args`), if any.
747fn unexpected_inputs(declared: &[String], given: &[String], docker: bool) -> Option<String> {
748 let mut valid: Vec<String> = declared.to_vec();
749 if docker {
750 valid.extend(["entryPoint".to_owned(), "args".to_owned()]);
751 }
752 let unexpected: Vec<&str> = given.iter().filter(|g| !valid.iter().any(|v| v.eq_ignore_ascii_case(g))).map(String::as_str).collect();
753 if unexpected.is_empty() {
754 return None;
755 }
756 Some(format!("Unexpected input(s) '{}', valid inputs are ['{}']", unexpected.join("', '"), valid.join("', '")))
757}
758
759/// The Node an action's `runs.using` asks for: for `node20` (or any
760/// `nodeNN`), the binary `G1T_NODE20` names or `/opt/node20/bin/node`,
761/// when there is one; otherwise, and for anything else, the `node` on
762/// `PATH`, the runner's own (Node 24). Nothing is logged either way.
763pub(crate) fn node_program(using: &str, var: impl Fn(&str) -> Option<String>, exists: impl Fn(&Path) -> bool) -> String {
764 let using = using.trim().to_ascii_lowercase();
765 let versioned = using.strip_prefix("node").is_some_and(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit()));
766 if versioned {
767 let named = var(&format!("G1T_{}", using.to_ascii_uppercase())).filter(|p| !p.is_empty() && exists(Path::new(p)));
768 if let Some(path) = named {
769 return path;
770 }
771 let standard = format!("/opt/{using}/bin/node");
772 if exists(Path::new(&standard)) {
773 return standard;
774 }
775 }
776 "node".into()
777}
778
779#[cfg(test)]
780mod tests {
781 use super::*;
782
783 #[test]
784 fn the_actions_fetched_before_the_first_step_are_other_repositories() {
785 let found = |uses: &str| remote_action(uses, true);
786 assert_eq!(found("actions/setup-node@v4"), Some(("actions".into(), "setup-node".into(), "v4".into())));
787 assert_eq!(found(" github/codeql-action/init@v3 "), Some(("github".into(), "codeql-action".into(), "v3".into())));
788 // actions/checkout and actions/cache are the real actions, fetched
789 // like any other.
790 assert_eq!(found("actions/checkout@v4"), Some(("actions".into(), "checkout".into(), "v4".into())));
791 assert_eq!(found("Actions/Cache/Restore@v4"), Some(("Actions".into(), "Cache".into(), "v4".into())));
792 // Done by the runner itself, local, an image, or no version.
793 assert_eq!(found("docker/setup-buildx-action@v3"), None);
794 assert!(remote_action("docker/setup-buildx-action@v3", false).is_some());
795 assert_eq!(found("./.g1t/actions/setup"), None);
796 assert_eq!(found("docker://alpine:3"), None);
797 assert_eq!(found("actions/setup-node"), None);
798 assert_eq!(found("setup-node@v4"), None);
799 }
800
801 #[test]
802 fn unexpected_inputs_are_named_with_the_valid_ones() {
803 let declared = vec!["token".to_owned(), "path".to_owned()];
804 assert_eq!(unexpected_inputs(&declared, &["Token".into()], false), None);
805 assert_eq!(
806 unexpected_inputs(&declared, &["token".into(), "foo".into(), "bar".into()], false).as_deref(),
807 Some("Unexpected input(s) 'foo', 'bar', valid inputs are ['token', 'path']")
808 );
809 assert_eq!(unexpected_inputs(&[], &["x".into()], false).as_deref(), Some("Unexpected input(s) 'x', valid inputs are ['']"));
810 // A Docker action takes `entrypoint` and `args` too.
811 assert_eq!(unexpected_inputs(&declared, &["entrypoint".into(), "args".into()], true), None);
812 assert!(unexpected_inputs(&declared, &["args".into()], false).is_some());
813 }
814
815 #[test]
816 fn node20_runs_with_a_node20_where_there_is_one() {
817 let none = |_: &str| None;
818 assert_eq!(node_program("node20", none, |_| false), "node");
819 assert_eq!(node_program("node20", none, |p| p == Path::new("/opt/node20/bin/node")), "/opt/node20/bin/node");
820 let named = |name: &str| (name == "G1T_NODE20").then(|| "/usr/local/n20/bin/node".to_owned());
821 assert_eq!(node_program("Node20", named, |_| true), "/usr/local/n20/bin/node");
822 // A named Node that is not there is passed over.
823 assert_eq!(node_program("node20", named, |p| p == Path::new("/opt/node20/bin/node")), "/opt/node20/bin/node");
824 assert_eq!(node_program("node24", none, |_| false), "node");
825 assert_eq!(node_program("composite", none, |_| true), "node");
826 assert_eq!(node_program("node", none, |_| true), "node");
827 }
828}