Skip to content
Sign up

flagon-io/g1t

public
0
816 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'main' into actions-toolkit-oidc-artifacts1//! Containers a job asks for, as GitHub's runner starts them, with the
2//! `docker` command: its `services:` (a database beside the steps), its
3//! `container:` (every step run inside an image), `uses: docker://image`
4//! steps and Docker actions (`runs.using: docker`).
5//!
6//! On g1t's machines the Engine is the job's own (crate::docker). Every
7//! container shares the job's network there, so a service is reached at
8//! `localhost:<port>` as on GitHub's runner, and by its name as from a job
9//! container: the API proxy makes each service's name mean 127.0.0.1.
10
11use std::collections::{BTreeMap, BTreeSet};
12use std::path::{Path, PathBuf};
13use std::process::Command;
14use std::time::{Duration, Instant};
15
16use g1t_actions::expr;
17use serde_json::{Map, Value, json};
18
19use super::files::StepFiles;
20use super::process::{self, Commands, Ended};
21use super::Job;
22
23/// Set in a job a self-hosted runner started inside its `container:` image.
24pub(crate) const IN_JOB_CONTAINER: &str = "G1T_JOB_CONTAINER";
25/// Where a job container finds the runner's Node, for JavaScript actions.
26pub(crate) const CONTAINER_NODE: &str = "/__e/node24/bin/node";
27/// GitHub's folders inside a Docker action's container.
28const GITHUB_WORKSPACE: &str = "/github/workspace";
29const GITHUB_HOME: &str = "/github/home";
30const GITHUB_WORKFLOW: &str = "/github/workflow";
31const GITHUB_FILE_COMMANDS: &str = "/github/file_commands";
32
33/// The job's own container, when it has `container:`.
34pub(crate) struct JobContainer {
35 pub(crate) id: String,
36 /// `bash`, or `sh` in an image without it.
37 pub(crate) shell: &'static str,
Merge branch 'claude/cool-newton-7nm8r9' (#36): real GitHub Actions on g1t, managed deploy workflows, g1t/deploy and the Sysbox runner 0.2.0. The merge keeps both sides' webhook events and API operations, says agent images need tar, guards the job engine's stop to Unix so the runner builds on Windows again, and keeps the moves tests' previews from going idle as the calendar passes their fixture's date.38 /// `python`, or `python3` in an image without it, for `shell: python`.
39 pub(crate) python: &'static str,
Merge branch 'main' into actions-toolkit-oidc-artifacts40 /// Whether the runner's Node runs in it.
41 pub(crate) node: bool,
42 /// The image's own `PATH`.
43 pub(crate) path: String,
44}
45
46/// A `services:` entry or `container:`, read.
47#[derive(Debug, Default, PartialEq)]
48pub(crate) struct ContainerSpec {
49 pub(crate) image: String,
50 pub(crate) env: BTreeMap<String, String>,
51 pub(crate) ports: Vec<String>,
52 pub(crate) volumes: Vec<String>,
53 pub(crate) options: Vec<String>,
54 pub(crate) credentials: Option<(String, String)>,
55 pub(crate) command: Vec<String>,
56 pub(crate) entrypoint: Option<String>,
57}
58
59/// Splits a command line as a shell would: words, with `'…'`, `"…"` and
60/// `\` quoting. No variables are expanded.
61pub(crate) fn split_words(text: &str) -> Vec<String> {
62 let mut words = Vec::new();
63 let mut word = String::new();
64 let mut started = false;
65 let mut chars = text.chars().peekable();
66 while let Some(c) = chars.next() {
67 match c {
68 '\'' => {
69 started = true;
70 for c in chars.by_ref() {
71 if c == '\'' {
72 break;
73 }
74 word.push(c);
75 }
76 }
77 '"' => {
78 started = true;
79 while let Some(c) = chars.next() {
80 match c {
81 '"' => break,
82 '\\' if matches!(chars.peek(), Some('"' | '\\' | '$' | '`')) => word.push(chars.next().unwrap_or('\\')),
83 c => word.push(c),
84 }
85 }
86 }
87 '\\' => {
88 started = true;
89 if let Some(next) = chars.next()
90 && next != '\n'
91 {
92 word.push(next);
93 }
94 }
95 c if c.is_whitespace() => {
96 if started {
97 words.push(std::mem::take(&mut word));
98 started = false;
99 }
100 }
101 c => {
102 started = true;
103 word.push(c);
104 }
105 }
106 }
107 if started {
108 words.push(word);
109 }
110 words
111}
112
113fn texts(value: Option<&Value>) -> Vec<String> {
114 match value {
115 Some(Value::Array(items)) => items.iter().map(expr::to_text).filter(|s| !s.is_empty()).collect(),
116 Some(Value::Null) | None => Vec::new(),
117 Some(other) => vec![expr::to_text(other)].into_iter().filter(|s| !s.is_empty()).collect(),
118 }
119}
120
121/// Reads a `services:` entry or `container:`, its expressions already
122/// read: a string (the image) or a mapping.
123pub(crate) fn container_spec(value: &Value) -> ContainerSpec {
124 match value {
125 Value::String(image) => ContainerSpec { image: image.trim().to_owned(), ..ContainerSpec::default() },
126 Value::Object(fields) => ContainerSpec {
127 image: fields.get("image").map(expr::to_text).unwrap_or_default().trim().to_owned(),
128 env: fields
129 .get("env")
130 .and_then(Value::as_object)
131 .map(|env| env.iter().map(|(k, v)| (k.clone(), expr::to_text(v))).collect())
132 .unwrap_or_default(),
133 ports: texts(fields.get("ports")),
134 volumes: texts(fields.get("volumes")),
135 options: fields.get("options").map(|o| split_words(&expr::to_text(o))).unwrap_or_default(),
136 credentials: fields.get("credentials").and_then(Value::as_object).and_then(|c| {
137 let username = c.get("username").map(expr::to_text).unwrap_or_default();
138 let password = c.get("password").map(expr::to_text).unwrap_or_default();
139 (!username.is_empty() || !password.is_empty()).then_some((username, password))
140 }),
141 command: fields.get("command").map(|c| split_words(&expr::to_text(c))).unwrap_or_default(),
142 entrypoint: fields.get("entrypoint").map(expr::to_text).filter(|e| !e.is_empty()),
143 },
144 _ => ContainerSpec::default(),
145 }
146}
147
148/// `job.services.<id>.ports`: each container port, and the host port it
149/// is reached on. On g1t's machines a port left for Docker to choose is
150/// the container's own.
151pub(crate) fn port_map(ports: &[String]) -> BTreeMap<String, String> {
152 let mut map = BTreeMap::new();
153 for port in ports {
154 let without_protocol = port.split('/').next().unwrap_or(port);
155 let parts: Vec<&str> = without_protocol.split(':').collect();
156 let (host, container) = match parts.as_slice() {
157 [container] => (*container, *container),
158 [host, container] => (if host.is_empty() { *container } else { *host }, *container),
159 [_, host, container] => (if host.is_empty() { *container } else { *host }, *container),
160 _ => continue,
161 };
162 if !container.is_empty() {
163 map.insert(container.to_owned(), host.to_owned());
164 }
165 }
166 map
167}
168
169/// The registry an image is pulled from, for signing in with
170/// `credentials:`.
171pub(crate) fn registry_of(image: &str) -> String {
172 match image.split_once('/') {
173 Some((first, _)) if first.contains('.') || first.contains(':') || first == "localhost" => first.to_owned(),
174 _ => "https://index.docker.io/v1/".to_owned(),
175 }
176}
177
178/// A name for Docker from any text: lower case, letters, digits, `_`,
179/// `.` and `-`.
180pub(crate) fn docker_name(text: &str) -> String {
181 let name: String = text
182 .to_ascii_lowercase()
183 .chars()
184 .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') { c } else { '_' })
185 .collect();
186 name.trim_matches(['_', '.', '-']).chars().take(100).collect()
187}
188
189/// The `docker create` arguments of a container. `env` is passed by name
190/// (`-e NAME`), its values in the command's environment, so secrets never
191/// stand on a command line. `keep_alive`: a job container, which waits
192/// while steps run in it.
193pub(crate) fn create_args(name: &str, network: Option<&str>, alias: Option<&str>, spec: &ContainerSpec, mounts: &[(String, String)], keep_alive: bool) -> Vec<String> {
194 let mut args: Vec<String> = vec!["create".into(), "--name".into(), name.into(), "--label".into(), "g1t-job".into()];
195 if let Some(network) = network {
196 args.extend(["--network".into(), network.into()]);
197 if let Some(alias) = alias {
198 args.extend(["--network-alias".into(), alias.into()]);
199 }
200 }
201 for port in &spec.ports {
202 args.extend(["-p".into(), port.clone()]);
203 }
204 for volume in &spec.volumes {
205 args.extend(["-v".into(), volume.clone()]);
206 }
207 for (from, to) in mounts {
208 args.extend(["-v".into(), format!("{from}:{to}")]);
209 }
210 for name in spec.env.keys() {
211 args.extend(["-e".into(), name.clone()]);
212 }
213 args.extend(spec.options.iter().cloned());
214 if keep_alive {
215 args.extend(["--entrypoint".into(), "tail".into(), spec.image.clone(), "-f".into(), "/dev/null".into()]);
216 } else {
217 if let Some(entrypoint) = &spec.entrypoint {
218 args.extend(["--entrypoint".into(), entrypoint.clone()]);
219 }
220 args.push(spec.image.clone());
221 args.extend(spec.command.iter().cloned());
222 }
223 args
224}
225
226/// A Docker action's run, or a `docker://` step's.
227#[derive(Clone, Debug)]
228pub(crate) struct DockerRun {
229 pub(crate) image: String,
230 pub(crate) entrypoint: Option<String>,
231 pub(crate) args: Vec<String>,
232 /// The step's variables, `INPUT_*` included.
233 pub(crate) env: BTreeMap<String, String>,
234}
235
236/// What a command line shows: secrets are passed by name, so nothing
237/// needs hiding, but long lists are kept readable.
238fn shown(args: &[String]) -> String {
239 args.iter().map(|a| if a.contains(' ') || a.is_empty() { format!("'{a}'") } else { a.clone() }).collect::<Vec<_>>().join(" ")
240}
241
242impl Job {
243 /// Runs `docker` with `args`, its output in the log. `env`: the
244 /// variables it passes to a container by name.
245 pub(crate) fn docker(&mut self, args: &[String], env: &BTreeMap<String, String>, timeout: Duration) -> bool {
246 self.log.line(&format!("[command]docker {}", shown(args)));
247 let mut command = Command::new("docker");
248 command.args(args).env_clear().envs(self.docker_cli_env()).envs(env);
249 let mut commands = Commands::default();
250 matches!(process::run(command, timeout.min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
251 }
252
253 /// What `docker` prints, or None if it fails.
254 pub(crate) fn docker_output(&self, args: &[&str]) -> Option<String> {
255 let output = Command::new("docker").args(args).env_clear().envs(self.docker_cli_env()).output().ok()?;
256 output.status.success().then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
257 }
258
259 /// What the `docker` command itself needs of the sandbox's variables:
260 /// where it is, where its config is, and how to reach the Engine.
261 fn docker_cli_env(&self) -> BTreeMap<String, String> {
262 ["PATH", "HOME", "DOCKER_HOST", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY"]
263 .iter()
264 .filter_map(|name| self.base_env_value(name).map(|value| (name.to_string(), value)))
265 .collect()
266 }
267
268 /// Makes sure Docker answers before a job's containers start: the
269 /// job's own Engine, on g1t's machines.
270 fn docker_ready(&mut self) -> bool {
271 if self.docker_hosted {
272 let started = crate::docker::engine::ensure_started();
273 self.log_docker_notes();
274 // Why it could not start is among the notes just logged.
275 return started.is_ok();
276 }
277 if self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_some() {
278 return true;
279 }
Merge branch 'claude/cool-newton-7nm8r9' (#36): real GitHub Actions on g1t, managed deploy workflows, g1t/deploy and the Sysbox runner 0.2.0. The merge keeps both sides' webhook events and API operations, says agent images need tar, guards the job engine's stop to Unix so the runner builds on Windows again, and keeps the moves tests' previews from going idle as the calendar passes their fixture's date.280 self.log.line("##[error]This job needs Docker (`services`, `container` or a Docker action), and Docker does not answer here. On a self-hosted runner, run the runner's image under Sysbox, which gives each job that uses Docker an engine of its own.");
Merge branch 'main' into actions-toolkit-oidc-artifacts281 false
282 }
283
284 pub(crate) fn log_docker_notes(&mut self) {
285 for line in crate::docker::take_notes() {
286 self.log.line(&line);
287 }
288 }
289
290 /// Pulls an image, signed in with `credentials` if it has them.
291 fn pull(&mut self, image: &str, credentials: Option<&(String, String)>) -> bool {
292 let Some((username, password)) = credentials else {
293 return self.docker(&["pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800));
294 };
295 // A config of its own, so the credentials go no further than this pull.
296 let config = self.temp.join(format!("docker-config-{:x}", super::rand_id()));
297 let _ = std::fs::create_dir_all(&config);
298 let registry = registry_of(image);
299 let config_text = config.display().to_string();
300 self.log.line(&format!("[command]docker --config {config_text} login {registry} --username *** --password-stdin"));
301 let login = Command::new("docker")
302 .args(["--config", &config_text, "login", &registry, "--username", username, "--password-stdin"])
303 .env_clear()
304 .envs(self.docker_cli_env())
305 .stdin(std::process::Stdio::piped())
306 .stdout(std::process::Stdio::piped())
307 .stderr(std::process::Stdio::piped())
308 .spawn()
309 .and_then(|mut child| {
310 use std::io::Write;
311 if let Some(mut stdin) = child.stdin.take() {
312 let _ = stdin.write_all(password.as_bytes());
313 }
314 child.wait_with_output()
315 });
316 let ok = match login {
317 Ok(output) if output.status.success() => {
318 self.docker(&["--config".into(), config_text.clone(), "pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800))
319 }
320 Ok(output) => {
321 self.log.line(&format!("##[error]Could not sign in to {registry}: {}", String::from_utf8_lossy(&output.stderr).trim()));
322 false
323 }
324 Err(error) => {
325 self.log.line(&format!("##[error]Could not run docker login: {error}"));
326 false
327 }
328 };
329 let _ = std::fs::remove_dir_all(&config);
330 ok
331 }
332
333 /// Reads a `services:` entry or `container:` with the job's contexts.
334 fn read_container(&self, value: &Value) -> ContainerSpec {
335 let frame = super::Frame::default();
336 let env = self.env_context(&frame);
337 let contexts = self.contexts_for(&frame, &env);
338 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
339 container_spec(&value)
340 }
341
342 /// Starts the job's services and its container, before its steps:
343 /// GitHub's "Initialize containers". Returns whether they all started.
344 pub(crate) fn start_containers(&mut self) -> bool {
345 let services: Vec<(String, Value)> = self.spec["spec"]["services"].as_object().map(|s| s.iter().map(|(k, v)| (k.clone(), v.clone())).collect()).unwrap_or_default();
346 let container = self.spec["spec"].get("container").filter(|c| !c.is_null()).cloned();
347 let container = container.map(|c| self.read_container(&c)).filter(|c| !c.image.is_empty());
348 let services: Vec<(String, ContainerSpec)> = services.into_iter().map(|(name, value)| (name, self.read_container(&value))).filter(|(_, spec)| !spec.image.is_empty()).collect();
349 // A self-hosted runner in Docker mode started this job in its
350 // `container:` image already (selfhosted/exec.rs).
351 let container = container.filter(|_| std::env::var_os(IN_JOB_CONTAINER).is_none());
352 if services.is_empty() && container.is_none() {
353 return true;
354 }
355 // A self-hosted machine without Docker runs the steps as before,
356 // on the machine, without the containers.
357 if !self.docker_hosted && self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_none() {
Merge branch 'claude/cool-newton-7nm8r9' (#36): real GitHub Actions on g1t, managed deploy workflows, g1t/deploy and the Sysbox runner 0.2.0. The merge keeps both sides' webhook events and API operations, says agent images need tar, guards the job engine's stop to Unix so the runner builds on Windows again, and keeps the moves tests' previews from going idle as the calendar passes their fixture's date.358 self.log.line("##[warning]Docker does not answer on this runner, so the job's `services` and `container` are not started and its steps run on the machine. Run the runner's image under Sysbox to give each job Docker of its own.");
Merge branch 'main' into actions-toolkit-oidc-artifacts359 return true;
360 }
361 self.log.line("##[group]Initialize containers");
362 let ok = self.docker_ready() && self.start_containers_now(services, container);
363 self.log.line("##[endgroup]");
364 ok
365 }
366
367 fn start_containers_now(&mut self, services: Vec<(String, ContainerSpec)>, container: Option<ContainerSpec>) -> bool {
368 let job_id = docker_name(&format!("{}_{:x}", self.spec["name"].as_str().unwrap_or("job"), super::rand_id() & 0xffff_ffff));
369 let network = format!("g1t_{job_id}");
370 if !self.docker(&["network".into(), "create".into(), "--label".into(), "g1t-job".into(), network.clone()], &BTreeMap::new(), Duration::from_secs(60)) {
371 self.log.line("##[error]Could not make the job's network.");
372 return false;
373 }
374 self.network = Some(network.clone());
375
376 let mut services_context = Map::new();
377 for (service, spec) in &services {
378 if !self.pull(&spec.image, spec.credentials.as_ref()) {
379 self.log.line(&format!("##[error]Could not pull {} for the service `{service}`.", spec.image));
380 return false;
381 }
382 let name = docker_name(&format!("{service}_{job_id}"));
383 let args = create_args(&name, Some(&network), Some(service), spec, &[], false);
384 if !self.docker(&args, &spec.env, Duration::from_secs(300)) {
385 self.log.line(&format!("##[error]Could not create the service `{service}`."));
386 return false;
387 }
388 self.services.push((service.clone(), name.clone()));
389 if !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
390 self.log.line(&format!("##[error]Could not start the service `{service}`."));
391 return false;
392 }
393 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
394 let ports: Map<String, Value> = port_map(&spec.ports).into_iter().map(|(k, v)| (k, json!(v))).collect();
395 services_context.insert(service.clone(), json!({ "id": id, "network": network, "ports": ports }));
396 }
397
398 if let Some(spec) = container {
399 if !self.pull(&spec.image, spec.credentials.as_ref()) {
400 self.log.line(&format!("##[error]Could not pull {} for the job's container.", spec.image));
401 return false;
402 }
403 let name = docker_name(&format!("job_{job_id}"));
404 let mounts = self.container_mounts();
405 let mut spec = spec;
406 spec.env.insert("HOME".into(), GITHUB_HOME.into());
407 spec.env.insert("CI".into(), "true".into());
408 spec.env.insert("GITHUB_ACTIONS".into(), "true".into());
409 let mut args = create_args(&name, Some(&network), None, &spec, &mounts, true);
410 // Steps start in the workspace.
411 args.splice(1..1, ["-w".to_owned(), self.workspace.display().to_string()]);
412 if !self.docker(&args, &spec.env, Duration::from_secs(300)) || !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
413 self.log.line("##[error]Could not start the job's container.");
414 return false;
415 }
416 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
417 let has_bash = self.docker_output(&["exec", &name, "sh", "-c", "command -v bash"]).is_some_and(|out| !out.is_empty());
418 let node = self.docker_output(&["exec", &name, CONTAINER_NODE, "--version"]).is_some();
419 let path = self.docker_output(&["exec", &name, "sh", "-c", "printf %s \"$PATH\""]).unwrap_or_else(|| "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into());
420 if !node {
421 self.log.line("##[warning]The runner's Node does not run in this image (it needs glibc and libstdc++), so JavaScript actions run beside the container, on g1t's image, with the same files.");
422 }
423 self.job_context.insert("container".into(), json!({ "id": id, "network": network }));
Merge branch 'claude/cool-newton-7nm8r9' (#36): real GitHub Actions on g1t, managed deploy workflows, g1t/deploy and the Sysbox runner 0.2.0. The merge keeps both sides' webhook events and API operations, says agent images need tar, guards the job engine's stop to Unix so the runner builds on Windows again, and keeps the moves tests' previews from going idle as the calendar passes their fixture's date.424 let has_python = self.docker_output(&["exec", &name, "sh", "-c", "command -v python"]).is_some_and(|out| !out.is_empty());
425 self.container = Some(JobContainer {
426 id: name,
427 shell: if has_bash { "bash" } else { "sh" },
428 python: if has_python { "python" } else { "python3" },
429 node,
430 path,
431 });
Merge branch 'main' into actions-toolkit-oidc-artifacts432 }
433
434 // Services with a health check are waited for.
435 for (service, name) in self.services.clone() {
436 if !self.wait_healthy(&service, &name) {
437 return false;
438 }
439 }
440 if !services_context.is_empty() {
441 self.job_context.insert("services".into(), Value::Object(services_context));
442 }
443 self.log_docker_notes();
444 true
445 }
446
447 /// GitHub's runner's folders, at the same paths, and Docker's socket.
448 fn container_mounts(&self) -> Vec<(String, String)> {
449 let home = super::paths::under_home(super::paths::HOME_RUNNER);
450 let github_home = self.temp.join("_github_home");
451 let _ = std::fs::create_dir_all(&github_home);
452 let mut mounts: Vec<(String, String)> = ["work", "_temp", "_actions", "_tool"]
453 .iter()
454 .map(|dir| {
455 let path = home.join(dir);
456 let _ = std::fs::create_dir_all(&path);
457 (path.display().to_string(), path.display().to_string())
458 })
459 .collect();
460 mounts.push((github_home.display().to_string(), GITHUB_HOME.into()));
461 if let Some(node) = which_node() {
462 mounts.push((node.display().to_string(), format!("{CONTAINER_NODE}:ro")));
463 }
464 mounts.push((crate::docker::engine::DOCKER_SOCKET.into(), crate::docker::engine::DOCKER_SOCKET.into()));
465 mounts
466 }
467
468 fn wait_healthy(&mut self, service: &str, name: &str) -> bool {
469 let limit = Duration::from_secs(600).min(self.remaining_time());
470 let until = Instant::now() + limit;
471 let mut wait = Duration::from_secs(1);
472 loop {
473 let status = self.docker_output(&["inspect", "--format", "{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}", name]).unwrap_or_default();
474 match status.as_str() {
475 "" => return true,
476 "healthy" => {
477 self.log.line(&format!("{service} is healthy."));
478 return true;
479 }
480 "unhealthy" => {
481 self.log.line(&format!("##[error]The service `{service}` is unhealthy."));
482 self.service_logs(service, name);
483 return false;
484 }
485 _ => {}
486 }
487 if Instant::now() >= until {
488 self.log.line(&format!("##[error]The service `{service}` did not become healthy in {} s.", limit.as_secs()));
489 self.service_logs(service, name);
490 return false;
491 }
492 self.log.line(&format!("Waiting for {service} to be healthy ({status})."));
493 std::thread::sleep(wait);
494 wait = (wait * 2).min(Duration::from_secs(8));
495 }
496 }
497
498 fn service_logs(&mut self, service: &str, name: &str) {
499 self.log.line(&format!("##[group]Service container {service}"));
500 self.docker(&["logs".into(), "--tail".into(), "200".into(), name.into()], &BTreeMap::new(), Duration::from_secs(60));
501 self.log.line("##[endgroup]");
502 }
503
504 /// Whether the job has containers to stop when it ends.
505 pub(crate) fn has_containers(&self) -> bool {
506 self.network.is_some()
507 }
508
509 /// GitHub's "Stop containers": each service's log, then the job's
510 /// containers and network removed.
511 pub(crate) fn stop_containers(&mut self) -> bool {
512 for (service, name) in self.services.clone() {
513 self.service_logs(&service, &name);
514 }
515 let mut names: Vec<String> = self.services.iter().map(|(_, name)| name.clone()).collect();
516 if let Some(container) = &self.container {
517 names.push(container.id.clone());
518 }
519 if !names.is_empty() {
520 let mut args = vec!["rm".to_owned(), "--force".to_owned()];
521 args.extend(names);
522 self.docker(&args, &BTreeMap::new(), Duration::from_secs(120));
523 }
524 if let Some(network) = self.network.take() {
525 self.docker(&["network".into(), "rm".into(), network], &BTreeMap::new(), Duration::from_secs(60));
526 }
527 self.container = None;
528 true
529 }
530
531 /// The variables a process inside a container is given: the step's,
532 /// GitHub's and the job's, but not the sandbox's own (its `PATH`,
533 /// `HOME` and the like, which mean nothing in another image).
534 pub(crate) fn container_env(&self, step_env: &BTreeMap<String, String>, full: BTreeMap<String, String>, image_path: &str) -> BTreeMap<String, String> {
535 let mut out: BTreeMap<String, String> = full
536 .into_iter()
537 .filter(|(name, _)| step_env.contains_key(name) || !self.host_env.contains(name) || name.starts_with("GITHUB_") || name.starts_with("RUNNER_"))
538 .collect();
539 if !step_env.contains_key("PATH") {
540 out.remove("PATH");
541 if !self.path_prepend_entries().is_empty() {
542 out.insert("PATH".into(), format!("{}:{image_path}", self.path_prepend_entries().join(":")));
543 }
544 }
545 if !step_env.contains_key("HOME") {
546 out.remove("HOME");
547 }
548 out
549 }
550
551 /// A step's command, run inside the job's container instead.
552 pub(crate) fn in_container(&self, program: &str, args: &[String], dir: &Path, env: BTreeMap<String, String>) -> Option<Command> {
553 let container = self.container.as_ref()?;
554 let mut command = Command::new("docker");
555 command.args(["exec", "-w", &dir.display().to_string()]);
556 for name in env.keys() {
557 command.args(["-e", name]);
558 }
559 command.arg(&container.id).arg(program).args(args);
560 command.env_clear().envs(self.docker_cli_env()).envs(env);
561 Some(command)
562 }
563
564 /// Runs a container for a Docker action or a `docker://` step, as
565 /// GitHub's runner does: the workspace at /github/workspace, the step's
566 /// files at /github/file_commands, the job's network.
567 pub(crate) fn run_docker(&mut self, run: &DockerRun) -> (bool, BTreeMap<String, String>, BTreeMap<String, String>) {
568 let fail = (false, BTreeMap::new(), BTreeMap::new());
569 if !self.docker_ready() {
570 return fail;
571 }
572 let id = format!("{:x}", super::rand_id());
573 let Ok(files) = StepFiles::new(&self.temp, &id) else { return fail };
574 let commands_dir = self.temp.join("_runner_file_commands");
575 let workflow_dir = self.temp.join("_github_workflow");
576 let home_dir = self.temp.join("_github_home");
577 for dir in [&workflow_dir, &home_dir] {
578 let _ = std::fs::create_dir_all(dir);
579 }
580 let _ = std::fs::copy(self.temp.join("event.json"), workflow_dir.join("event.json"));
581
582 let full = self.process_env(&run.env, &files);
583 let mut env = self.container_env(&run.env, full, "");
584 env.remove("PATH");
585 // Paths as the container sees them.
586 let moved = |path: &Path| format!("{GITHUB_FILE_COMMANDS}/{}", path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default());
587 env.insert("GITHUB_OUTPUT".into(), moved(&files.output));
588 env.insert("GITHUB_ENV".into(), moved(&files.env));
589 env.insert("GITHUB_PATH".into(), moved(&files.path));
590 env.insert("GITHUB_STATE".into(), moved(&files.state));
591 env.insert("GITHUB_STEP_SUMMARY".into(), moved(&files.summary));
592 env.insert("GITHUB_WORKSPACE".into(), GITHUB_WORKSPACE.into());
593 env.insert("GITHUB_EVENT_PATH".into(), format!("{GITHUB_WORKFLOW}/event.json"));
594 env.insert("HOME".into(), GITHUB_HOME.into());
595 env.remove("GITHUB_ACTION_PATH");
596
597 let mut args: Vec<String> = vec!["run".into(), "--rm".into(), "--label".into(), "g1t-job".into(), "--workdir".into(), GITHUB_WORKSPACE.into()];
598 if let Some(network) = &self.network {
599 args.extend(["--network".into(), network.clone()]);
600 }
601 for (from, to) in [
602 (self.workspace.clone(), GITHUB_WORKSPACE),
603 (home_dir, GITHUB_HOME),
604 (workflow_dir, GITHUB_WORKFLOW),
605 (commands_dir, GITHUB_FILE_COMMANDS),
606 (PathBuf::from(crate::docker::engine::DOCKER_SOCKET), crate::docker::engine::DOCKER_SOCKET),
607 ] {
608 args.extend(["-v".into(), format!("{}:{to}", from.display())]);
609 }
610 for name in env.keys() {
611 args.extend(["-e".into(), name.clone()]);
612 }
613 if let Some(entrypoint) = &run.entrypoint {
614 args.extend(["--entrypoint".into(), entrypoint.clone()]);
615 }
616 args.push(run.image.clone());
617 args.extend(run.args.iter().cloned());
618
619 crate::abuse::touch();
620 if let Some(miner) = crate::abuse::miner_in(&shown(&args)) {
621 self.log.line(&format!("##[error]g1t does not run cryptocurrency miners ({miner}). This step was not run."));
622 return fail;
623 }
624 self.log.line(&format!("[command]docker {}", shown(&args)));
625 let mut command = Command::new("docker");
626 command.args(&args).env_clear().envs(self.docker_cli_env()).envs(&env);
627 let mut commands = Commands::default();
628 let ended = process::run(command, self.remaining_time(), &mut self.log, &mut commands);
629 self.log_docker_notes();
630 let ok = match ended {
631 Ok(Ended::Exited(0)) => true,
632 Ok(Ended::Exited(code)) => {
633 self.log.line(&format!("##[error]Docker run failed with exit code {code}."));
634 false
635 }
636 Ok(Ended::TimedOut) => {
637 self.log.line("##[error]The step ran past its time limit and was stopped.");
638 false
639 }
Merge Actions runs: summaries, attempts and re-runs, graceful cancel, log downloads, badges (actions 0009)640 Ok(Ended::Cancelled) => false,
Merge branch 'main' into actions-toolkit-oidc-artifacts641 Err(error) => {
642 self.log.line(&format!("##[error]docker could not be started: {error}"));
643 false
644 }
645 };
646 let (outputs, state) = self.absorb(&files, &commands);
647 (ok, outputs, state)
648 }
649
650 /// Builds a Docker action's image from its Dockerfile, once per job.
651 pub(crate) fn build_action_image(&mut self, dir: &Path, dockerfile: &str, tag_of: &str) -> Option<String> {
652 let tag = format!("g1t-action/{}", docker_name(tag_of));
653 if self.built_actions.contains(&tag) {
654 return Some(tag);
655 }
656 if !self.docker_ready() {
657 return None;
658 }
659 let file = dir.join(dockerfile);
660 let args = vec!["build".into(), "-t".into(), tag.clone(), "-f".into(), file.display().to_string(), dir.display().to_string()];
661 if !self.docker(&args, &BTreeMap::new(), Duration::from_secs(1800)) {
662 self.log.line("##[error]The action's image did not build.");
663 return None;
664 }
665 self.built_actions.insert(tag.clone());
666 Some(tag)
667 }
668
669 /// `docker/setup-buildx-action` on g1t's machines: the job's own
670 /// Engine is the builder (BuildKit, the `docker` driver, with the
671 /// containerd image store, so cache export and attestations work).
672 pub(crate) fn setup_buildx(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
673 if !self.docker_ready() {
674 return (false, BTreeMap::new());
675 }
676 if let Some(driver) = with.get("driver").filter(|d| !d.is_empty() && *d != "docker") {
677 self.log.line(&format!("`driver: {driver}` is not used on g1t's machines: builds run on this job's own Docker Engine (BuildKit, the `docker` driver), which keeps the sandbox's network and guardrails."));
678 }
679 for input in ["driver-opts", "buildkitd-flags", "buildkitd-config", "buildkitd-config-inline", "endpoint"] {
680 if with.get(input).is_some_and(|v| !v.trim().is_empty()) {
681 self.log.line(&format!("`{input}` is not used on g1t's machines."));
682 }
683 }
684 self.docker(&["buildx".into(), "version".into()], &BTreeMap::new(), Duration::from_secs(60));
685 let inspect = self.docker_output(&["buildx", "inspect", "default"]).unwrap_or_default();
686 let platforms = inspect
687 .lines()
688 .find_map(|line| line.trim().strip_prefix("Platforms:"))
689 .map(|p| p.split(',').map(|s| s.trim().trim_end_matches('*').to_owned()).filter(|s| !s.is_empty()).collect::<Vec<_>>().join(","))
690 .unwrap_or_else(|| "linux/amd64".into());
691 self.log.line(&format!("Builder: default (this job's Docker Engine), platforms {platforms}"));
692 let mut outputs = BTreeMap::new();
693 outputs.insert("name".into(), "default".into());
694 outputs.insert("driver".into(), "docker".into());
695 outputs.insert("platforms".into(), platforms.clone());
696 outputs.insert("endpoint".into(), "default".into());
697 outputs.insert("status".into(), "running".into());
698 outputs.insert("flags".into(), String::new());
699 outputs.insert(
700 "nodes".into(),
701 json!([{ "name": "default", "endpoint": "default", "status": "running", "platforms": platforms }]).to_string(),
702 );
703 (true, outputs)
704 }
705}
706
707/// The runner's Node, its real file (not a link), to mount into a job
708/// container.
709fn which_node() -> Option<PathBuf> {
710 let path = std::env::var_os("PATH")?;
711 std::env::split_paths(&path).map(|dir| dir.join("node")).find(|p| p.is_file()).and_then(|p| std::fs::canonicalize(p).ok())
712}
713
714/// The job context's `container` and `services`, merged with its status.
715pub(crate) fn job_context(status: &str, extra: &Map<String, Value>) -> Value {
716 let mut job = Map::new();
717 job.insert("status".into(), json!(status));
718 for (key, value) in extra {
719 job.insert(key.clone(), value.clone());
720 }
721 Value::Object(job)
722}
723
724/// Names a set of tags already built in this job.
725pub(crate) type Built = BTreeSet<String>;
726
727#[cfg(test)]
728mod tests {
729 use super::*;
730
731 #[test]
732 fn options_split_as_a_shell_would() {
733 assert_eq!(
734 split_words(r#"--health-cmd "pg_isready -U postgres" --health-interval 10s --health-retries=5"#),
735 vec!["--health-cmd", "pg_isready -U postgres", "--health-interval", "10s", "--health-retries=5"]
736 );
737 assert_eq!(split_words("--health-cmd 'redis-cli ping' --tmpfs /var/lib/x"), vec!["--health-cmd", "redis-cli ping", "--tmpfs", "/var/lib/x"]);
738 assert_eq!(split_words(r#"a\ b "c\"d" ''"#), vec!["a b", "c\"d", ""]);
739 assert!(split_words(" ").is_empty());
740 }
741
742 #[test]
743 fn services_are_read_from_either_form() {
744 assert_eq!(container_spec(&json!("redis:7")).image, "redis:7");
745 let spec = container_spec(&json!({
746 "image": "postgres:17",
747 "env": { "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "app" },
748 "ports": ["5432:5432", 6543],
749 "volumes": ["/data:/var/lib/postgresql/data"],
750 "options": "--health-cmd pg_isready --health-interval 10s",
751 "credentials": { "username": "me", "password": "token" },
752 }));
753 assert_eq!(spec.image, "postgres:17");
754 assert_eq!(spec.env["POSTGRES_DB"], "app");
755 assert_eq!(spec.ports, vec!["5432:5432", "6543"]);
756 assert_eq!(spec.options, vec!["--health-cmd", "pg_isready", "--health-interval", "10s"]);
757 assert_eq!(spec.credentials, Some(("me".into(), "token".into())));
758 // An empty image is a service the job leaves out, as on GitHub.
759 assert_eq!(container_spec(&json!({ "image": "" })).image, "");
760 }
761
762 #[test]
763 fn ports_map_container_to_host() {
764 let map = port_map(&["5432:5432".into(), "6543:5432/tcp".into(), "6379".into(), "127.0.0.1:8080:80".into(), ":9000".into()]);
765 assert_eq!(map["5432"], "6543");
766 assert_eq!(map["6379"], "6379");
767 assert_eq!(map["80"], "8080");
768 assert_eq!(map["9000"], "9000");
769 }
770
771 #[test]
772 fn credentials_sign_in_to_the_images_registry() {
773 assert_eq!(registry_of("ghcr.io/acme/db:1"), "ghcr.io");
774 assert_eq!(registry_of("g1t.sh/acme/web"), "g1t.sh");
775 assert_eq!(registry_of("localhost:5000/x"), "localhost:5000");
776 assert_eq!(registry_of("acme/private"), "https://index.docker.io/v1/");
777 assert_eq!(registry_of("postgres"), "https://index.docker.io/v1/");
778 }
779
780 #[test]
781 fn a_service_is_created_with_its_values_passed_by_name() {
782 let spec = container_spec(&json!({
783 "image": "postgres:17",
784 "env": { "POSTGRES_PASSWORD": "secret" },
785 "ports": ["5432:5432"],
786 "options": "--health-cmd pg_isready",
787 }));
788 let args = create_args("postgres_job", Some("g1t_job"), Some("postgres"), &spec, &[], false);
789 assert_eq!(
790 args,
791 vec![
792 "create", "--name", "postgres_job", "--label", "g1t-job", "--network", "g1t_job", "--network-alias", "postgres", "-p", "5432:5432", "-e",
793 "POSTGRES_PASSWORD", "--health-cmd", "pg_isready", "postgres:17"
794 ]
795 );
796 assert!(!args.iter().any(|a| a.contains("secret")));
797 let job = create_args("job_x", Some("g1t_job"), None, &container_spec(&json!("node:24")), &[("/home/runner/work".into(), "/home/runner/work".into())], true);
798 assert!(job.ends_with(&["--entrypoint".into(), "tail".into(), "node:24".into(), "-f".into(), "/dev/null".into()]));
799 assert!(job.contains(&"/home/runner/work:/home/runner/work".to_owned()));
800 }
801
802 #[test]
803 fn names_are_docker_names() {
804 assert_eq!(docker_name("Build & test_1a2b"), "build___test_1a2b");
805 assert_eq!(docker_name("docker/login-action@v3"), "docker_login-action_v3");
806 }
807
808 #[test]
809 fn the_job_context_carries_containers() {
810 let mut extra = Map::new();
811 extra.insert("services".into(), json!({ "db": { "ports": { "5432": "5432" } } }));
812 let job = job_context("success", &extra);
813 assert_eq!(job["status"], "success");
814 assert_eq!(job["services"]["db"]["ports"]["5432"], "5432");
815 }
816}

This file's history is long; its oldest lines are credited to the oldest commit read.