Skip to content
Sign in

flagon-io/g1t

public
0

g1t/crates/runner/src/actions/containers.rs

806 lines38,258 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'main' into actions-toolkit-oidc-artifacts1//! Containers a job asks for, as GitHub's runner starts them, with the
2//! `docker` command: its `services:` (a database beside the steps), its
3//! `container:` (every step run inside an image), `uses: docker://image`
4//! steps and Docker actions (`runs.using: docker`).
5//!
6//! On g1t's machines the Engine is the job's own (crate::docker). Every
7//! container shares the job's network there, so a service is reached at
8//! `localhost:<port>` as on GitHub's runner, and by its name as from a job
9//! container: the API proxy makes each service's name mean 127.0.0.1.
10
11use std::collections::{BTreeMap, BTreeSet};
12use std::path::{Path, PathBuf};
13use std::process::Command;
14use std::time::{Duration, Instant};
15
16use g1t_actions::expr;
17use serde_json::{Map, Value, json};
18
19use super::files::StepFiles;
20use super::process::{self, Commands, Ended};
21use super::Job;
22
23/// Set in a job a self-hosted runner started inside its `container:` image.
24pub(crate) const IN_JOB_CONTAINER: &str = "G1T_JOB_CONTAINER";
25/// Where a job container finds the runner's Node, for JavaScript actions.
26pub(crate) const CONTAINER_NODE: &str = "/__e/node24/bin/node";
27/// GitHub's folders inside a Docker action's container.
28const GITHUB_WORKSPACE: &str = "/github/workspace";
29const GITHUB_HOME: &str = "/github/home";
30const GITHUB_WORKFLOW: &str = "/github/workflow";
31const GITHUB_FILE_COMMANDS: &str = "/github/file_commands";
32
33/// The job's own container, when it has `container:`.
34pub(crate) struct JobContainer {
35 pub(crate) id: String,
36 /// `bash`, or `sh` in an image without it.
37 pub(crate) shell: &'static str,
38 /// Whether the runner's Node runs in it.
39 pub(crate) node: bool,
40 /// The image's own `PATH`.
41 pub(crate) path: String,
42}
43
44/// A `services:` entry or `container:`, read.
45#[derive(Debug, Default, PartialEq)]
46pub(crate) struct ContainerSpec {
47 pub(crate) image: String,
48 pub(crate) env: BTreeMap<String, String>,
49 pub(crate) ports: Vec<String>,
50 pub(crate) volumes: Vec<String>,
51 pub(crate) options: Vec<String>,
52 pub(crate) credentials: Option<(String, String)>,
53 pub(crate) command: Vec<String>,
54 pub(crate) entrypoint: Option<String>,
55}
56
57/// Splits a command line as a shell would: words, with `'…'`, `"…"` and
58/// `\` quoting. No variables are expanded.
59pub(crate) fn split_words(text: &str) -> Vec<String> {
60 let mut words = Vec::new();
61 let mut word = String::new();
62 let mut started = false;
63 let mut chars = text.chars().peekable();
64 while let Some(c) = chars.next() {
65 match c {
66 '\'' => {
67 started = true;
68 for c in chars.by_ref() {
69 if c == '\'' {
70 break;
71 }
72 word.push(c);
73 }
74 }
75 '"' => {
76 started = true;
77 while let Some(c) = chars.next() {
78 match c {
79 '"' => break,
80 '\\' if matches!(chars.peek(), Some('"' | '\\' | '$' | '`')) => word.push(chars.next().unwrap_or('\\')),
81 c => word.push(c),
82 }
83 }
84 }
85 '\\' => {
86 started = true;
87 if let Some(next) = chars.next()
88 && next != '\n'
89 {
90 word.push(next);
91 }
92 }
93 c if c.is_whitespace() => {
94 if started {
95 words.push(std::mem::take(&mut word));
96 started = false;
97 }
98 }
99 c => {
100 started = true;
101 word.push(c);
102 }
103 }
104 }
105 if started {
106 words.push(word);
107 }
108 words
109}
110
111fn texts(value: Option<&Value>) -> Vec<String> {
112 match value {
113 Some(Value::Array(items)) => items.iter().map(expr::to_text).filter(|s| !s.is_empty()).collect(),
114 Some(Value::Null) | None => Vec::new(),
115 Some(other) => vec![expr::to_text(other)].into_iter().filter(|s| !s.is_empty()).collect(),
116 }
117}
118
119/// Reads a `services:` entry or `container:`, its expressions already
120/// read: a string (the image) or a mapping.
121pub(crate) fn container_spec(value: &Value) -> ContainerSpec {
122 match value {
123 Value::String(image) => ContainerSpec { image: image.trim().to_owned(), ..ContainerSpec::default() },
124 Value::Object(fields) => ContainerSpec {
125 image: fields.get("image").map(expr::to_text).unwrap_or_default().trim().to_owned(),
126 env: fields
127 .get("env")
128 .and_then(Value::as_object)
129 .map(|env| env.iter().map(|(k, v)| (k.clone(), expr::to_text(v))).collect())
130 .unwrap_or_default(),
131 ports: texts(fields.get("ports")),
132 volumes: texts(fields.get("volumes")),
133 options: fields.get("options").map(|o| split_words(&expr::to_text(o))).unwrap_or_default(),
134 credentials: fields.get("credentials").and_then(Value::as_object).and_then(|c| {
135 let username = c.get("username").map(expr::to_text).unwrap_or_default();
136 let password = c.get("password").map(expr::to_text).unwrap_or_default();
137 (!username.is_empty() || !password.is_empty()).then_some((username, password))
138 }),
139 command: fields.get("command").map(|c| split_words(&expr::to_text(c))).unwrap_or_default(),
140 entrypoint: fields.get("entrypoint").map(expr::to_text).filter(|e| !e.is_empty()),
141 },
142 _ => ContainerSpec::default(),
143 }
144}
145
146/// `job.services.<id>.ports`: each container port, and the host port it
147/// is reached on. On g1t's machines a port left for Docker to choose is
148/// the container's own.
149pub(crate) fn port_map(ports: &[String]) -> BTreeMap<String, String> {
150 let mut map = BTreeMap::new();
151 for port in ports {
152 let without_protocol = port.split('/').next().unwrap_or(port);
153 let parts: Vec<&str> = without_protocol.split(':').collect();
154 let (host, container) = match parts.as_slice() {
155 [container] => (*container, *container),
156 [host, container] => (if host.is_empty() { *container } else { *host }, *container),
157 [_, host, container] => (if host.is_empty() { *container } else { *host }, *container),
158 _ => continue,
159 };
160 if !container.is_empty() {
161 map.insert(container.to_owned(), host.to_owned());
162 }
163 }
164 map
165}
166
167/// The registry an image is pulled from, for signing in with
168/// `credentials:`.
169pub(crate) fn registry_of(image: &str) -> String {
170 match image.split_once('/') {
171 Some((first, _)) if first.contains('.') || first.contains(':') || first == "localhost" => first.to_owned(),
172 _ => "https://index.docker.io/v1/".to_owned(),
173 }
174}
175
176/// A name for Docker from any text: lower case, letters, digits, `_`,
177/// `.` and `-`.
178pub(crate) fn docker_name(text: &str) -> String {
179 let name: String = text
180 .to_ascii_lowercase()
181 .chars()
182 .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') { c } else { '_' })
183 .collect();
184 name.trim_matches(['_', '.', '-']).chars().take(100).collect()
185}
186
187/// The `docker create` arguments of a container. `env` is passed by name
188/// (`-e NAME`), its values in the command's environment, so secrets never
189/// stand on a command line. `keep_alive`: a job container, which waits
190/// while steps run in it.
191pub(crate) fn create_args(name: &str, network: Option<&str>, alias: Option<&str>, spec: &ContainerSpec, mounts: &[(String, String)], keep_alive: bool) -> Vec<String> {
192 let mut args: Vec<String> = vec!["create".into(), "--name".into(), name.into(), "--label".into(), "g1t-job".into()];
193 if let Some(network) = network {
194 args.extend(["--network".into(), network.into()]);
195 if let Some(alias) = alias {
196 args.extend(["--network-alias".into(), alias.into()]);
197 }
198 }
199 for port in &spec.ports {
200 args.extend(["-p".into(), port.clone()]);
201 }
202 for volume in &spec.volumes {
203 args.extend(["-v".into(), volume.clone()]);
204 }
205 for (from, to) in mounts {
206 args.extend(["-v".into(), format!("{from}:{to}")]);
207 }
208 for name in spec.env.keys() {
209 args.extend(["-e".into(), name.clone()]);
210 }
211 args.extend(spec.options.iter().cloned());
212 if keep_alive {
213 args.extend(["--entrypoint".into(), "tail".into(), spec.image.clone(), "-f".into(), "/dev/null".into()]);
214 } else {
215 if let Some(entrypoint) = &spec.entrypoint {
216 args.extend(["--entrypoint".into(), entrypoint.clone()]);
217 }
218 args.push(spec.image.clone());
219 args.extend(spec.command.iter().cloned());
220 }
221 args
222}
223
224/// A Docker action's run, or a `docker://` step's.
225#[derive(Clone, Debug)]
226pub(crate) struct DockerRun {
227 pub(crate) image: String,
228 pub(crate) entrypoint: Option<String>,
229 pub(crate) args: Vec<String>,
230 /// The step's variables, `INPUT_*` included.
231 pub(crate) env: BTreeMap<String, String>,
232}
233
234/// What a command line shows: secrets are passed by name, so nothing
235/// needs hiding, but long lists are kept readable.
236fn shown(args: &[String]) -> String {
237 args.iter().map(|a| if a.contains(' ') || a.is_empty() { format!("'{a}'") } else { a.clone() }).collect::<Vec<_>>().join(" ")
238}
239
240impl Job {
241 /// Runs `docker` with `args`, its output in the log. `env`: the
242 /// variables it passes to a container by name.
243 pub(crate) fn docker(&mut self, args: &[String], env: &BTreeMap<String, String>, timeout: Duration) -> bool {
244 self.log.line(&format!("[command]docker {}", shown(args)));
245 let mut command = Command::new("docker");
246 command.args(args).env_clear().envs(self.docker_cli_env()).envs(env);
247 let mut commands = Commands::default();
248 matches!(process::run(command, timeout.min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
249 }
250
251 /// What `docker` prints, or None if it fails.
252 pub(crate) fn docker_output(&self, args: &[&str]) -> Option<String> {
253 let output = Command::new("docker").args(args).env_clear().envs(self.docker_cli_env()).output().ok()?;
254 output.status.success().then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
255 }
256
257 /// What the `docker` command itself needs of the sandbox's variables:
258 /// where it is, where its config is, and how to reach the Engine.
259 fn docker_cli_env(&self) -> BTreeMap<String, String> {
260 ["PATH", "HOME", "DOCKER_HOST", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY"]
261 .iter()
262 .filter_map(|name| self.base_env_value(name).map(|value| (name.to_string(), value)))
263 .collect()
264 }
265
266 /// Makes sure Docker answers before a job's containers start: the
267 /// job's own Engine, on g1t's machines.
268 fn docker_ready(&mut self) -> bool {
269 if self.docker_hosted {
270 let started = crate::docker::engine::ensure_started();
271 self.log_docker_notes();
272 // Why it could not start is among the notes just logged.
273 return started.is_ok();
274 }
275 if self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_some() {
276 return true;
277 }
278 self.log.line("##[error]This job needs Docker (`services`, `container` or a Docker action), and Docker does not answer here. On a self-hosted runner, run the runner directly on a machine with Docker (`--no-docker`).");
279 false
280 }
281
282 pub(crate) fn log_docker_notes(&mut self) {
283 for line in crate::docker::take_notes() {
284 self.log.line(&line);
285 }
286 }
287
288 /// Pulls an image, signed in with `credentials` if it has them.
289 fn pull(&mut self, image: &str, credentials: Option<&(String, String)>) -> bool {
290 let Some((username, password)) = credentials else {
291 return self.docker(&["pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800));
292 };
293 // A config of its own, so the credentials go no further than this pull.
294 let config = self.temp.join(format!("docker-config-{:x}", super::rand_id()));
295 let _ = std::fs::create_dir_all(&config);
296 let registry = registry_of(image);
297 let config_text = config.display().to_string();
298 self.log.line(&format!("[command]docker --config {config_text} login {registry} --username *** --password-stdin"));
299 let login = Command::new("docker")
300 .args(["--config", &config_text, "login", &registry, "--username", username, "--password-stdin"])
301 .env_clear()
302 .envs(self.docker_cli_env())
303 .stdin(std::process::Stdio::piped())
304 .stdout(std::process::Stdio::piped())
305 .stderr(std::process::Stdio::piped())
306 .spawn()
307 .and_then(|mut child| {
308 use std::io::Write;
309 if let Some(mut stdin) = child.stdin.take() {
310 let _ = stdin.write_all(password.as_bytes());
311 }
312 child.wait_with_output()
313 });
314 let ok = match login {
315 Ok(output) if output.status.success() => {
316 self.docker(&["--config".into(), config_text.clone(), "pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800))
317 }
318 Ok(output) => {
319 self.log.line(&format!("##[error]Could not sign in to {registry}: {}", String::from_utf8_lossy(&output.stderr).trim()));
320 false
321 }
322 Err(error) => {
323 self.log.line(&format!("##[error]Could not run docker login: {error}"));
324 false
325 }
326 };
327 let _ = std::fs::remove_dir_all(&config);
328 ok
329 }
330
331 /// Reads a `services:` entry or `container:` with the job's contexts.
332 fn read_container(&self, value: &Value) -> ContainerSpec {
333 let frame = super::Frame::default();
334 let env = self.env_context(&frame);
335 let contexts = self.contexts_for(&frame, &env);
336 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
337 container_spec(&value)
338 }
339
340 /// Starts the job's services and its container, before its steps:
341 /// GitHub's "Initialize containers". Returns whether they all started.
342 pub(crate) fn start_containers(&mut self) -> bool {
343 let services: Vec<(String, Value)> = self.spec["spec"]["services"].as_object().map(|s| s.iter().map(|(k, v)| (k.clone(), v.clone())).collect()).unwrap_or_default();
344 let container = self.spec["spec"].get("container").filter(|c| !c.is_null()).cloned();
345 let container = container.map(|c| self.read_container(&c)).filter(|c| !c.image.is_empty());
346 let services: Vec<(String, ContainerSpec)> = services.into_iter().map(|(name, value)| (name, self.read_container(&value))).filter(|(_, spec)| !spec.image.is_empty()).collect();
347 // A self-hosted runner in Docker mode started this job in its
348 // `container:` image already (selfhosted/exec.rs).
349 let container = container.filter(|_| std::env::var_os(IN_JOB_CONTAINER).is_none());
350 if services.is_empty() && container.is_none() {
351 return true;
352 }
353 // A self-hosted machine without Docker runs the steps as before,
354 // on the machine, without the containers.
355 if !self.docker_hosted && self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_none() {
356 self.log.line("##[warning]Docker does not answer on this runner, so the job's `services` and `container` are not started and its steps run on the machine. Run the runner directly on a machine with Docker (`--no-docker`) to start them.");
357 return true;
358 }
359 self.log.line("##[group]Initialize containers");
360 let ok = self.docker_ready() && self.start_containers_now(services, container);
361 self.log.line("##[endgroup]");
362 ok
363 }
364
365 fn start_containers_now(&mut self, services: Vec<(String, ContainerSpec)>, container: Option<ContainerSpec>) -> bool {
366 let job_id = docker_name(&format!("{}_{:x}", self.spec["name"].as_str().unwrap_or("job"), super::rand_id() & 0xffff_ffff));
367 let network = format!("g1t_{job_id}");
368 if !self.docker(&["network".into(), "create".into(), "--label".into(), "g1t-job".into(), network.clone()], &BTreeMap::new(), Duration::from_secs(60)) {
369 self.log.line("##[error]Could not make the job's network.");
370 return false;
371 }
372 self.network = Some(network.clone());
373
374 let mut services_context = Map::new();
375 for (service, spec) in &services {
376 if !self.pull(&spec.image, spec.credentials.as_ref()) {
377 self.log.line(&format!("##[error]Could not pull {} for the service `{service}`.", spec.image));
378 return false;
379 }
380 let name = docker_name(&format!("{service}_{job_id}"));
381 let args = create_args(&name, Some(&network), Some(service), spec, &[], false);
382 if !self.docker(&args, &spec.env, Duration::from_secs(300)) {
383 self.log.line(&format!("##[error]Could not create the service `{service}`."));
384 return false;
385 }
386 self.services.push((service.clone(), name.clone()));
387 if !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
388 self.log.line(&format!("##[error]Could not start the service `{service}`."));
389 return false;
390 }
391 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
392 let ports: Map<String, Value> = port_map(&spec.ports).into_iter().map(|(k, v)| (k, json!(v))).collect();
393 services_context.insert(service.clone(), json!({ "id": id, "network": network, "ports": ports }));
394 }
395
396 if let Some(spec) = container {
397 if !self.pull(&spec.image, spec.credentials.as_ref()) {
398 self.log.line(&format!("##[error]Could not pull {} for the job's container.", spec.image));
399 return false;
400 }
401 let name = docker_name(&format!("job_{job_id}"));
402 let mounts = self.container_mounts();
403 let mut spec = spec;
404 spec.env.insert("HOME".into(), GITHUB_HOME.into());
405 spec.env.insert("CI".into(), "true".into());
406 spec.env.insert("GITHUB_ACTIONS".into(), "true".into());
407 let mut args = create_args(&name, Some(&network), None, &spec, &mounts, true);
408 // Steps start in the workspace.
409 args.splice(1..1, ["-w".to_owned(), self.workspace.display().to_string()]);
410 if !self.docker(&args, &spec.env, Duration::from_secs(300)) || !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
411 self.log.line("##[error]Could not start the job's container.");
412 return false;
413 }
414 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
415 let has_bash = self.docker_output(&["exec", &name, "sh", "-c", "command -v bash"]).is_some_and(|out| !out.is_empty());
416 let node = self.docker_output(&["exec", &name, CONTAINER_NODE, "--version"]).is_some();
417 let path = self.docker_output(&["exec", &name, "sh", "-c", "printf %s \"$PATH\""]).unwrap_or_else(|| "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into());
418 if !node {
419 self.log.line("##[warning]The runner's Node does not run in this image (it needs glibc and libstdc++), so JavaScript actions run beside the container, on g1t's image, with the same files.");
420 }
421 self.job_context.insert("container".into(), json!({ "id": id, "network": network }));
422 self.container = Some(JobContainer { id: name, shell: if has_bash { "bash" } else { "sh" }, node, path });
423 }
424
425 // Services with a health check are waited for.
426 for (service, name) in self.services.clone() {
427 if !self.wait_healthy(&service, &name) {
428 return false;
429 }
430 }
431 if !services_context.is_empty() {
432 self.job_context.insert("services".into(), Value::Object(services_context));
433 }
434 self.log_docker_notes();
435 true
436 }
437
438 /// GitHub's runner's folders, at the same paths, and Docker's socket.
439 fn container_mounts(&self) -> Vec<(String, String)> {
440 let home = super::paths::under_home(super::paths::HOME_RUNNER);
441 let github_home = self.temp.join("_github_home");
442 let _ = std::fs::create_dir_all(&github_home);
443 let mut mounts: Vec<(String, String)> = ["work", "_temp", "_actions", "_tool"]
444 .iter()
445 .map(|dir| {
446 let path = home.join(dir);
447 let _ = std::fs::create_dir_all(&path);
448 (path.display().to_string(), path.display().to_string())
449 })
450 .collect();
451 mounts.push((github_home.display().to_string(), GITHUB_HOME.into()));
452 if let Some(node) = which_node() {
453 mounts.push((node.display().to_string(), format!("{CONTAINER_NODE}:ro")));
454 }
455 mounts.push((crate::docker::engine::DOCKER_SOCKET.into(), crate::docker::engine::DOCKER_SOCKET.into()));
456 mounts
457 }
458
459 fn wait_healthy(&mut self, service: &str, name: &str) -> bool {
460 let limit = Duration::from_secs(600).min(self.remaining_time());
461 let until = Instant::now() + limit;
462 let mut wait = Duration::from_secs(1);
463 loop {
464 let status = self.docker_output(&["inspect", "--format", "{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}", name]).unwrap_or_default();
465 match status.as_str() {
466 "" => return true,
467 "healthy" => {
468 self.log.line(&format!("{service} is healthy."));
469 return true;
470 }
471 "unhealthy" => {
472 self.log.line(&format!("##[error]The service `{service}` is unhealthy."));
473 self.service_logs(service, name);
474 return false;
475 }
476 _ => {}
477 }
478 if Instant::now() >= until {
479 self.log.line(&format!("##[error]The service `{service}` did not become healthy in {} s.", limit.as_secs()));
480 self.service_logs(service, name);
481 return false;
482 }
483 self.log.line(&format!("Waiting for {service} to be healthy ({status})."));
484 std::thread::sleep(wait);
485 wait = (wait * 2).min(Duration::from_secs(8));
486 }
487 }
488
489 fn service_logs(&mut self, service: &str, name: &str) {
490 self.log.line(&format!("##[group]Service container {service}"));
491 self.docker(&["logs".into(), "--tail".into(), "200".into(), name.into()], &BTreeMap::new(), Duration::from_secs(60));
492 self.log.line("##[endgroup]");
493 }
494
495 /// Whether the job has containers to stop when it ends.
496 pub(crate) fn has_containers(&self) -> bool {
497 self.network.is_some()
498 }
499
500 /// GitHub's "Stop containers": each service's log, then the job's
501 /// containers and network removed.
502 pub(crate) fn stop_containers(&mut self) -> bool {
503 for (service, name) in self.services.clone() {
504 self.service_logs(&service, &name);
505 }
506 let mut names: Vec<String> = self.services.iter().map(|(_, name)| name.clone()).collect();
507 if let Some(container) = &self.container {
508 names.push(container.id.clone());
509 }
510 if !names.is_empty() {
511 let mut args = vec!["rm".to_owned(), "--force".to_owned()];
512 args.extend(names);
513 self.docker(&args, &BTreeMap::new(), Duration::from_secs(120));
514 }
515 if let Some(network) = self.network.take() {
516 self.docker(&["network".into(), "rm".into(), network], &BTreeMap::new(), Duration::from_secs(60));
517 }
518 self.container = None;
519 true
520 }
521
522 /// The variables a process inside a container is given: the step's,
523 /// GitHub's and the job's, but not the sandbox's own (its `PATH`,
524 /// `HOME` and the like, which mean nothing in another image).
525 pub(crate) fn container_env(&self, step_env: &BTreeMap<String, String>, full: BTreeMap<String, String>, image_path: &str) -> BTreeMap<String, String> {
526 let mut out: BTreeMap<String, String> = full
527 .into_iter()
528 .filter(|(name, _)| step_env.contains_key(name) || !self.host_env.contains(name) || name.starts_with("GITHUB_") || name.starts_with("RUNNER_"))
529 .collect();
530 if !step_env.contains_key("PATH") {
531 out.remove("PATH");
532 if !self.path_prepend_entries().is_empty() {
533 out.insert("PATH".into(), format!("{}:{image_path}", self.path_prepend_entries().join(":")));
534 }
535 }
536 if !step_env.contains_key("HOME") {
537 out.remove("HOME");
538 }
539 out
540 }
541
542 /// A step's command, run inside the job's container instead.
543 pub(crate) fn in_container(&self, program: &str, args: &[String], dir: &Path, env: BTreeMap<String, String>) -> Option<Command> {
544 let container = self.container.as_ref()?;
545 let mut command = Command::new("docker");
546 command.args(["exec", "-w", &dir.display().to_string()]);
547 for name in env.keys() {
548 command.args(["-e", name]);
549 }
550 command.arg(&container.id).arg(program).args(args);
551 command.env_clear().envs(self.docker_cli_env()).envs(env);
552 Some(command)
553 }
554
555 /// Runs a container for a Docker action or a `docker://` step, as
556 /// GitHub's runner does: the workspace at /github/workspace, the step's
557 /// files at /github/file_commands, the job's network.
558 pub(crate) fn run_docker(&mut self, run: &DockerRun) -> (bool, BTreeMap<String, String>, BTreeMap<String, String>) {
559 let fail = (false, BTreeMap::new(), BTreeMap::new());
560 if !self.docker_ready() {
561 return fail;
562 }
563 let id = format!("{:x}", super::rand_id());
564 let Ok(files) = StepFiles::new(&self.temp, &id) else { return fail };
565 let commands_dir = self.temp.join("_runner_file_commands");
566 let workflow_dir = self.temp.join("_github_workflow");
567 let home_dir = self.temp.join("_github_home");
568 for dir in [&workflow_dir, &home_dir] {
569 let _ = std::fs::create_dir_all(dir);
570 }
571 let _ = std::fs::copy(self.temp.join("event.json"), workflow_dir.join("event.json"));
572
573 let full = self.process_env(&run.env, &files);
574 let mut env = self.container_env(&run.env, full, "");
575 env.remove("PATH");
576 // Paths as the container sees them.
577 let moved = |path: &Path| format!("{GITHUB_FILE_COMMANDS}/{}", path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default());
578 env.insert("GITHUB_OUTPUT".into(), moved(&files.output));
579 env.insert("GITHUB_ENV".into(), moved(&files.env));
580 env.insert("GITHUB_PATH".into(), moved(&files.path));
581 env.insert("GITHUB_STATE".into(), moved(&files.state));
582 env.insert("GITHUB_STEP_SUMMARY".into(), moved(&files.summary));
583 env.insert("GITHUB_WORKSPACE".into(), GITHUB_WORKSPACE.into());
584 env.insert("GITHUB_EVENT_PATH".into(), format!("{GITHUB_WORKFLOW}/event.json"));
585 env.insert("HOME".into(), GITHUB_HOME.into());
586 env.remove("GITHUB_ACTION_PATH");
587
588 let mut args: Vec<String> = vec!["run".into(), "--rm".into(), "--label".into(), "g1t-job".into(), "--workdir".into(), GITHUB_WORKSPACE.into()];
589 if let Some(network) = &self.network {
590 args.extend(["--network".into(), network.clone()]);
591 }
592 for (from, to) in [
593 (self.workspace.clone(), GITHUB_WORKSPACE),
594 (home_dir, GITHUB_HOME),
595 (workflow_dir, GITHUB_WORKFLOW),
596 (commands_dir, GITHUB_FILE_COMMANDS),
597 (PathBuf::from(crate::docker::engine::DOCKER_SOCKET), crate::docker::engine::DOCKER_SOCKET),
598 ] {
599 args.extend(["-v".into(), format!("{}:{to}", from.display())]);
600 }
601 for name in env.keys() {
602 args.extend(["-e".into(), name.clone()]);
603 }
604 if let Some(entrypoint) = &run.entrypoint {
605 args.extend(["--entrypoint".into(), entrypoint.clone()]);
606 }
607 args.push(run.image.clone());
608 args.extend(run.args.iter().cloned());
609
610 crate::abuse::touch();
611 if let Some(miner) = crate::abuse::miner_in(&shown(&args)) {
612 self.log.line(&format!("##[error]g1t does not run cryptocurrency miners ({miner}). This step was not run."));
613 return fail;
614 }
615 self.log.line(&format!("[command]docker {}", shown(&args)));
616 let mut command = Command::new("docker");
617 command.args(&args).env_clear().envs(self.docker_cli_env()).envs(&env);
618 let mut commands = Commands::default();
619 let ended = process::run(command, self.remaining_time(), &mut self.log, &mut commands);
620 self.log_docker_notes();
621 let ok = match ended {
622 Ok(Ended::Exited(0)) => true,
623 Ok(Ended::Exited(code)) => {
624 self.log.line(&format!("##[error]Docker run failed with exit code {code}."));
625 false
626 }
627 Ok(Ended::TimedOut) => {
628 self.log.line("##[error]The step ran past its time limit and was stopped.");
629 false
630 }
631 Err(error) => {
632 self.log.line(&format!("##[error]docker could not be started: {error}"));
633 false
634 }
635 };
636 let (outputs, state) = self.absorb(&files, &commands);
637 (ok, outputs, state)
638 }
639
640 /// Builds a Docker action's image from its Dockerfile, once per job.
641 pub(crate) fn build_action_image(&mut self, dir: &Path, dockerfile: &str, tag_of: &str) -> Option<String> {
642 let tag = format!("g1t-action/{}", docker_name(tag_of));
643 if self.built_actions.contains(&tag) {
644 return Some(tag);
645 }
646 if !self.docker_ready() {
647 return None;
648 }
649 let file = dir.join(dockerfile);
650 let args = vec!["build".into(), "-t".into(), tag.clone(), "-f".into(), file.display().to_string(), dir.display().to_string()];
651 if !self.docker(&args, &BTreeMap::new(), Duration::from_secs(1800)) {
652 self.log.line("##[error]The action's image did not build.");
653 return None;
654 }
655 self.built_actions.insert(tag.clone());
656 Some(tag)
657 }
658
659 /// `docker/setup-buildx-action` on g1t's machines: the job's own
660 /// Engine is the builder (BuildKit, the `docker` driver, with the
661 /// containerd image store, so cache export and attestations work).
662 pub(crate) fn setup_buildx(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
663 if !self.docker_ready() {
664 return (false, BTreeMap::new());
665 }
666 if let Some(driver) = with.get("driver").filter(|d| !d.is_empty() && *d != "docker") {
667 self.log.line(&format!("`driver: {driver}` is not used on g1t's machines: builds run on this job's own Docker Engine (BuildKit, the `docker` driver), which keeps the sandbox's network and guardrails."));
668 }
669 for input in ["driver-opts", "buildkitd-flags", "buildkitd-config", "buildkitd-config-inline", "endpoint"] {
670 if with.get(input).is_some_and(|v| !v.trim().is_empty()) {
671 self.log.line(&format!("`{input}` is not used on g1t's machines."));
672 }
673 }
674 self.docker(&["buildx".into(), "version".into()], &BTreeMap::new(), Duration::from_secs(60));
675 let inspect = self.docker_output(&["buildx", "inspect", "default"]).unwrap_or_default();
676 let platforms = inspect
677 .lines()
678 .find_map(|line| line.trim().strip_prefix("Platforms:"))
679 .map(|p| p.split(',').map(|s| s.trim().trim_end_matches('*').to_owned()).filter(|s| !s.is_empty()).collect::<Vec<_>>().join(","))
680 .unwrap_or_else(|| "linux/amd64".into());
681 self.log.line(&format!("Builder: default (this job's Docker Engine), platforms {platforms}"));
682 let mut outputs = BTreeMap::new();
683 outputs.insert("name".into(), "default".into());
684 outputs.insert("driver".into(), "docker".into());
685 outputs.insert("platforms".into(), platforms.clone());
686 outputs.insert("endpoint".into(), "default".into());
687 outputs.insert("status".into(), "running".into());
688 outputs.insert("flags".into(), String::new());
689 outputs.insert(
690 "nodes".into(),
691 json!([{ "name": "default", "endpoint": "default", "status": "running", "platforms": platforms }]).to_string(),
692 );
693 (true, outputs)
694 }
695}
696
697/// The runner's Node, its real file (not a link), to mount into a job
698/// container.
699fn which_node() -> Option<PathBuf> {
700 let path = std::env::var_os("PATH")?;
701 std::env::split_paths(&path).map(|dir| dir.join("node")).find(|p| p.is_file()).and_then(|p| std::fs::canonicalize(p).ok())
702}
703
704/// The job context's `container` and `services`, merged with its status.
705pub(crate) fn job_context(status: &str, extra: &Map<String, Value>) -> Value {
706 let mut job = Map::new();
707 job.insert("status".into(), json!(status));
708 for (key, value) in extra {
709 job.insert(key.clone(), value.clone());
710 }
711 Value::Object(job)
712}
713
714/// Names a set of tags already built in this job.
715pub(crate) type Built = BTreeSet<String>;
716
717#[cfg(test)]
718mod tests {
719 use super::*;
720
721 #[test]
722 fn options_split_as_a_shell_would() {
723 assert_eq!(
724 split_words(r#"--health-cmd "pg_isready -U postgres" --health-interval 10s --health-retries=5"#),
725 vec!["--health-cmd", "pg_isready -U postgres", "--health-interval", "10s", "--health-retries=5"]
726 );
727 assert_eq!(split_words("--health-cmd 'redis-cli ping' --tmpfs /var/lib/x"), vec!["--health-cmd", "redis-cli ping", "--tmpfs", "/var/lib/x"]);
728 assert_eq!(split_words(r#"a\ b "c\"d" ''"#), vec!["a b", "c\"d", ""]);
729 assert!(split_words(" ").is_empty());
730 }
731
732 #[test]
733 fn services_are_read_from_either_form() {
734 assert_eq!(container_spec(&json!("redis:7")).image, "redis:7");
735 let spec = container_spec(&json!({
736 "image": "postgres:17",
737 "env": { "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "app" },
738 "ports": ["5432:5432", 6543],
739 "volumes": ["/data:/var/lib/postgresql/data"],
740 "options": "--health-cmd pg_isready --health-interval 10s",
741 "credentials": { "username": "me", "password": "token" },
742 }));
743 assert_eq!(spec.image, "postgres:17");
744 assert_eq!(spec.env["POSTGRES_DB"], "app");
745 assert_eq!(spec.ports, vec!["5432:5432", "6543"]);
746 assert_eq!(spec.options, vec!["--health-cmd", "pg_isready", "--health-interval", "10s"]);
747 assert_eq!(spec.credentials, Some(("me".into(), "token".into())));
748 // An empty image is a service the job leaves out, as on GitHub.
749 assert_eq!(container_spec(&json!({ "image": "" })).image, "");
750 }
751
752 #[test]
753 fn ports_map_container_to_host() {
754 let map = port_map(&["5432:5432".into(), "6543:5432/tcp".into(), "6379".into(), "127.0.0.1:8080:80".into(), ":9000".into()]);
755 assert_eq!(map["5432"], "6543");
756 assert_eq!(map["6379"], "6379");
757 assert_eq!(map["80"], "8080");
758 assert_eq!(map["9000"], "9000");
759 }
760
761 #[test]
762 fn credentials_sign_in_to_the_images_registry() {
763 assert_eq!(registry_of("ghcr.io/acme/db:1"), "ghcr.io");
764 assert_eq!(registry_of("g1t.sh/acme/web"), "g1t.sh");
765 assert_eq!(registry_of("localhost:5000/x"), "localhost:5000");
766 assert_eq!(registry_of("acme/private"), "https://index.docker.io/v1/");
767 assert_eq!(registry_of("postgres"), "https://index.docker.io/v1/");
768 }
769
770 #[test]
771 fn a_service_is_created_with_its_values_passed_by_name() {
772 let spec = container_spec(&json!({
773 "image": "postgres:17",
774 "env": { "POSTGRES_PASSWORD": "secret" },
775 "ports": ["5432:5432"],
776 "options": "--health-cmd pg_isready",
777 }));
778 let args = create_args("postgres_job", Some("g1t_job"), Some("postgres"), &spec, &[], false);
779 assert_eq!(
780 args,
781 vec![
782 "create", "--name", "postgres_job", "--label", "g1t-job", "--network", "g1t_job", "--network-alias", "postgres", "-p", "5432:5432", "-e",
783 "POSTGRES_PASSWORD", "--health-cmd", "pg_isready", "postgres:17"
784 ]
785 );
786 assert!(!args.iter().any(|a| a.contains("secret")));
787 let job = create_args("job_x", Some("g1t_job"), None, &container_spec(&json!("node:24")), &[("/home/runner/work".into(), "/home/runner/work".into())], true);
788 assert!(job.ends_with(&["--entrypoint".into(), "tail".into(), "node:24".into(), "-f".into(), "/dev/null".into()]));
789 assert!(job.contains(&"/home/runner/work:/home/runner/work".to_owned()));
790 }
791
792 #[test]
793 fn names_are_docker_names() {
794 assert_eq!(docker_name("Build & test_1a2b"), "build___test_1a2b");
795 assert_eq!(docker_name("docker/login-action@v3"), "docker_login-action_v3");
796 }
797
798 #[test]
799 fn the_job_context_carries_containers() {
800 let mut extra = Map::new();
801 extra.insert("services".into(), json!({ "db": { "ports": { "5432": "5432" } } }));
802 let job = job_context("success", &extra);
803 assert_eq!(job["status"], "success");
804 assert_eq!(job["services"]["db"]["ports"]["5432"], "5432");
805 }
806}

This file's history is long; its oldest lines are credited to the oldest commit read.