Skip to content
Sign up

flagon-io/g1t

public
0
807 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge branch 'main' into actions-toolkit-oidc-artifacts1//! Containers a job asks for, as GitHub's runner starts them, with the
2//! `docker` command: its `services:` (a database beside the steps), its
3//! `container:` (every step run inside an image), `uses: docker://image`
4//! steps and Docker actions (`runs.using: docker`).
5//!
6//! On g1t's machines the Engine is the job's own (crate::docker). Every
7//! container shares the job's network there, so a service is reached at
8//! `localhost:<port>` as on GitHub's runner, and by its name as from a job
9//! container: the API proxy makes each service's name mean 127.0.0.1.
10
11use std::collections::{BTreeMap, BTreeSet};
12use std::path::{Path, PathBuf};
13use std::process::Command;
14use std::time::{Duration, Instant};
15
16use g1t_actions::expr;
17use serde_json::{Map, Value, json};
18
19use super::files::StepFiles;
20use super::process::{self, Commands, Ended};
21use super::Job;
22
23/// Set in a job a self-hosted runner started inside its `container:` image.
24pub(crate) const IN_JOB_CONTAINER: &str = "G1T_JOB_CONTAINER";
25/// Where a job container finds the runner's Node, for JavaScript actions.
26pub(crate) const CONTAINER_NODE: &str = "/__e/node24/bin/node";
27/// GitHub's folders inside a Docker action's container.
28const GITHUB_WORKSPACE: &str = "/github/workspace";
29const GITHUB_HOME: &str = "/github/home";
30const GITHUB_WORKFLOW: &str = "/github/workflow";
31const GITHUB_FILE_COMMANDS: &str = "/github/file_commands";
32
33/// The job's own container, when it has `container:`.
34pub(crate) struct JobContainer {
35 pub(crate) id: String,
36 /// `bash`, or `sh` in an image without it.
37 pub(crate) shell: &'static str,
38 /// Whether the runner's Node runs in it.
39 pub(crate) node: bool,
40 /// The image's own `PATH`.
41 pub(crate) path: String,
42}
43
44/// A `services:` entry or `container:`, read.
45#[derive(Debug, Default, PartialEq)]
46pub(crate) struct ContainerSpec {
47 pub(crate) image: String,
48 pub(crate) env: BTreeMap<String, String>,
49 pub(crate) ports: Vec<String>,
50 pub(crate) volumes: Vec<String>,
51 pub(crate) options: Vec<String>,
52 pub(crate) credentials: Option<(String, String)>,
53 pub(crate) command: Vec<String>,
54 pub(crate) entrypoint: Option<String>,
55}
56
57/// Splits a command line as a shell would: words, with `'…'`, `"…"` and
58/// `\` quoting. No variables are expanded.
59pub(crate) fn split_words(text: &str) -> Vec<String> {
60 let mut words = Vec::new();
61 let mut word = String::new();
62 let mut started = false;
63 let mut chars = text.chars().peekable();
64 while let Some(c) = chars.next() {
65 match c {
66 '\'' => {
67 started = true;
68 for c in chars.by_ref() {
69 if c == '\'' {
70 break;
71 }
72 word.push(c);
73 }
74 }
75 '"' => {
76 started = true;
77 while let Some(c) = chars.next() {
78 match c {
79 '"' => break,
80 '\\' if matches!(chars.peek(), Some('"' | '\\' | '$' | '`')) => word.push(chars.next().unwrap_or('\\')),
81 c => word.push(c),
82 }
83 }
84 }
85 '\\' => {
86 started = true;
87 if let Some(next) = chars.next()
88 && next != '\n'
89 {
90 word.push(next);
91 }
92 }
93 c if c.is_whitespace() => {
94 if started {
95 words.push(std::mem::take(&mut word));
96 started = false;
97 }
98 }
99 c => {
100 started = true;
101 word.push(c);
102 }
103 }
104 }
105 if started {
106 words.push(word);
107 }
108 words
109}
110
111fn texts(value: Option<&Value>) -> Vec<String> {
112 match value {
113 Some(Value::Array(items)) => items.iter().map(expr::to_text).filter(|s| !s.is_empty()).collect(),
114 Some(Value::Null) | None => Vec::new(),
115 Some(other) => vec![expr::to_text(other)].into_iter().filter(|s| !s.is_empty()).collect(),
116 }
117}
118
119/// Reads a `services:` entry or `container:`, its expressions already
120/// read: a string (the image) or a mapping.
121pub(crate) fn container_spec(value: &Value) -> ContainerSpec {
122 match value {
123 Value::String(image) => ContainerSpec { image: image.trim().to_owned(), ..ContainerSpec::default() },
124 Value::Object(fields) => ContainerSpec {
125 image: fields.get("image").map(expr::to_text).unwrap_or_default().trim().to_owned(),
126 env: fields
127 .get("env")
128 .and_then(Value::as_object)
129 .map(|env| env.iter().map(|(k, v)| (k.clone(), expr::to_text(v))).collect())
130 .unwrap_or_default(),
131 ports: texts(fields.get("ports")),
132 volumes: texts(fields.get("volumes")),
133 options: fields.get("options").map(|o| split_words(&expr::to_text(o))).unwrap_or_default(),
134 credentials: fields.get("credentials").and_then(Value::as_object).and_then(|c| {
135 let username = c.get("username").map(expr::to_text).unwrap_or_default();
136 let password = c.get("password").map(expr::to_text).unwrap_or_default();
137 (!username.is_empty() || !password.is_empty()).then_some((username, password))
138 }),
139 command: fields.get("command").map(|c| split_words(&expr::to_text(c))).unwrap_or_default(),
140 entrypoint: fields.get("entrypoint").map(expr::to_text).filter(|e| !e.is_empty()),
141 },
142 _ => ContainerSpec::default(),
143 }
144}
145
146/// `job.services.<id>.ports`: each container port, and the host port it
147/// is reached on. On g1t's machines a port left for Docker to choose is
148/// the container's own.
149pub(crate) fn port_map(ports: &[String]) -> BTreeMap<String, String> {
150 let mut map = BTreeMap::new();
151 for port in ports {
152 let without_protocol = port.split('/').next().unwrap_or(port);
153 let parts: Vec<&str> = without_protocol.split(':').collect();
154 let (host, container) = match parts.as_slice() {
155 [container] => (*container, *container),
156 [host, container] => (if host.is_empty() { *container } else { *host }, *container),
157 [_, host, container] => (if host.is_empty() { *container } else { *host }, *container),
158 _ => continue,
159 };
160 if !container.is_empty() {
161 map.insert(container.to_owned(), host.to_owned());
162 }
163 }
164 map
165}
166
167/// The registry an image is pulled from, for signing in with
168/// `credentials:`.
169pub(crate) fn registry_of(image: &str) -> String {
170 match image.split_once('/') {
171 Some((first, _)) if first.contains('.') || first.contains(':') || first == "localhost" => first.to_owned(),
172 _ => "https://index.docker.io/v1/".to_owned(),
173 }
174}
175
176/// A name for Docker from any text: lower case, letters, digits, `_`,
177/// `.` and `-`.
178pub(crate) fn docker_name(text: &str) -> String {
179 let name: String = text
180 .to_ascii_lowercase()
181 .chars()
182 .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-') { c } else { '_' })
183 .collect();
184 name.trim_matches(['_', '.', '-']).chars().take(100).collect()
185}
186
187/// The `docker create` arguments of a container. `env` is passed by name
188/// (`-e NAME`), its values in the command's environment, so secrets never
189/// stand on a command line. `keep_alive`: a job container, which waits
190/// while steps run in it.
191pub(crate) fn create_args(name: &str, network: Option<&str>, alias: Option<&str>, spec: &ContainerSpec, mounts: &[(String, String)], keep_alive: bool) -> Vec<String> {
192 let mut args: Vec<String> = vec!["create".into(), "--name".into(), name.into(), "--label".into(), "g1t-job".into()];
193 if let Some(network) = network {
194 args.extend(["--network".into(), network.into()]);
195 if let Some(alias) = alias {
196 args.extend(["--network-alias".into(), alias.into()]);
197 }
198 }
199 for port in &spec.ports {
200 args.extend(["-p".into(), port.clone()]);
201 }
202 for volume in &spec.volumes {
203 args.extend(["-v".into(), volume.clone()]);
204 }
205 for (from, to) in mounts {
206 args.extend(["-v".into(), format!("{from}:{to}")]);
207 }
208 for name in spec.env.keys() {
209 args.extend(["-e".into(), name.clone()]);
210 }
211 args.extend(spec.options.iter().cloned());
212 if keep_alive {
213 args.extend(["--entrypoint".into(), "tail".into(), spec.image.clone(), "-f".into(), "/dev/null".into()]);
214 } else {
215 if let Some(entrypoint) = &spec.entrypoint {
216 args.extend(["--entrypoint".into(), entrypoint.clone()]);
217 }
218 args.push(spec.image.clone());
219 args.extend(spec.command.iter().cloned());
220 }
221 args
222}
223
224/// A Docker action's run, or a `docker://` step's.
225#[derive(Clone, Debug)]
226pub(crate) struct DockerRun {
227 pub(crate) image: String,
228 pub(crate) entrypoint: Option<String>,
229 pub(crate) args: Vec<String>,
230 /// The step's variables, `INPUT_*` included.
231 pub(crate) env: BTreeMap<String, String>,
232}
233
234/// What a command line shows: secrets are passed by name, so nothing
235/// needs hiding, but long lists are kept readable.
236fn shown(args: &[String]) -> String {
237 args.iter().map(|a| if a.contains(' ') || a.is_empty() { format!("'{a}'") } else { a.clone() }).collect::<Vec<_>>().join(" ")
238}
239
240impl Job {
241 /// Runs `docker` with `args`, its output in the log. `env`: the
242 /// variables it passes to a container by name.
243 pub(crate) fn docker(&mut self, args: &[String], env: &BTreeMap<String, String>, timeout: Duration) -> bool {
244 self.log.line(&format!("[command]docker {}", shown(args)));
245 let mut command = Command::new("docker");
246 command.args(args).env_clear().envs(self.docker_cli_env()).envs(env);
247 let mut commands = Commands::default();
248 matches!(process::run(command, timeout.min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
249 }
250
251 /// What `docker` prints, or None if it fails.
252 pub(crate) fn docker_output(&self, args: &[&str]) -> Option<String> {
253 let output = Command::new("docker").args(args).env_clear().envs(self.docker_cli_env()).output().ok()?;
254 output.status.success().then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
255 }
256
257 /// What the `docker` command itself needs of the sandbox's variables:
258 /// where it is, where its config is, and how to reach the Engine.
259 fn docker_cli_env(&self) -> BTreeMap<String, String> {
260 ["PATH", "HOME", "DOCKER_HOST", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY"]
261 .iter()
262 .filter_map(|name| self.base_env_value(name).map(|value| (name.to_string(), value)))
263 .collect()
264 }
265
266 /// Makes sure Docker answers before a job's containers start: the
267 /// job's own Engine, on g1t's machines.
268 fn docker_ready(&mut self) -> bool {
269 if self.docker_hosted {
270 let started = crate::docker::engine::ensure_started();
271 self.log_docker_notes();
272 // Why it could not start is among the notes just logged.
273 return started.is_ok();
274 }
275 if self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_some() {
276 return true;
277 }
The self-hosted runner runs as a container under Sysbox, with a Docker Engine of its own inside, and each job runs in a fresh container on that engine, so neither the runner nor its jobs reach the machine: `docker run --runtime=sysbox-runc -v g1t-runner:/data -e G1T_RUNNER_URL=… -e G1T_RUNNER_TOKEN=… g1t.sh/flagon-io/g1t-runner`, with docker-init as PID 1 supervising g1t-runner and g1t-runner supervising dockerd. The image refuses to start when it can see the machine's Docker socket mounted in, an engine socket it did not make, DOCKER_HOST, or a root that is the machine's (no Sysbox, or --privileged), unless G1T_RUNNER_UNSAFE_ALLOW_HOST_ACCESS=1 is set, and its log says which isolation it has: nested-container, container or host, which it now also tells g1t when it registers (kept on the runner, shown nowhere yet). Every job container drops all capabilities but eight it needs, has no-new-privileges (agent work keeps sudo for its guardrails), a process, memory and CPU limit (--pids-limit, --memory, --cpus, with defaults), a network and a /home/runner volume of its own removed after it, and no mount but the read-only harness; a job that uses Docker gets an engine of its own beside it, started the first time it is used and removed after, unless the runner is registered with --no-job-docker. --no-docker keeps working but is deprecated: the runner says so when it starts, every job's log starts with a warning that it ran on the host unsandboxed, and it refuses a public repository's work now that g1t says a job's repository visibility. Windows and macOS sandboxes (Hyper-V-isolated containers, ephemeral macOS VMs) are designed and not built. The runner is 0.2.0; the self-hosted runners guide, the limitations page and the New runner command say so.278 self.log.line("##[error]This job needs Docker (`services`, `container` or a Docker action), and Docker does not answer here. On a self-hosted runner, run the runner's image under Sysbox, which gives each job that uses Docker an engine of its own.");
Merge branch 'main' into actions-toolkit-oidc-artifacts279 false
280 }
281
282 pub(crate) fn log_docker_notes(&mut self) {
283 for line in crate::docker::take_notes() {
284 self.log.line(&line);
285 }
286 }
287
288 /// Pulls an image, signed in with `credentials` if it has them.
289 fn pull(&mut self, image: &str, credentials: Option<&(String, String)>) -> bool {
290 let Some((username, password)) = credentials else {
291 return self.docker(&["pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800));
292 };
293 // A config of its own, so the credentials go no further than this pull.
294 let config = self.temp.join(format!("docker-config-{:x}", super::rand_id()));
295 let _ = std::fs::create_dir_all(&config);
296 let registry = registry_of(image);
297 let config_text = config.display().to_string();
298 self.log.line(&format!("[command]docker --config {config_text} login {registry} --username *** --password-stdin"));
299 let login = Command::new("docker")
300 .args(["--config", &config_text, "login", &registry, "--username", username, "--password-stdin"])
301 .env_clear()
302 .envs(self.docker_cli_env())
303 .stdin(std::process::Stdio::piped())
304 .stdout(std::process::Stdio::piped())
305 .stderr(std::process::Stdio::piped())
306 .spawn()
307 .and_then(|mut child| {
308 use std::io::Write;
309 if let Some(mut stdin) = child.stdin.take() {
310 let _ = stdin.write_all(password.as_bytes());
311 }
312 child.wait_with_output()
313 });
314 let ok = match login {
315 Ok(output) if output.status.success() => {
316 self.docker(&["--config".into(), config_text.clone(), "pull".into(), image.into()], &BTreeMap::new(), Duration::from_secs(1800))
317 }
318 Ok(output) => {
319 self.log.line(&format!("##[error]Could not sign in to {registry}: {}", String::from_utf8_lossy(&output.stderr).trim()));
320 false
321 }
322 Err(error) => {
323 self.log.line(&format!("##[error]Could not run docker login: {error}"));
324 false
325 }
326 };
327 let _ = std::fs::remove_dir_all(&config);
328 ok
329 }
330
331 /// Reads a `services:` entry or `container:` with the job's contexts.
332 fn read_container(&self, value: &Value) -> ContainerSpec {
333 let frame = super::Frame::default();
334 let env = self.env_context(&frame);
335 let contexts = self.contexts_for(&frame, &env);
336 let value = self.with_scope(&contexts, |scope| expr::interpolate_value(value, scope)).unwrap_or(Value::Null);
337 container_spec(&value)
338 }
339
340 /// Starts the job's services and its container, before its steps:
341 /// GitHub's "Initialize containers". Returns whether they all started.
342 pub(crate) fn start_containers(&mut self) -> bool {
343 let services: Vec<(String, Value)> = self.spec["spec"]["services"].as_object().map(|s| s.iter().map(|(k, v)| (k.clone(), v.clone())).collect()).unwrap_or_default();
344 let container = self.spec["spec"].get("container").filter(|c| !c.is_null()).cloned();
345 let container = container.map(|c| self.read_container(&c)).filter(|c| !c.image.is_empty());
346 let services: Vec<(String, ContainerSpec)> = services.into_iter().map(|(name, value)| (name, self.read_container(&value))).filter(|(_, spec)| !spec.image.is_empty()).collect();
347 // A self-hosted runner in Docker mode started this job in its
348 // `container:` image already (selfhosted/exec.rs).
349 let container = container.filter(|_| std::env::var_os(IN_JOB_CONTAINER).is_none());
350 if services.is_empty() && container.is_none() {
351 return true;
352 }
353 // A self-hosted machine without Docker runs the steps as before,
354 // on the machine, without the containers.
355 if !self.docker_hosted && self.docker_output(&["version", "--format", "{{.Server.Version}}"]).is_none() {
The self-hosted runner runs as a container under Sysbox, with a Docker Engine of its own inside, and each job runs in a fresh container on that engine, so neither the runner nor its jobs reach the machine: `docker run --runtime=sysbox-runc -v g1t-runner:/data -e G1T_RUNNER_URL=… -e G1T_RUNNER_TOKEN=… g1t.sh/flagon-io/g1t-runner`, with docker-init as PID 1 supervising g1t-runner and g1t-runner supervising dockerd. The image refuses to start when it can see the machine's Docker socket mounted in, an engine socket it did not make, DOCKER_HOST, or a root that is the machine's (no Sysbox, or --privileged), unless G1T_RUNNER_UNSAFE_ALLOW_HOST_ACCESS=1 is set, and its log says which isolation it has: nested-container, container or host, which it now also tells g1t when it registers (kept on the runner, shown nowhere yet). Every job container drops all capabilities but eight it needs, has no-new-privileges (agent work keeps sudo for its guardrails), a process, memory and CPU limit (--pids-limit, --memory, --cpus, with defaults), a network and a /home/runner volume of its own removed after it, and no mount but the read-only harness; a job that uses Docker gets an engine of its own beside it, started the first time it is used and removed after, unless the runner is registered with --no-job-docker. --no-docker keeps working but is deprecated: the runner says so when it starts, every job's log starts with a warning that it ran on the host unsandboxed, and it refuses a public repository's work now that g1t says a job's repository visibility. Windows and macOS sandboxes (Hyper-V-isolated containers, ephemeral macOS VMs) are designed and not built. The runner is 0.2.0; the self-hosted runners guide, the limitations page and the New runner command say so.356 self.log.line("##[warning]Docker does not answer on this runner, so the job's `services` and `container` are not started and its steps run on the machine. Run the runner's image under Sysbox to give each job Docker of its own.");
Merge branch 'main' into actions-toolkit-oidc-artifacts357 return true;
358 }
359 self.log.line("##[group]Initialize containers");
360 let ok = self.docker_ready() && self.start_containers_now(services, container);
361 self.log.line("##[endgroup]");
362 ok
363 }
364
365 fn start_containers_now(&mut self, services: Vec<(String, ContainerSpec)>, container: Option<ContainerSpec>) -> bool {
366 let job_id = docker_name(&format!("{}_{:x}", self.spec["name"].as_str().unwrap_or("job"), super::rand_id() & 0xffff_ffff));
367 let network = format!("g1t_{job_id}");
368 if !self.docker(&["network".into(), "create".into(), "--label".into(), "g1t-job".into(), network.clone()], &BTreeMap::new(), Duration::from_secs(60)) {
369 self.log.line("##[error]Could not make the job's network.");
370 return false;
371 }
372 self.network = Some(network.clone());
373
374 let mut services_context = Map::new();
375 for (service, spec) in &services {
376 if !self.pull(&spec.image, spec.credentials.as_ref()) {
377 self.log.line(&format!("##[error]Could not pull {} for the service `{service}`.", spec.image));
378 return false;
379 }
380 let name = docker_name(&format!("{service}_{job_id}"));
381 let args = create_args(&name, Some(&network), Some(service), spec, &[], false);
382 if !self.docker(&args, &spec.env, Duration::from_secs(300)) {
383 self.log.line(&format!("##[error]Could not create the service `{service}`."));
384 return false;
385 }
386 self.services.push((service.clone(), name.clone()));
387 if !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
388 self.log.line(&format!("##[error]Could not start the service `{service}`."));
389 return false;
390 }
391 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
392 let ports: Map<String, Value> = port_map(&spec.ports).into_iter().map(|(k, v)| (k, json!(v))).collect();
393 services_context.insert(service.clone(), json!({ "id": id, "network": network, "ports": ports }));
394 }
395
396 if let Some(spec) = container {
397 if !self.pull(&spec.image, spec.credentials.as_ref()) {
398 self.log.line(&format!("##[error]Could not pull {} for the job's container.", spec.image));
399 return false;
400 }
401 let name = docker_name(&format!("job_{job_id}"));
402 let mounts = self.container_mounts();
403 let mut spec = spec;
404 spec.env.insert("HOME".into(), GITHUB_HOME.into());
405 spec.env.insert("CI".into(), "true".into());
406 spec.env.insert("GITHUB_ACTIONS".into(), "true".into());
407 let mut args = create_args(&name, Some(&network), None, &spec, &mounts, true);
408 // Steps start in the workspace.
409 args.splice(1..1, ["-w".to_owned(), self.workspace.display().to_string()]);
410 if !self.docker(&args, &spec.env, Duration::from_secs(300)) || !self.docker(&["start".into(), name.clone()], &BTreeMap::new(), Duration::from_secs(300)) {
411 self.log.line("##[error]Could not start the job's container.");
412 return false;
413 }
414 let id = self.docker_output(&["inspect", "--format", "{{.Id}}", &name]).unwrap_or_default();
415 let has_bash = self.docker_output(&["exec", &name, "sh", "-c", "command -v bash"]).is_some_and(|out| !out.is_empty());
416 let node = self.docker_output(&["exec", &name, CONTAINER_NODE, "--version"]).is_some();
417 let path = self.docker_output(&["exec", &name, "sh", "-c", "printf %s \"$PATH\""]).unwrap_or_else(|| "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into());
418 if !node {
419 self.log.line("##[warning]The runner's Node does not run in this image (it needs glibc and libstdc++), so JavaScript actions run beside the container, on g1t's image, with the same files.");
420 }
421 self.job_context.insert("container".into(), json!({ "id": id, "network": network }));
422 self.container = Some(JobContainer { id: name, shell: if has_bash { "bash" } else { "sh" }, node, path });
423 }
424
425 // Services with a health check are waited for.
426 for (service, name) in self.services.clone() {
427 if !self.wait_healthy(&service, &name) {
428 return false;
429 }
430 }
431 if !services_context.is_empty() {
432 self.job_context.insert("services".into(), Value::Object(services_context));
433 }
434 self.log_docker_notes();
435 true
436 }
437
438 /// GitHub's runner's folders, at the same paths, and Docker's socket.
439 fn container_mounts(&self) -> Vec<(String, String)> {
440 let home = super::paths::under_home(super::paths::HOME_RUNNER);
441 let github_home = self.temp.join("_github_home");
442 let _ = std::fs::create_dir_all(&github_home);
443 let mut mounts: Vec<(String, String)> = ["work", "_temp", "_actions", "_tool"]
444 .iter()
445 .map(|dir| {
446 let path = home.join(dir);
447 let _ = std::fs::create_dir_all(&path);
448 (path.display().to_string(), path.display().to_string())
449 })
450 .collect();
451 mounts.push((github_home.display().to_string(), GITHUB_HOME.into()));
452 if let Some(node) = which_node() {
453 mounts.push((node.display().to_string(), format!("{CONTAINER_NODE}:ro")));
454 }
455 mounts.push((crate::docker::engine::DOCKER_SOCKET.into(), crate::docker::engine::DOCKER_SOCKET.into()));
456 mounts
457 }
458
459 fn wait_healthy(&mut self, service: &str, name: &str) -> bool {
460 let limit = Duration::from_secs(600).min(self.remaining_time());
461 let until = Instant::now() + limit;
462 let mut wait = Duration::from_secs(1);
463 loop {
464 let status = self.docker_output(&["inspect", "--format", "{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}", name]).unwrap_or_default();
465 match status.as_str() {
466 "" => return true,
467 "healthy" => {
468 self.log.line(&format!("{service} is healthy."));
469 return true;
470 }
471 "unhealthy" => {
472 self.log.line(&format!("##[error]The service `{service}` is unhealthy."));
473 self.service_logs(service, name);
474 return false;
475 }
476 _ => {}
477 }
478 if Instant::now() >= until {
479 self.log.line(&format!("##[error]The service `{service}` did not become healthy in {} s.", limit.as_secs()));
480 self.service_logs(service, name);
481 return false;
482 }
483 self.log.line(&format!("Waiting for {service} to be healthy ({status})."));
484 std::thread::sleep(wait);
485 wait = (wait * 2).min(Duration::from_secs(8));
486 }
487 }
488
489 fn service_logs(&mut self, service: &str, name: &str) {
490 self.log.line(&format!("##[group]Service container {service}"));
491 self.docker(&["logs".into(), "--tail".into(), "200".into(), name.into()], &BTreeMap::new(), Duration::from_secs(60));
492 self.log.line("##[endgroup]");
493 }
494
495 /// Whether the job has containers to stop when it ends.
496 pub(crate) fn has_containers(&self) -> bool {
497 self.network.is_some()
498 }
499
500 /// GitHub's "Stop containers": each service's log, then the job's
501 /// containers and network removed.
502 pub(crate) fn stop_containers(&mut self) -> bool {
503 for (service, name) in self.services.clone() {
504 self.service_logs(&service, &name);
505 }
506 let mut names: Vec<String> = self.services.iter().map(|(_, name)| name.clone()).collect();
507 if let Some(container) = &self.container {
508 names.push(container.id.clone());
509 }
510 if !names.is_empty() {
511 let mut args = vec!["rm".to_owned(), "--force".to_owned()];
512 args.extend(names);
513 self.docker(&args, &BTreeMap::new(), Duration::from_secs(120));
514 }
515 if let Some(network) = self.network.take() {
516 self.docker(&["network".into(), "rm".into(), network], &BTreeMap::new(), Duration::from_secs(60));
517 }
518 self.container = None;
519 true
520 }
521
522 /// The variables a process inside a container is given: the step's,
523 /// GitHub's and the job's, but not the sandbox's own (its `PATH`,
524 /// `HOME` and the like, which mean nothing in another image).
525 pub(crate) fn container_env(&self, step_env: &BTreeMap<String, String>, full: BTreeMap<String, String>, image_path: &str) -> BTreeMap<String, String> {
526 let mut out: BTreeMap<String, String> = full
527 .into_iter()
528 .filter(|(name, _)| step_env.contains_key(name) || !self.host_env.contains(name) || name.starts_with("GITHUB_") || name.starts_with("RUNNER_"))
529 .collect();
530 if !step_env.contains_key("PATH") {
531 out.remove("PATH");
532 if !self.path_prepend_entries().is_empty() {
533 out.insert("PATH".into(), format!("{}:{image_path}", self.path_prepend_entries().join(":")));
534 }
535 }
536 if !step_env.contains_key("HOME") {
537 out.remove("HOME");
538 }
539 out
540 }
541
542 /// A step's command, run inside the job's container instead.
543 pub(crate) fn in_container(&self, program: &str, args: &[String], dir: &Path, env: BTreeMap<String, String>) -> Option<Command> {
544 let container = self.container.as_ref()?;
545 let mut command = Command::new("docker");
546 command.args(["exec", "-w", &dir.display().to_string()]);
547 for name in env.keys() {
548 command.args(["-e", name]);
549 }
550 command.arg(&container.id).arg(program).args(args);
551 command.env_clear().envs(self.docker_cli_env()).envs(env);
552 Some(command)
553 }
554
555 /// Runs a container for a Docker action or a `docker://` step, as
556 /// GitHub's runner does: the workspace at /github/workspace, the step's
557 /// files at /github/file_commands, the job's network.
558 pub(crate) fn run_docker(&mut self, run: &DockerRun) -> (bool, BTreeMap<String, String>, BTreeMap<String, String>) {
559 let fail = (false, BTreeMap::new(), BTreeMap::new());
560 if !self.docker_ready() {
561 return fail;
562 }
563 let id = format!("{:x}", super::rand_id());
564 let Ok(files) = StepFiles::new(&self.temp, &id) else { return fail };
565 let commands_dir = self.temp.join("_runner_file_commands");
566 let workflow_dir = self.temp.join("_github_workflow");
567 let home_dir = self.temp.join("_github_home");
568 for dir in [&workflow_dir, &home_dir] {
569 let _ = std::fs::create_dir_all(dir);
570 }
571 let _ = std::fs::copy(self.temp.join("event.json"), workflow_dir.join("event.json"));
572
573 let full = self.process_env(&run.env, &files);
574 let mut env = self.container_env(&run.env, full, "");
575 env.remove("PATH");
576 // Paths as the container sees them.
577 let moved = |path: &Path| format!("{GITHUB_FILE_COMMANDS}/{}", path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default());
578 env.insert("GITHUB_OUTPUT".into(), moved(&files.output));
579 env.insert("GITHUB_ENV".into(), moved(&files.env));
580 env.insert("GITHUB_PATH".into(), moved(&files.path));
581 env.insert("GITHUB_STATE".into(), moved(&files.state));
582 env.insert("GITHUB_STEP_SUMMARY".into(), moved(&files.summary));
583 env.insert("GITHUB_WORKSPACE".into(), GITHUB_WORKSPACE.into());
584 env.insert("GITHUB_EVENT_PATH".into(), format!("{GITHUB_WORKFLOW}/event.json"));
585 env.insert("HOME".into(), GITHUB_HOME.into());
586 env.remove("GITHUB_ACTION_PATH");
587
588 let mut args: Vec<String> = vec!["run".into(), "--rm".into(), "--label".into(), "g1t-job".into(), "--workdir".into(), GITHUB_WORKSPACE.into()];
589 if let Some(network) = &self.network {
590 args.extend(["--network".into(), network.clone()]);
591 }
592 for (from, to) in [
593 (self.workspace.clone(), GITHUB_WORKSPACE),
594 (home_dir, GITHUB_HOME),
595 (workflow_dir, GITHUB_WORKFLOW),
596 (commands_dir, GITHUB_FILE_COMMANDS),
597 (PathBuf::from(crate::docker::engine::DOCKER_SOCKET), crate::docker::engine::DOCKER_SOCKET),
598 ] {
599 args.extend(["-v".into(), format!("{}:{to}", from.display())]);
600 }
601 for name in env.keys() {
602 args.extend(["-e".into(), name.clone()]);
603 }
604 if let Some(entrypoint) = &run.entrypoint {
605 args.extend(["--entrypoint".into(), entrypoint.clone()]);
606 }
607 args.push(run.image.clone());
608 args.extend(run.args.iter().cloned());
609
610 crate::abuse::touch();
611 if let Some(miner) = crate::abuse::miner_in(&shown(&args)) {
612 self.log.line(&format!("##[error]g1t does not run cryptocurrency miners ({miner}). This step was not run."));
613 return fail;
614 }
615 self.log.line(&format!("[command]docker {}", shown(&args)));
616 let mut command = Command::new("docker");
617 command.args(&args).env_clear().envs(self.docker_cli_env()).envs(&env);
618 let mut commands = Commands::default();
619 let ended = process::run(command, self.remaining_time(), &mut self.log, &mut commands);
620 self.log_docker_notes();
621 let ok = match ended {
622 Ok(Ended::Exited(0)) => true,
623 Ok(Ended::Exited(code)) => {
624 self.log.line(&format!("##[error]Docker run failed with exit code {code}."));
625 false
626 }
627 Ok(Ended::TimedOut) => {
628 self.log.line("##[error]The step ran past its time limit and was stopped.");
629 false
630 }
Merge Actions runs: summaries, attempts and re-runs, graceful cancel, log downloads, badges (actions 0009)631 Ok(Ended::Cancelled) => false,
Merge branch 'main' into actions-toolkit-oidc-artifacts632 Err(error) => {
633 self.log.line(&format!("##[error]docker could not be started: {error}"));
634 false
635 }
636 };
637 let (outputs, state) = self.absorb(&files, &commands);
638 (ok, outputs, state)
639 }
640
641 /// Builds a Docker action's image from its Dockerfile, once per job.
642 pub(crate) fn build_action_image(&mut self, dir: &Path, dockerfile: &str, tag_of: &str) -> Option<String> {
643 let tag = format!("g1t-action/{}", docker_name(tag_of));
644 if self.built_actions.contains(&tag) {
645 return Some(tag);
646 }
647 if !self.docker_ready() {
648 return None;
649 }
650 let file = dir.join(dockerfile);
651 let args = vec!["build".into(), "-t".into(), tag.clone(), "-f".into(), file.display().to_string(), dir.display().to_string()];
652 if !self.docker(&args, &BTreeMap::new(), Duration::from_secs(1800)) {
653 self.log.line("##[error]The action's image did not build.");
654 return None;
655 }
656 self.built_actions.insert(tag.clone());
657 Some(tag)
658 }
659
660 /// `docker/setup-buildx-action` on g1t's machines: the job's own
661 /// Engine is the builder (BuildKit, the `docker` driver, with the
662 /// containerd image store, so cache export and attestations work).
663 pub(crate) fn setup_buildx(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
664 if !self.docker_ready() {
665 return (false, BTreeMap::new());
666 }
667 if let Some(driver) = with.get("driver").filter(|d| !d.is_empty() && *d != "docker") {
668 self.log.line(&format!("`driver: {driver}` is not used on g1t's machines: builds run on this job's own Docker Engine (BuildKit, the `docker` driver), which keeps the sandbox's network and guardrails."));
669 }
670 for input in ["driver-opts", "buildkitd-flags", "buildkitd-config", "buildkitd-config-inline", "endpoint"] {
671 if with.get(input).is_some_and(|v| !v.trim().is_empty()) {
672 self.log.line(&format!("`{input}` is not used on g1t's machines."));
673 }
674 }
675 self.docker(&["buildx".into(), "version".into()], &BTreeMap::new(), Duration::from_secs(60));
676 let inspect = self.docker_output(&["buildx", "inspect", "default"]).unwrap_or_default();
677 let platforms = inspect
678 .lines()
679 .find_map(|line| line.trim().strip_prefix("Platforms:"))
680 .map(|p| p.split(',').map(|s| s.trim().trim_end_matches('*').to_owned()).filter(|s| !s.is_empty()).collect::<Vec<_>>().join(","))
681 .unwrap_or_else(|| "linux/amd64".into());
682 self.log.line(&format!("Builder: default (this job's Docker Engine), platforms {platforms}"));
683 let mut outputs = BTreeMap::new();
684 outputs.insert("name".into(), "default".into());
685 outputs.insert("driver".into(), "docker".into());
686 outputs.insert("platforms".into(), platforms.clone());
687 outputs.insert("endpoint".into(), "default".into());
688 outputs.insert("status".into(), "running".into());
689 outputs.insert("flags".into(), String::new());
690 outputs.insert(
691 "nodes".into(),
692 json!([{ "name": "default", "endpoint": "default", "status": "running", "platforms": platforms }]).to_string(),
693 );
694 (true, outputs)
695 }
696}
697
698/// The runner's Node, its real file (not a link), to mount into a job
699/// container.
700fn which_node() -> Option<PathBuf> {
701 let path = std::env::var_os("PATH")?;
702 std::env::split_paths(&path).map(|dir| dir.join("node")).find(|p| p.is_file()).and_then(|p| std::fs::canonicalize(p).ok())
703}
704
705/// The job context's `container` and `services`, merged with its status.
706pub(crate) fn job_context(status: &str, extra: &Map<String, Value>) -> Value {
707 let mut job = Map::new();
708 job.insert("status".into(), json!(status));
709 for (key, value) in extra {
710 job.insert(key.clone(), value.clone());
711 }
712 Value::Object(job)
713}
714
715/// Names a set of tags already built in this job.
716pub(crate) type Built = BTreeSet<String>;
717
718#[cfg(test)]
719mod tests {
720 use super::*;
721
722 #[test]
723 fn options_split_as_a_shell_would() {
724 assert_eq!(
725 split_words(r#"--health-cmd "pg_isready -U postgres" --health-interval 10s --health-retries=5"#),
726 vec!["--health-cmd", "pg_isready -U postgres", "--health-interval", "10s", "--health-retries=5"]
727 );
728 assert_eq!(split_words("--health-cmd 'redis-cli ping' --tmpfs /var/lib/x"), vec!["--health-cmd", "redis-cli ping", "--tmpfs", "/var/lib/x"]);
729 assert_eq!(split_words(r#"a\ b "c\"d" ''"#), vec!["a b", "c\"d", ""]);
730 assert!(split_words(" ").is_empty());
731 }
732
733 #[test]
734 fn services_are_read_from_either_form() {
735 assert_eq!(container_spec(&json!("redis:7")).image, "redis:7");
736 let spec = container_spec(&json!({
737 "image": "postgres:17",
738 "env": { "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "app" },
739 "ports": ["5432:5432", 6543],
740 "volumes": ["/data:/var/lib/postgresql/data"],
741 "options": "--health-cmd pg_isready --health-interval 10s",
742 "credentials": { "username": "me", "password": "token" },
743 }));
744 assert_eq!(spec.image, "postgres:17");
745 assert_eq!(spec.env["POSTGRES_DB"], "app");
746 assert_eq!(spec.ports, vec!["5432:5432", "6543"]);
747 assert_eq!(spec.options, vec!["--health-cmd", "pg_isready", "--health-interval", "10s"]);
748 assert_eq!(spec.credentials, Some(("me".into(), "token".into())));
749 // An empty image is a service the job leaves out, as on GitHub.
750 assert_eq!(container_spec(&json!({ "image": "" })).image, "");
751 }
752
753 #[test]
754 fn ports_map_container_to_host() {
755 let map = port_map(&["5432:5432".into(), "6543:5432/tcp".into(), "6379".into(), "127.0.0.1:8080:80".into(), ":9000".into()]);
756 assert_eq!(map["5432"], "6543");
757 assert_eq!(map["6379"], "6379");
758 assert_eq!(map["80"], "8080");
759 assert_eq!(map["9000"], "9000");
760 }
761
762 #[test]
763 fn credentials_sign_in_to_the_images_registry() {
764 assert_eq!(registry_of("ghcr.io/acme/db:1"), "ghcr.io");
765 assert_eq!(registry_of("g1t.sh/acme/web"), "g1t.sh");
766 assert_eq!(registry_of("localhost:5000/x"), "localhost:5000");
767 assert_eq!(registry_of("acme/private"), "https://index.docker.io/v1/");
768 assert_eq!(registry_of("postgres"), "https://index.docker.io/v1/");
769 }
770
771 #[test]
772 fn a_service_is_created_with_its_values_passed_by_name() {
773 let spec = container_spec(&json!({
774 "image": "postgres:17",
775 "env": { "POSTGRES_PASSWORD": "secret" },
776 "ports": ["5432:5432"],
777 "options": "--health-cmd pg_isready",
778 }));
779 let args = create_args("postgres_job", Some("g1t_job"), Some("postgres"), &spec, &[], false);
780 assert_eq!(
781 args,
782 vec![
783 "create", "--name", "postgres_job", "--label", "g1t-job", "--network", "g1t_job", "--network-alias", "postgres", "-p", "5432:5432", "-e",
784 "POSTGRES_PASSWORD", "--health-cmd", "pg_isready", "postgres:17"
785 ]
786 );
787 assert!(!args.iter().any(|a| a.contains("secret")));
788 let job = create_args("job_x", Some("g1t_job"), None, &container_spec(&json!("node:24")), &[("/home/runner/work".into(), "/home/runner/work".into())], true);
789 assert!(job.ends_with(&["--entrypoint".into(), "tail".into(), "node:24".into(), "-f".into(), "/dev/null".into()]));
790 assert!(job.contains(&"/home/runner/work:/home/runner/work".to_owned()));
791 }
792
793 #[test]
794 fn names_are_docker_names() {
795 assert_eq!(docker_name("Build & test_1a2b"), "build___test_1a2b");
796 assert_eq!(docker_name("docker/login-action@v3"), "docker_login-action_v3");
797 }
798
799 #[test]
800 fn the_job_context_carries_containers() {
801 let mut extra = Map::new();
802 extra.insert("services".into(), json!({ "db": { "ports": { "5432": "5432" } } }));
803 let job = job_context("success", &extra);
804 assert_eq!(job["status"], "success");
805 assert_eq!(job["services"]["db"]["ports"]["5432"], "5432");
806 }
807}

This file's history is long; its oldest lines are credited to the oldest commit read.