Skip to content
Sign up

flagon-io/g1t

public
0
429 linesCodeBlameRaw
1//! How the runner keeps work from the machine it is on, and what it checks
2//! before it takes any.
3//!
4//! The goal is two walls: the runner in a sandbox of its own, away from
5//! its machine, and each job in a fresh sandbox of its own inside that,
6//! made for the job and destroyed after. Where the runner stands today:
7//!
8//! | Isolation | How | Wall from the machine |
9//! | --- | --- | --- |
10//! | `nested-container` | The runner is a container under Sysbox (`docker run --runtime=sysbox-runc`), with its own Docker Engine inside; each job is a container on that engine. | Two: the job's container, then the runner's, whose root is no one on the machine. |
11//! | `container` | The runner drives a Docker Engine it shares with the machine (installed on it, or a container with the machine's Docker socket); each job is a container on it. | One: the job's container. The runner itself can do anything the machine's Docker can, which is everything. |
12//! | `host` | `--no-docker`: each job is a process on the machine, in a folder of its own. Deprecated. | None. |
13//!
14//! [`SandboxDriver`] is what makes a job's sandbox. [`NestedDocker`] (and
15//! [`SharedDocker`], the same on a shared engine) is built: exec.rs runs
16//! jobs through it. [`HyperVContainer`] and [`MacVm`] are the designs for
17//! Windows and macOS, declared so the shape is fixed, and not built.
18
19use std::path::Path;
20
21use anyhow::{Result, bail};
22
23/// How well a runner keeps work from its machine, as it tells g1t when it
24/// registers (`isolation`).
25#[derive(Clone, Copy, Debug, PartialEq, Eq)]
26pub enum Isolation {
27 /// A container per job, inside the runner's own sandboxed container.
28 NestedContainer,
29 /// A container per job, on a Docker Engine shared with the machine.
30 Container,
31 /// Directly on the machine.
32 Host,
33}
34
35impl Isolation {
36 pub fn as_str(self) -> &'static str {
37 match self {
38 Isolation::NestedContainer => "nested-container",
39 Isolation::Container => "container",
40 Isolation::Host => "host",
41 }
42 }
43
44 /// For the log at start.
45 pub fn describe(self) -> &'static str {
46 match self {
47 Isolation::NestedContainer => {
48 "nested-container: each job runs in a fresh container on this runner's own Docker Engine, inside the runner's sandboxed container; neither reaches the machine"
49 }
50 Isolation::Container => {
51 "container: each job runs in a fresh container, but the runner is not sandboxed from the machine (it shares the machine's Docker Engine, or its container's root is the machine's), so the runner, and anything that takes it over, controls the machine. Run the runner's image under Sysbox for nested-container isolation"
52 }
53 Isolation::Host => "host: jobs run directly on this machine, unsandboxed",
54 }
55 }
56}
57
58/// What `G1T_RUNNER_SANDBOX` asks for: `nested`, the runner's image's
59/// setting, starts a Docker Engine inside the runner's container.
60pub const SANDBOX_ENV: &str = "G1T_RUNNER_SANDBOX";
61
62/// The override for the checks below, loudly named: with it, a runner
63/// whose jobs can reach the machine starts anyway.
64pub const UNSAFE_ENV: &str = "G1T_RUNNER_UNSAFE_ALLOW_HOST_ACCESS";
65
66pub fn nested_requested() -> bool {
67 std::env::var(SANDBOX_ENV).is_ok_and(|v| v.eq_ignore_ascii_case("nested"))
68}
69
70pub fn unsafe_allowed() -> bool {
71 std::env::var(UNSAFE_ENV).is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes"))
72}
73
74// --- Host mode's deprecation ---------------------------------------------
75
76/// What the runner's log says when it starts in host mode.
77pub const HOST_MODE_AT_START: &str = "Deprecated: this runner runs jobs directly on this machine (--no-docker), unsandboxed. Every job can read, change and leave behind anything this user can. Host mode will be removed: run the runner's image under Sysbox instead, see https://docs.g1t.sh/guides/self-hosted-runners/#isolation";
78
79/// What every job's log says when it ran in host mode.
80pub const HOST_MODE_IN_JOB: &str = "This job ran on the host, unsandboxed: the self-hosted runner runs jobs directly on its machine (--no-docker). Host mode will be removed; see https://docs.g1t.sh/guides/self-hosted-runners/#isolation";
81
82/// Why a public repository's job is refused in host mode.
83pub fn host_mode_refuses(repo: &str) -> String {
84 format!(
85 "{repo} is public, and this self-hosted runner runs jobs directly on its machine (--no-docker), unsandboxed, so it does not take a public repository's work. Run the runner's image under Sysbox, or register it without --no-docker. See https://docs.g1t.sh/guides/self-hosted-runners/#isolation"
86 )
87}
88
89// --- What the runner's container can see ----------------------------------
90
91/// Socket names that are a container engine's: whoever has one controls
92/// that engine, and through it the machine it runs on.
93const ENGINE_SOCKETS: &[&str] = &["docker.sock", "containerd.sock", "podman.sock", "crio.sock", "dockershim.sock", "cri-dockerd.sock"];
94
95fn is_engine_socket(path: &str) -> bool {
96 let name = path.rsplit('/').next().unwrap_or(path);
97 ENGINE_SOCKETS.contains(&name)
98}
99
100/// mountinfo escapes spaces and the like as `\040`; undoes it.
101fn unescape(field: &str) -> String {
102 let bytes = field.as_bytes();
103 let mut out = Vec::with_capacity(bytes.len());
104 let mut i = 0;
105 while i < bytes.len() {
106 if bytes[i] == b'\\'
107 && let Some(code) = bytes.get(i + 1..i + 4).and_then(|d| std::str::from_utf8(d).ok()).and_then(|d| u8::from_str_radix(d, 8).ok())
108 {
109 out.push(code);
110 i += 4;
111 continue;
112 }
113 out.push(bytes[i]);
114 i += 1;
115 }
116 String::from_utf8_lossy(&out).into_owned()
117}
118
119/// The engine sockets mounted into this container from outside it, from
120/// `/proc/self/mountinfo`: a mount point, or the file a mount came from,
121/// named like an engine's socket (`-v /var/run/docker.sock:/var/run/docker.sock`,
122/// or the same socket under another name).
123pub fn mounted_engine_sockets(mountinfo: &str) -> Vec<String> {
124 let mut found = Vec::new();
125 for line in mountinfo.lines() {
126 // `id parent major:minor root mount-point options [optional…] - fstype source super-options`
127 let fields: Vec<&str> = line.split_whitespace().collect();
128 if fields.len() < 5 {
129 continue;
130 }
131 let (root, mount_point) = (unescape(fields[3]), unescape(fields[4]));
132 if is_engine_socket(&mount_point) || is_engine_socket(&root) {
133 found.push(mount_point);
134 }
135 }
136 found
137}
138
139/// Whether this container runs under Sysbox, whose virtual files
140/// (`/proc/uptime`, `/sys/kernel`, …) come from `sysboxfs`.
141pub fn under_sysbox(mountinfo: &str) -> bool {
142 mountinfo.lines().any(|line| line.split(" - ").nth(1).is_some_and(|rest| rest.split_whitespace().nth(1) == Some("sysboxfs")))
143}
144
145/// Whether root here is root on the machine: no user namespace, every id
146/// mapped to itself (`/proc/self/uid_map` is `0 0 4294967295`).
147pub fn root_is_the_machines(uid_map: &str) -> bool {
148 let rows: Vec<Vec<&str>> = uid_map.lines().map(|l| l.split_whitespace().collect()).filter(|r: &Vec<&str>| r.len() == 3).collect();
149 rows.iter().any(|r| r[0] == "0" && r[1] == "0" && r[2].parse::<u64>().is_ok_and(|n| n >= 4_294_967_295))
150}
151
152/// What a nested runner found about its container.
153#[derive(Debug, Default)]
154pub struct Findings {
155 /// Engine sockets mounted in from outside.
156 pub sockets: Vec<String>,
157 /// An engine socket that is just there, where the runner's own engine
158 /// does not put one: a directory such as `/var/run` mounted from the
159 /// machine.
160 pub stray_sockets: Vec<String>,
161 /// `DOCKER_HOST` pointing somewhere.
162 pub docker_host: Option<String>,
163 pub sysbox: bool,
164 pub machine_root: bool,
165}
166
167impl Findings {
168 pub fn read() -> Findings {
169 let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
170 let uid_map = std::fs::read_to_string("/proc/self/uid_map").unwrap_or_default();
171 let mut findings = Findings::from_text(&mountinfo, &uid_map);
172 for path in ["/var/run/docker.sock", "/run/docker.sock", "/run/containerd/containerd.sock", "/run/podman/podman.sock"] {
173 // `/var/run` is often `/run`: the same file is said once.
174 let same = |a: &str, b: &str| a == b || std::fs::canonicalize(a).ok().zip(std::fs::canonicalize(b).ok()).is_some_and(|(a, b)| a == b);
175 if Path::new(path).exists() && !findings.sockets.iter().any(|s| same(s, path)) {
176 findings.stray_sockets.push(path.to_owned());
177 }
178 }
179 findings.docker_host = std::env::var("DOCKER_HOST").ok().filter(|h| !h.is_empty());
180 findings
181 }
182
183 pub fn from_text(mountinfo: &str, uid_map: &str) -> Findings {
184 Findings {
185 sockets: mounted_engine_sockets(mountinfo),
186 sysbox: under_sysbox(mountinfo),
187 machine_root: root_is_the_machines(uid_map),
188 ..Findings::default()
189 }
190 }
191
192 /// What reaches the machine from here, one reason a line.
193 pub fn problems(&self) -> Vec<String> {
194 let mut problems = Vec::new();
195 for socket in &self.sockets {
196 problems.push(format!("{socket} is a container engine's socket mounted in from outside this container: whoever uses it controls the machine"));
197 }
198 for socket in &self.stray_sockets {
199 problems.push(format!("{socket} is a container engine's socket that this runner did not make (a folder mounted from the machine?)"));
200 }
201 if let Some(host) = &self.docker_host {
202 problems.push(format!("DOCKER_HOST is set ({host}): jobs would run on that engine, not this runner's own"));
203 }
204 if self.machine_root && !self.sysbox {
205 problems.push("root in this container is root on the machine (it has no user namespace of its own): run it with --runtime=sysbox-runc, and without --privileged".to_owned());
206 }
207 problems
208 }
209}
210
211/// Decides a nested runner's isolation, or refuses to start. With
212/// [`UNSAFE_ENV`] it starts anyway, as `container`, and says why loudly.
213pub fn check_nested(findings: &Findings, allow_unsafe: bool) -> Result<Isolation> {
214 let problems = findings.problems();
215 if problems.is_empty() {
216 return Ok(Isolation::NestedContainer);
217 }
218 if allow_unsafe {
219 return Ok(Isolation::Container);
220 }
221 bail!(
222 "this runner is not sandboxed from its machine, so it will not take work:\n - {}\nRun it as `docker run --runtime=sysbox-runc …` without the machine's Docker socket (see https://docs.g1t.sh/guides/self-hosted-runners/#run-it-as-a-container), or set {UNSAFE_ENV}=1 to run anyway with jobs able to reach the machine.",
223 problems.join("\n - ")
224 )
225}
226
227// --- Sandbox drivers --------------------------------------------------------
228
229/// What makes and destroys a job's sandbox. Each job gets a fresh one,
230/// which never touches the machine, and which is gone after the job.
231/// Making and destroying a Docker sandbox is exec.rs; a driver for another
232/// kind adds its own when it is built.
233pub trait SandboxDriver {
234 /// What it is called in the log.
235 fn name(&self) -> &'static str;
236 /// Whether this machine can make sandboxes with it: the isolation it
237 /// gives here, or why it cannot.
238 fn preflight(&self) -> Result<Isolation>;
239}
240
241/// Linux, built: the runner's own Docker Engine, inside the runner's
242/// Sysbox container (engine.rs), and a container per job on it (exec.rs),
243/// with no mounts from the machine. `allow_unsafe`: [`UNSAFE_ENV`] is set.
244pub struct NestedDocker {
245 pub allow_unsafe: bool,
246}
247
248impl SandboxDriver for NestedDocker {
249 fn name(&self) -> &'static str {
250 "Docker inside the runner's container"
251 }
252 fn preflight(&self) -> Result<Isolation> {
253 check_nested(&Findings::read(), self.allow_unsafe)
254 }
255}
256
257/// The same containers, on a Docker Engine shared with the machine: a
258/// runner installed on the machine, or a container given its socket.
259pub struct SharedDocker;
260
261impl SandboxDriver for SharedDocker {
262 fn name(&self) -> &'static str {
263 "the machine's Docker"
264 }
265 fn preflight(&self) -> Result<Isolation> {
266 Ok(Isolation::Container)
267 }
268}
269
270/// The driver for jobs native to this OS, once there is one: what a
271/// Windows or macOS runner in host mode says is coming.
272pub fn native_driver() -> Option<Box<dyn SandboxDriver>> {
273 match std::env::consts::OS {
274 "windows" => Some(Box::new(HyperVContainer)),
275 "macos" => Some(Box::new(MacVm)),
276 _ => None,
277 }
278}
279
280/// **Windows: not built.** The design:
281///
282/// - A minimal launcher on the host, the only part that runs there. It
283/// polls nothing and runs no job code: it creates a sandbox from a
284/// signed image when the runner inside asks for one, and destroys it
285/// when the job ends. Its whole job is "create from signed image" and
286/// "destroy", so there is little to attack.
287/// - Each job is a Windows container with Hyper-V isolation
288/// (`docker run --isolation=hyperv`, or the HCS API directly): its own
289/// utility VM and kernel, so a job that escapes its container is still
290/// in a VM, not on the host. Process isolation shares the host's kernel
291/// and is not enough.
292/// - The image is pinned by digest and its signature checked before it is
293/// used; the job gets the harness read-only, its own token, and a
294/// scratch disk destroyed with the VM.
295///
296/// It needs a Windows host with Hyper-V and the Containers feature, and
297/// cannot be built or tested on Linux. Until it is, Windows jobs run in
298/// host mode, which is deprecated.
299pub struct HyperVContainer;
300
301impl SandboxDriver for HyperVContainer {
302 fn name(&self) -> &'static str {
303 "Windows containers with Hyper-V isolation"
304 }
305 fn preflight(&self) -> Result<Isolation> {
306 bail!("Hyper-V-isolated Windows containers are not built yet; Windows jobs run on the host (deprecated)")
307 }
308}
309
310/// **macOS: not built.** The design:
311///
312/// - The same minimal host launcher as on Windows, using Apple's
313/// Virtualization framework (`VZVirtualMachine`) to boot an ephemeral
314/// macOS VM per job from a signed base image, cloned copy-on-write
315/// (APFS clones) so a fresh VM costs seconds, and deleted after.
316/// - Apple's licence allows at most two macOS VMs per Mac at once, so a
317/// Mac runs two jobs at a time at most, and the launcher refuses a
318/// third.
319/// - The runner and harness run inside the VM; the host keeps only the
320/// launcher and the signed images.
321///
322/// It needs Apple silicon and macOS 13 or later, and cannot be built or
323/// tested on Linux. Until it is, macOS jobs run in host mode, which is
324/// deprecated.
325pub struct MacVm;
326
327impl MacVm {
328 /// Apple's macOS licence: two virtual copies per Mac.
329 pub const MAX_VMS: u32 = 2;
330}
331
332impl SandboxDriver for MacVm {
333 fn name(&self) -> &'static str {
334 "ephemeral macOS VMs"
335 }
336 fn preflight(&self) -> Result<Isolation> {
337 bail!("ephemeral macOS VMs (at most {} per Mac) are not built yet; macOS jobs run on the host (deprecated)", MacVm::MAX_VMS)
338 }
339}
340
341#[cfg(test)]
342mod tests {
343 use super::*;
344
345 const PLAIN: &str = "\
346611 590 0:52 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A,upperdir=/var/lib/docker/overlay2/x/diff
347612 611 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
348620 611 254:1 /var/lib/docker/volumes/g1t-runner/_data /data rw,relatime - ext4 /dev/vda1 rw
349621 611 254:1 /var/lib/docker/containers/abc/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw
350";
351
352 const WITH_SOCKET: &str = "\
353611 590 0:52 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A
354622 611 0:26 /docker.sock /run/docker.sock rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,size=3280376k,mode=755
355";
356
357 const RENAMED_SOCKET: &str = "\
358623 611 0:26 /docker.sock /tmp/engine rw,nosuid,nodev - tmpfs tmpfs rw
359624 611 0:27 /containerd/containerd.sock /run/containerd/containerd.sock rw - tmpfs tmpfs rw
360";
361
362 const SYSBOX: &str = "\
3631000 990 0:60 / / rw,relatime - shiftfs /var/lib/sysbox/shiftfs/x rw
3641001 1000 0:61 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
3651002 1001 0:62 /proc/uptime /proc/uptime rw,nosuid,nodev,relatime - fuse sysboxfs rw,user_id=0,group_id=0,default_permissions,allow_other
3661003 1000 254:1 /var/lib/sysbox/docker/abc /var/lib/docker rw,relatime - ext4 /dev/vda1 rw
367";
368
369 #[test]
370 fn a_mounted_docker_socket_is_found() {
371 assert!(mounted_engine_sockets(PLAIN).is_empty());
372 assert_eq!(mounted_engine_sockets(WITH_SOCKET), vec!["/run/docker.sock".to_owned()]);
373 // The same socket under another name, and containerd's.
374 assert_eq!(mounted_engine_sockets(RENAMED_SOCKET), vec!["/tmp/engine".to_owned(), "/run/containerd/containerd.sock".to_owned()]);
375 // Sysbox's own Docker data mount is not a socket.
376 assert!(mounted_engine_sockets(SYSBOX).is_empty());
377 }
378
379 #[test]
380 fn escaped_paths_are_read() {
381 assert_eq!(mounted_engine_sockets("1 0 0:1 /a\\040b/docker.sock /x\\040y rw - tmpfs tmpfs rw"), vec!["/x y".to_owned()]);
382 }
383
384 #[test]
385 fn sysbox_and_a_user_namespace_are_told() {
386 assert!(under_sysbox(SYSBOX));
387 assert!(!under_sysbox(PLAIN));
388 assert!(root_is_the_machines(" 0 0 4294967295\n"));
389 assert!(!root_is_the_machines(" 0 165536 65536\n"));
390 }
391
392 #[test]
393 fn a_nested_runner_refuses_the_machines_socket_unless_told_loudly() {
394 let sysbox = Findings::from_text(SYSBOX, "0 165536 65536");
395 assert_eq!(check_nested(&sysbox, false).unwrap(), Isolation::NestedContainer);
396
397 let socket = Findings::from_text(&format!("{SYSBOX}{WITH_SOCKET}"), "0 165536 65536");
398 let refused = check_nested(&socket, false).unwrap_err().to_string();
399 assert!(refused.contains("/run/docker.sock"), "{refused}");
400 assert!(refused.contains(UNSAFE_ENV), "{refused}");
401 assert_eq!(check_nested(&socket, true).unwrap(), Isolation::Container);
402
403 // --privileged, without Sysbox: root here is the machine's.
404 let privileged = Findings::from_text(PLAIN, "0 0 4294967295");
405 assert!(check_nested(&privileged, false).unwrap_err().to_string().contains("sysbox-runc"));
406
407 let host = Findings { docker_host: Some("tcp://10.0.0.1:2375".into()), ..Findings::from_text(SYSBOX, "0 165536 65536") };
408 assert!(check_nested(&host, false).is_err());
409 }
410
411 #[test]
412 fn host_mode_says_it_is_deprecated() {
413 assert!(HOST_MODE_AT_START.starts_with("Deprecated:"));
414 assert!(HOST_MODE_AT_START.contains("will be removed"));
415 assert!(HOST_MODE_IN_JOB.starts_with("This job ran on the host, unsandboxed"));
416 assert!(HOST_MODE_IN_JOB.contains("Host mode will be removed"));
417 assert!(host_mode_refuses("acme/web").contains("acme/web is public"));
418 assert_eq!(Isolation::NestedContainer.as_str(), "nested-container");
419 assert_eq!(Isolation::Host.as_str(), "host");
420 }
421
422 #[test]
423 fn the_unbuilt_drivers_say_so() {
424 assert!(HyperVContainer.preflight().is_err());
425 assert!(MacVm.preflight().unwrap_err().to_string().contains("at most 2"));
426 assert_eq!(SharedDocker.preflight().unwrap(), Isolation::Container);
427 assert_eq!(MacVm::MAX_VMS, 2);
428 }
429}