| 1 | //! How the runner keeps work from the machine it is on, and what it checks |
| 2 | //! before it takes any. |
| 3 | //! |
| 4 | //! The goal is two walls: the runner in a sandbox of its own, away from |
| 5 | //! its machine, and each job in a fresh sandbox of its own inside that, |
| 6 | //! made for the job and destroyed after. Where the runner stands today: |
| 7 | //! |
| 8 | //! | Isolation | How | Wall from the machine | |
| 9 | //! | --- | --- | --- | |
| 10 | //! | `nested-container` | The runner is a container under Sysbox (`docker run --runtime=sysbox-runc`), with its own Docker Engine inside; each job is a container on that engine. On a desktop, `g1t-runner sandbox up` runs exactly this inside a Linux VM of the runner's own (launcher/). | Two: the job's container, then the runner's, whose root is no one on the machine; a third, the VM, under `sandbox up`. | |
| 11 | //! | `vm` | Jobs native to the machine's OS, each in a throwaway VM: Windows Sandbox on Windows, a cloned macOS VM on a Mac (native.rs). The runner stays on the machine and only makes and destroys those VMs. | One, a hypervisor: the job's VM, which gets no folder of the machine's but a read-only copy of the harness. | |
| 12 | //! | `container` | The runner drives a Docker Engine it shares with the machine (installed on it, or a container with the machine's Docker socket); each job is a container on it. | One: the job's container. The runner itself can do anything the machine's Docker can, which is everything. | |
| 13 | //! | `host` | `--no-docker`: each job is a process on the machine, in a folder of its own. Deprecated. | None. | |
| 14 | //! |
| 15 | //! [`SandboxDriver`] is what makes a job's sandbox. [`NestedDocker`] (and |
| 16 | //! [`SharedDocker`], the same on a shared engine) is built: exec.rs runs |
| 17 | //! jobs through it. For jobs native to the machine's OS, |
| 18 | //! [`super::native::WindowsSandbox`] is built and runs on Windows 11 Pro, |
| 19 | //! Enterprise and Education; [`super::native::MacVm`] is designed, its |
| 20 | //! commands built and tested, and not yet run on a Mac. |
| 21 | |
| 22 | use std::path::Path; |
| 23 | |
| 24 | use anyhow::{Result, bail}; |
| 25 | |
| 26 | /// How well a runner keeps work from its machine, as it tells g1t when it |
| 27 | /// registers (`isolation`). |
| 28 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] |
| 29 | pub enum Isolation { |
| 30 | /// A container per job, inside the runner's own sandboxed container. |
| 31 | NestedContainer, |
| 32 | /// A throwaway VM per job, native to the machine's OS (native.rs). |
| 33 | Vm, |
| 34 | /// A container per job, on a Docker Engine shared with the machine. |
| 35 | Container, |
| 36 | /// Directly on the machine. |
| 37 | Host, |
| 38 | } |
| 39 | |
| 40 | impl Isolation { |
| 41 | pub fn as_str(self) -> &'static str { |
| 42 | match self { |
| 43 | Isolation::NestedContainer => "nested-container", |
| 44 | Isolation::Vm => "vm", |
| 45 | Isolation::Container => "container", |
| 46 | Isolation::Host => "host", |
| 47 | } |
| 48 | } |
| 49 | |
| 50 | /// For the log at start. |
| 51 | pub fn describe(self) -> &'static str { |
| 52 | match self { |
| 53 | Isolation::NestedContainer => { |
| 54 | "nested-container: each job runs in a fresh container on this runner's own Docker Engine, inside the runner's sandboxed container; neither reaches the machine" |
| 55 | } |
| 56 | Isolation::Vm => { |
| 57 | "vm: each job runs in a throwaway virtual machine of its own, native to this OS, made for it and destroyed after; it gets no folder of this machine's but a read-only copy of the harness" |
| 58 | } |
| 59 | Isolation::Container => { |
| 60 | "container: each job runs in a fresh container, but the runner is not sandboxed from the machine (it shares the machine's Docker Engine, or its container's root is the machine's), so the runner, and anything that takes it over, controls the machine. Run the runner's image under Sysbox for nested-container isolation" |
| 61 | } |
| 62 | Isolation::Host => "host: jobs run directly on this machine, unsandboxed", |
| 63 | } |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | /// What `G1T_RUNNER_SANDBOX` asks for: `nested`, the runner's image's |
| 68 | /// setting, starts a Docker Engine inside the runner's container. |
| 69 | pub const SANDBOX_ENV: &str = "G1T_RUNNER_SANDBOX"; |
| 70 | |
| 71 | /// The override for the checks below, loudly named: with it, a runner |
| 72 | /// whose jobs can reach the machine starts anyway. |
| 73 | pub const UNSAFE_ENV: &str = "G1T_RUNNER_UNSAFE_ALLOW_HOST_ACCESS"; |
| 74 | |
| 75 | pub fn nested_requested() -> bool { |
| 76 | std::env::var(SANDBOX_ENV).is_ok_and(|v| v.eq_ignore_ascii_case("nested")) |
| 77 | } |
| 78 | |
| 79 | pub fn unsafe_allowed() -> bool { |
| 80 | std::env::var(UNSAFE_ENV).is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes")) |
| 81 | } |
| 82 | |
| 83 | // --- Host mode's deprecation --------------------------------------------- |
| 84 | |
| 85 | /// What the runner's log says when it starts in host mode. |
| 86 | pub const HOST_MODE_AT_START: &str = "Deprecated: this runner runs jobs directly on this machine (--no-docker), unsandboxed. Every job can read, change and leave behind anything this user can. Host mode will be removed: run the runner's image under Sysbox instead, see https://docs.g1t.sh/guides/self-hosted-runners/#isolation"; |
| 87 | |
| 88 | /// What every job's log says when it ran in host mode. |
| 89 | pub const HOST_MODE_IN_JOB: &str = "This job ran on the host, unsandboxed: the self-hosted runner runs jobs directly on its machine (--no-docker). Host mode will be removed; see https://docs.g1t.sh/guides/self-hosted-runners/#isolation"; |
| 90 | |
| 91 | /// Why a public repository's job is refused in host mode. |
| 92 | pub fn host_mode_refuses(repo: &str) -> String { |
| 93 | format!( |
| 94 | "{repo} is public, and this self-hosted runner runs jobs directly on its machine (--no-docker), unsandboxed, so it does not take a public repository's work. Run the runner's image under Sysbox, or register it without --no-docker. See https://docs.g1t.sh/guides/self-hosted-runners/#isolation" |
| 95 | ) |
| 96 | } |
| 97 | |
| 98 | // --- What the runner's container can see ---------------------------------- |
| 99 | |
| 100 | /// Socket names that are a container engine's: whoever has one controls |
| 101 | /// that engine, and through it the machine it runs on. |
| 102 | const ENGINE_SOCKETS: &[&str] = &["docker.sock", "containerd.sock", "podman.sock", "crio.sock", "dockershim.sock", "cri-dockerd.sock"]; |
| 103 | |
| 104 | fn is_engine_socket(path: &str) -> bool { |
| 105 | let name = path.rsplit('/').next().unwrap_or(path); |
| 106 | ENGINE_SOCKETS.contains(&name) |
| 107 | } |
| 108 | |
| 109 | /// mountinfo escapes spaces and the like as `\040`; undoes it. |
| 110 | fn unescape(field: &str) -> String { |
| 111 | let bytes = field.as_bytes(); |
| 112 | let mut out = Vec::with_capacity(bytes.len()); |
| 113 | let mut i = 0; |
| 114 | while i < bytes.len() { |
| 115 | if bytes[i] == b'\\' |
| 116 | && let Some(code) = bytes.get(i + 1..i + 4).and_then(|d| std::str::from_utf8(d).ok()).and_then(|d| u8::from_str_radix(d, 8).ok()) |
| 117 | { |
| 118 | out.push(code); |
| 119 | i += 4; |
| 120 | continue; |
| 121 | } |
| 122 | out.push(bytes[i]); |
| 123 | i += 1; |
| 124 | } |
| 125 | String::from_utf8_lossy(&out).into_owned() |
| 126 | } |
| 127 | |
| 128 | /// The engine sockets mounted into this container from outside it, from |
| 129 | /// `/proc/self/mountinfo`: a mount point, or the file a mount came from, |
| 130 | /// named like an engine's socket (`-v /var/run/docker.sock:/var/run/docker.sock`, |
| 131 | /// or the same socket under another name). |
| 132 | pub fn mounted_engine_sockets(mountinfo: &str) -> Vec<String> { |
| 133 | let mut found = Vec::new(); |
| 134 | for line in mountinfo.lines() { |
| 135 | // `id parent major:minor root mount-point options [optional…] - fstype source super-options` |
| 136 | let fields: Vec<&str> = line.split_whitespace().collect(); |
| 137 | if fields.len() < 5 { |
| 138 | continue; |
| 139 | } |
| 140 | let (root, mount_point) = (unescape(fields[3]), unescape(fields[4])); |
| 141 | if is_engine_socket(&mount_point) || is_engine_socket(&root) { |
| 142 | found.push(mount_point); |
| 143 | } |
| 144 | } |
| 145 | found |
| 146 | } |
| 147 | |
| 148 | /// Whether this container runs under Sysbox, whose virtual files |
| 149 | /// (`/proc/uptime`, `/sys/kernel`, …) come from `sysboxfs`. |
| 150 | pub fn under_sysbox(mountinfo: &str) -> bool { |
| 151 | mountinfo.lines().any(|line| line.split(" - ").nth(1).is_some_and(|rest| rest.split_whitespace().nth(1) == Some("sysboxfs"))) |
| 152 | } |
| 153 | |
| 154 | /// Whether root here is root on the machine: no user namespace, every id |
| 155 | /// mapped to itself (`/proc/self/uid_map` is `0 0 4294967295`). |
| 156 | pub fn root_is_the_machines(uid_map: &str) -> bool { |
| 157 | let rows: Vec<Vec<&str>> = uid_map.lines().map(|l| l.split_whitespace().collect()).filter(|r: &Vec<&str>| r.len() == 3).collect(); |
| 158 | rows.iter().any(|r| r[0] == "0" && r[1] == "0" && r[2].parse::<u64>().is_ok_and(|n| n >= 4_294_967_295)) |
| 159 | } |
| 160 | |
| 161 | /// What a nested runner found about its container. |
| 162 | #[derive(Debug, Default)] |
| 163 | pub struct Findings { |
| 164 | /// Engine sockets mounted in from outside. |
| 165 | pub sockets: Vec<String>, |
| 166 | /// An engine socket that is just there, where the runner's own engine |
| 167 | /// does not put one: a directory such as `/var/run` mounted from the |
| 168 | /// machine. |
| 169 | pub stray_sockets: Vec<String>, |
| 170 | /// `DOCKER_HOST` pointing somewhere. |
| 171 | pub docker_host: Option<String>, |
| 172 | pub sysbox: bool, |
| 173 | pub machine_root: bool, |
| 174 | } |
| 175 | |
| 176 | impl Findings { |
| 177 | pub fn read() -> Findings { |
| 178 | let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default(); |
| 179 | let uid_map = std::fs::read_to_string("/proc/self/uid_map").unwrap_or_default(); |
| 180 | let mut findings = Findings::from_text(&mountinfo, &uid_map); |
| 181 | for path in ["/var/run/docker.sock", "/run/docker.sock", "/run/containerd/containerd.sock", "/run/podman/podman.sock"] { |
| 182 | // `/var/run` is often `/run`: the same file is said once. |
| 183 | let same = |a: &str, b: &str| a == b || std::fs::canonicalize(a).ok().zip(std::fs::canonicalize(b).ok()).is_some_and(|(a, b)| a == b); |
| 184 | if Path::new(path).exists() && !findings.sockets.iter().any(|s| same(s, path)) { |
| 185 | findings.stray_sockets.push(path.to_owned()); |
| 186 | } |
| 187 | } |
| 188 | findings.docker_host = std::env::var("DOCKER_HOST").ok().filter(|h| !h.is_empty()); |
| 189 | findings |
| 190 | } |
| 191 | |
| 192 | pub fn from_text(mountinfo: &str, uid_map: &str) -> Findings { |
| 193 | Findings { |
| 194 | sockets: mounted_engine_sockets(mountinfo), |
| 195 | sysbox: under_sysbox(mountinfo), |
| 196 | machine_root: root_is_the_machines(uid_map), |
| 197 | ..Findings::default() |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | /// What reaches the machine from here, one reason a line. |
| 202 | pub fn problems(&self) -> Vec<String> { |
| 203 | let mut problems = Vec::new(); |
| 204 | for socket in &self.sockets { |
| 205 | problems.push(format!("{socket} is a container engine's socket mounted in from outside this container: whoever uses it controls the machine")); |
| 206 | } |
| 207 | for socket in &self.stray_sockets { |
| 208 | problems.push(format!("{socket} is a container engine's socket that this runner did not make (a folder mounted from the machine?)")); |
| 209 | } |
| 210 | if let Some(host) = &self.docker_host { |
| 211 | problems.push(format!("DOCKER_HOST is set ({host}): jobs would run on that engine, not this runner's own")); |
| 212 | } |
| 213 | if self.machine_root && !self.sysbox { |
| 214 | problems.push("root in this container is root on the machine (it has no user namespace of its own): run it with --runtime=sysbox-runc, and without --privileged".to_owned()); |
| 215 | } |
| 216 | problems |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | /// Decides a nested runner's isolation, or refuses to start. With |
| 221 | /// [`UNSAFE_ENV`] it starts anyway, as `container`, and says why loudly. |
| 222 | pub fn check_nested(findings: &Findings, allow_unsafe: bool) -> Result<Isolation> { |
| 223 | let problems = findings.problems(); |
| 224 | if problems.is_empty() { |
| 225 | return Ok(Isolation::NestedContainer); |
| 226 | } |
| 227 | if allow_unsafe { |
| 228 | return Ok(Isolation::Container); |
| 229 | } |
| 230 | bail!( |
| 231 | "this runner is not sandboxed from its machine, so it will not take work:\n - {}\nRun it as `docker run --runtime=sysbox-runc …` without the machine's Docker socket (see https://docs.g1t.sh/guides/self-hosted-runners/#run-it-as-a-container), or set {UNSAFE_ENV}=1 to run anyway with jobs able to reach the machine.", |
| 232 | problems.join("\n - ") |
| 233 | ) |
| 234 | } |
| 235 | |
| 236 | // --- Sandbox drivers -------------------------------------------------------- |
| 237 | |
| 238 | /// What makes and destroys a job's sandbox. Each job gets a fresh one, |
| 239 | /// which never touches the machine, and which is gone after the job. |
| 240 | /// Making and destroying a Docker sandbox is exec.rs; a driver for another |
| 241 | /// kind adds its own when it is built. |
| 242 | pub trait SandboxDriver { |
| 243 | /// What it is called in the log. |
| 244 | fn name(&self) -> &'static str; |
| 245 | /// Whether this machine can make sandboxes with it: the isolation it |
| 246 | /// gives here, or why it cannot. |
| 247 | fn preflight(&self) -> Result<Isolation>; |
| 248 | } |
| 249 | |
| 250 | /// Linux, built: the runner's own Docker Engine, inside the runner's |
| 251 | /// Sysbox container (engine.rs), and a container per job on it (exec.rs), |
| 252 | /// with no mounts from the machine. `allow_unsafe`: [`UNSAFE_ENV`] is set. |
| 253 | pub struct NestedDocker { |
| 254 | pub allow_unsafe: bool, |
| 255 | } |
| 256 | |
| 257 | impl SandboxDriver for NestedDocker { |
| 258 | fn name(&self) -> &'static str { |
| 259 | "Docker inside the runner's container" |
| 260 | } |
| 261 | fn preflight(&self) -> Result<Isolation> { |
| 262 | check_nested(&Findings::read(), self.allow_unsafe) |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | /// The same containers, on a Docker Engine shared with the machine: a |
| 267 | /// runner installed on the machine, or a container given its socket. |
| 268 | pub struct SharedDocker; |
| 269 | |
| 270 | impl SandboxDriver for SharedDocker { |
| 271 | fn name(&self) -> &'static str { |
| 272 | "the machine's Docker" |
| 273 | } |
| 274 | fn preflight(&self) -> Result<Isolation> { |
| 275 | Ok(Isolation::Container) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | /// The driver for jobs native to this OS, where there is one: Windows |
| 280 | /// Sandbox on Windows, a cloned macOS VM on a Mac (native.rs). |
| 281 | pub fn native_driver() -> Option<Box<dyn SandboxDriver>> { |
| 282 | match std::env::consts::OS { |
| 283 | "windows" => Some(Box::new(super::native::WindowsSandbox)), |
| 284 | "macos" => Some(Box::new(super::native::MacVm)), |
| 285 | _ => None, |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | #[cfg(test)] |
| 290 | mod tests { |
| 291 | use super::*; |
| 292 | |
| 293 | const PLAIN: &str = "\ |
| 294 | 611 590 0:52 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A,upperdir=/var/lib/docker/overlay2/x/diff |
| 295 | 612 611 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw |
| 296 | 620 611 254:1 /var/lib/docker/volumes/g1t-runner/_data /data rw,relatime - ext4 /dev/vda1 rw |
| 297 | 621 611 254:1 /var/lib/docker/containers/abc/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw |
| 298 | "; |
| 299 | |
| 300 | const WITH_SOCKET: &str = "\ |
| 301 | 611 590 0:52 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/A |
| 302 | 622 611 0:26 /docker.sock /run/docker.sock rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,size=3280376k,mode=755 |
| 303 | "; |
| 304 | |
| 305 | const RENAMED_SOCKET: &str = "\ |
| 306 | 623 611 0:26 /docker.sock /tmp/engine rw,nosuid,nodev - tmpfs tmpfs rw |
| 307 | 624 611 0:27 /containerd/containerd.sock /run/containerd/containerd.sock rw - tmpfs tmpfs rw |
| 308 | "; |
| 309 | |
| 310 | const SYSBOX: &str = "\ |
| 311 | 1000 990 0:60 / / rw,relatime - shiftfs /var/lib/sysbox/shiftfs/x rw |
| 312 | 1001 1000 0:61 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw |
| 313 | 1002 1001 0:62 /proc/uptime /proc/uptime rw,nosuid,nodev,relatime - fuse sysboxfs rw,user_id=0,group_id=0,default_permissions,allow_other |
| 314 | 1003 1000 254:1 /var/lib/sysbox/docker/abc /var/lib/docker rw,relatime - ext4 /dev/vda1 rw |
| 315 | "; |
| 316 | |
| 317 | #[test] |
| 318 | fn a_mounted_docker_socket_is_found() { |
| 319 | assert!(mounted_engine_sockets(PLAIN).is_empty()); |
| 320 | assert_eq!(mounted_engine_sockets(WITH_SOCKET), vec!["/run/docker.sock".to_owned()]); |
| 321 | // The same socket under another name, and containerd's. |
| 322 | assert_eq!(mounted_engine_sockets(RENAMED_SOCKET), vec!["/tmp/engine".to_owned(), "/run/containerd/containerd.sock".to_owned()]); |
| 323 | // Sysbox's own Docker data mount is not a socket. |
| 324 | assert!(mounted_engine_sockets(SYSBOX).is_empty()); |
| 325 | } |
| 326 | |
| 327 | #[test] |
| 328 | fn escaped_paths_are_read() { |
| 329 | assert_eq!(mounted_engine_sockets("1 0 0:1 /a\\040b/docker.sock /x\\040y rw - tmpfs tmpfs rw"), vec!["/x y".to_owned()]); |
| 330 | } |
| 331 | |
| 332 | #[test] |
| 333 | fn sysbox_and_a_user_namespace_are_told() { |
| 334 | assert!(under_sysbox(SYSBOX)); |
| 335 | assert!(!under_sysbox(PLAIN)); |
| 336 | assert!(root_is_the_machines(" 0 0 4294967295\n")); |
| 337 | assert!(!root_is_the_machines(" 0 165536 65536\n")); |
| 338 | } |
| 339 | |
| 340 | #[test] |
| 341 | fn a_nested_runner_refuses_the_machines_socket_unless_told_loudly() { |
| 342 | let sysbox = Findings::from_text(SYSBOX, "0 165536 65536"); |
| 343 | assert_eq!(check_nested(&sysbox, false).unwrap(), Isolation::NestedContainer); |
| 344 | |
| 345 | let socket = Findings::from_text(&format!("{SYSBOX}{WITH_SOCKET}"), "0 165536 65536"); |
| 346 | let refused = check_nested(&socket, false).unwrap_err().to_string(); |
| 347 | assert!(refused.contains("/run/docker.sock"), "{refused}"); |
| 348 | assert!(refused.contains(UNSAFE_ENV), "{refused}"); |
| 349 | assert_eq!(check_nested(&socket, true).unwrap(), Isolation::Container); |
| 350 | |
| 351 | // --privileged, without Sysbox: root here is the machine's. |
| 352 | let privileged = Findings::from_text(PLAIN, "0 0 4294967295"); |
| 353 | assert!(check_nested(&privileged, false).unwrap_err().to_string().contains("sysbox-runc")); |
| 354 | |
| 355 | let host = Findings { docker_host: Some("tcp://10.0.0.1:2375".into()), ..Findings::from_text(SYSBOX, "0 165536 65536") }; |
| 356 | assert!(check_nested(&host, false).is_err()); |
| 357 | } |
| 358 | |
| 359 | #[test] |
| 360 | fn host_mode_says_it_is_deprecated() { |
| 361 | assert!(HOST_MODE_AT_START.starts_with("Deprecated:")); |
| 362 | assert!(HOST_MODE_AT_START.contains("will be removed")); |
| 363 | assert!(HOST_MODE_IN_JOB.starts_with("This job ran on the host, unsandboxed")); |
| 364 | assert!(HOST_MODE_IN_JOB.contains("Host mode will be removed")); |
| 365 | assert!(host_mode_refuses("acme/web").contains("acme/web is public")); |
| 366 | assert_eq!(Isolation::NestedContainer.as_str(), "nested-container"); |
| 367 | assert_eq!(Isolation::Host.as_str(), "host"); |
| 368 | } |
| 369 | |
| 370 | #[test] |
| 371 | fn a_shared_engine_is_one_wall_and_a_vm_says_so() { |
| 372 | assert_eq!(SharedDocker.preflight().unwrap(), Isolation::Container); |
| 373 | assert_eq!(Isolation::Vm.as_str(), "vm"); |
| 374 | assert!(Isolation::Vm.describe().contains("throwaway virtual machine")); |
| 375 | } |
| 376 | } |