Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
955 linesCodeBlameRaw
1//! Inside the sandbox's VM: `g1t-runner sandbox guest …`, which the
2//! launcher on the computer runs (through `wsl.exe --exec` on Windows, as
3//! the VM's systemd service on macOS and Linux).
4//!
5//! `guest run` is the VM's supervisor. It
6//!
7//! 1. on WSL, takes away what WSL puts in every distribution that reaches
8//! Windows: `/mnt/wsl` (shared by every distribution, where Docker
9//! Desktop keeps sockets to Windows' Docker), WSLg's display and sound
10//! sockets, Windows' GPU driver folders, and `/dev/dxg`; and checks
11//! `/etc/wsl.conf` is the image's, which turns off drives and Windows
12//! programs ([`WSL_CONF`]);
13//! 2. refuses to go on if anything of the computer's is still mounted, or
14//! a container engine's socket other than its own is reachable
15//! ([`problems`]);
16//! 3. starts Sysbox and a Docker Engine of the VM's own, and adds rules
17//! that keep every container off the computer's own address (the VM's
18//! gateway) but for DNS ([`firewall_rules`]);
19//! 4. runs the runner's image under `--runtime=sysbox-runc` with the
20//! registration on `/var/lib/g1t-sandbox/state`, and keeps saying how
21//! it is in `/var/lib/g1t-sandbox/status.json` (and on the console);
22//! 5. on `guest stop` (or SIGTERM), asks the runner to drain or stop,
23//! waits for it, and stops everything.
24//!
25//! Root in this VM is not a wall on WSL: every WSL distribution, the
26//! person's own among them, runs in one VM, and root in any of them can
27//! reach the others' disks. That is why the runner runs under Sysbox here
28//! and its jobs in containers inside it: a job is two walls from this
29//! VM's root, and three from the computer.
30
31use std::net::Ipv4Addr;
32use std::path::Path;
33
34use anyhow::{Context, Result, bail};
35use serde::{Deserialize, Serialize};
36use serde_json::{Value, json};
37
38use crate::selfhosted::RunnerStatus;
39
40pub const ROOT: &str = "/var/lib/g1t-sandbox";
41/// The runner's folder (`/data` in its container): its registration, and
42/// the status file it keeps.
43pub const STATE_DIR: &str = "/var/lib/g1t-sandbox/state";
44pub const SETTINGS: &str = "/var/lib/g1t-sandbox/settings.json";
45pub const REGISTER_ENV: &str = "/var/lib/g1t-sandbox/register.env";
46pub const STATUS: &str = "/var/lib/g1t-sandbox/status.json";
47/// `drain` or `now`: asks the supervisor to stop.
48pub const STOP_FILE: &str = "/var/lib/g1t-sandbox/stop";
49/// A runner image the launcher put here for the supervisor to load
50/// (`sandbox up --runner-image-file`, for a build that is not released).
51pub const IMAGE_TAR: &str = "/var/lib/g1t-sandbox/runner-image.tar";
52pub const RUN_DIR: &str = "/run/g1t-sandbox";
53/// The VM's Docker Engine: not `/var/run/docker.sock`.
54pub const DOCKER_SOCKET: &str = "/run/g1t-sandbox/docker.sock";
55pub const PID_FILE: &str = "/run/g1t-sandbox/supervisor.pid";
56pub const RUNNER_CONTAINER: &str = "g1t-runner";
57/// How a status line starts on a VM's console.
58pub const STATUS_PREFIX: &str = "G1T-SANDBOX-STATUS ";
59
60/// The image's `/etc/wsl.conf`.
61pub const WSL_CONF: &str = include_str!("../../../../../deploy/sandbox/wsl.conf");
62
63/// What WSL mounts into every distribution that reaches Windows or other
64/// distributions, taken away at start.
65pub const WSL_CHANNELS: &[&str] = &["/mnt/wsl", "/mnt/wslg", "/tmp/.X11-unix", "/usr/lib/wsl/drivers", "/usr/lib/wsl/lib"];
66
67/// What the launcher tells the guest (`guest configure`, or the seed disk).
68#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
69pub struct Settings {
70 /// The site to register with, and a registration token: only until the
71 /// runner has registered.
72 #[serde(default)]
73 pub url: Option<String>,
74 #[serde(default)]
75 pub token: Option<String>,
76 #[serde(default)]
77 pub name: Option<String>,
78 #[serde(default)]
79 pub labels: Vec<String>,
80 #[serde(default)]
81 pub group: Option<String>,
82 /// The computer's fingerprint (`machine_fingerprint`), which the
83 /// runner in the VM can't work out for itself.
84 #[serde(default)]
85 pub machine: Option<String>,
86 /// The runner's container's share of the VM, and so its jobs'.
87 #[serde(default)]
88 pub cpus: u32,
89 #[serde(default)]
90 pub memory_mib: u64,
91 /// The runner's image, by digest from the signed release.
92 #[serde(default)]
93 pub runner_image: String,
94 #[serde(default)]
95 pub image_version: String,
96 /// For a VM, how to stop when it is shut down: `drain` or `now`.
97 #[serde(default)]
98 pub stop: Option<String>,
99}
100
101/// What `guest status` says, and the launcher reports.
102#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
103pub struct GuestStatus {
104 /// `starting`, `running`, `stopping`, `stopped` or `error`.
105 pub state: String,
106 #[serde(default)]
107 pub runner: Option<RunnerStatus>,
108 #[serde(default)]
109 pub error: Option<String>,
110 #[serde(default)]
111 pub image_version: Option<String>,
112 #[serde(default)]
113 pub updated_at: String,
114}
115
116/// A mount that is the computer's, or WSL's way to it, from
117/// `/proc/self/mountinfo`.
118pub fn host_mounts(mountinfo: &str) -> Vec<String> {
119 let mut found = Vec::new();
120 for line in mountinfo.lines() {
121 let fields: Vec<&str> = line.split_whitespace().collect();
122 if fields.len() < 5 {
123 continue;
124 }
125 let point = fields[4];
126 let fstype = line.split(" - ").nth(1).and_then(|rest| rest.split_whitespace().next()).unwrap_or("");
127 let shared_fs = matches!(fstype, "9p" | "drvfs" | "virtiofs" | "vboxsf" | "prl_fs" | "vmhgfs" | "fuse.vmhgfs-fuse" | "smb3" | "cifs");
128 let channel = WSL_CHANNELS.iter().any(|c| point == *c || point.starts_with(&format!("{c}/")));
129 if shared_fs || channel || point.starts_with("/mnt/") {
130 found.push(format!("{point} ({fstype})"));
131 }
132 }
133 found
134}
135
136/// What keeps the VM from being a sandbox: mounts of the computer's, an
137/// engine socket that is not this VM's own, WSL's interop with Windows.
138pub fn problems(mountinfo: &str, sockets: &[String], interop: bool, wsl_conf: Option<&str>) -> Vec<String> {
139 let mut problems: Vec<String> = host_mounts(mountinfo).into_iter().map(|m| format!("{m} is mounted from the computer, or is WSL's way to it")).collect();
140 for socket in sockets {
141 problems.push(format!("{socket} is a container engine's socket that is not this sandbox's own"));
142 }
143 if interop {
144 problems.push("WSL interop is on: Windows programs can be started from inside the sandbox".to_owned());
145 }
146 if let Some(conf) = wsl_conf
147 && conf.trim() != WSL_CONF.trim()
148 {
149 problems.push("/etc/wsl.conf is not the sandbox image's, which turns off Windows drives and programs".to_owned());
150 }
151 problems
152}
153
154/// The VM's default gateway, which is the computer, from `/proc/net/route`
155/// (addresses in little-endian hex).
156pub fn gateway(route: &str) -> Option<Ipv4Addr> {
157 route.lines().skip(1).find_map(|line| {
158 let fields: Vec<&str> = line.split_whitespace().collect();
159 if fields.len() > 2 && fields[1] == "00000000" {
160 u32::from_str_radix(fields[2], 16).ok().map(|n| Ipv4Addr::from(n.swap_bytes())).filter(|a| !a.is_unspecified())
161 } else {
162 None
163 }
164 })
165}
166
167/// The sandbox's `/etc/resolv.conf`: the resolvers WSL offered, else the
168/// gateway, which answers DNS on WSL's NAT.
169pub fn resolv_conf(offered: &str, gateway: Option<Ipv4Addr>) -> String {
170 let mut servers = nameservers(offered);
171 if servers.is_empty() {
172 servers.extend(gateway);
173 }
174 servers.iter().map(|s| format!("nameserver {s}\n")).collect()
175}
176
177pub fn nameservers(resolv: &str) -> Vec<Ipv4Addr> {
178 resolv
179 .lines()
180 .filter_map(|l| l.trim().strip_prefix("nameserver"))
181 .filter_map(|a| a.trim().parse().ok())
182 .collect()
183}
184
185/// `iptables` arguments, in order, for the `DOCKER-USER` chain, which
186/// every container's traffic passes: DNS to the VM's resolvers, and
187/// nothing else to the computer's address (the gateway), to a resolver
188/// the computer answers on, or to link-local addresses. The internet
189/// stays reachable, as a job needs.
190pub fn firewall_rules(gateway: Option<Ipv4Addr>, resolvers: &[Ipv4Addr]) -> Vec<Vec<String>> {
191 let s = |items: &[&str]| items.iter().map(|i| i.to_string()).collect::<Vec<String>>();
192 let mut rules = Vec::new();
193 let mut walled: Vec<Ipv4Addr> = gateway.into_iter().collect();
194 for resolver in resolvers {
195 if resolver.is_loopback() {
196 continue;
197 }
198 let address = resolver.to_string();
199 for proto in ["udp", "tcp"] {
200 rules.push(s(&["-d", &address, "-p", proto, "--dport", "53", "-j", "ACCEPT"]));
201 }
202 // A resolver that is the computer, or WSL's DNS tunnel, takes DNS
203 // and nothing else.
204 if (resolver.is_private() || resolver.is_link_local()) && !walled.contains(resolver) {
205 walled.push(*resolver);
206 }
207 }
208 for address in walled {
209 rules.push(s(&["-d", &address.to_string(), "-j", "DROP"]));
210 }
211 rules.push(s(&["-d", "169.254.0.0/16", "-j", "DROP"]));
212 rules
213}
214
215/// The VM's Docker Engine: Sysbox as a runtime, its own socket, address
216/// ranges away from Docker's defaults (the runner's own engine, inside,
217/// uses those).
218pub fn daemon_json() -> Value {
219 json!({
220 "hosts": [format!("unix://{DOCKER_SOCKET}")],
221 "pidfile": format!("{RUN_DIR}/dockerd.pid"),
222 "runtimes": { "sysbox-runc": { "path": "/usr/local/bin/sysbox-runc" } },
223 "bip": "172.29.254.1/24",
224 "default-address-pools": [{ "base": "172.28.0.0/16", "size": 24 }],
225 "log-driver": "json-file",
226 "log-opts": { "max-size": "20m", "max-file": "2" },
227 })
228}
229
230/// The registration's variables, for `register-and-run` in the runner's
231/// image: a file, so the token is on no command line.
232pub fn register_env(settings: &Settings) -> Result<String> {
233 let (Some(url), Some(token)) = (&settings.url, &settings.token) else { bail!("the sandbox's runner is not registered, and no --url and --token were given") };
234 let mut lines = vec![format!("G1T_RUNNER_URL={url}"), format!("G1T_RUNNER_TOKEN={token}")];
235 if let Some(name) = &settings.name {
236 lines.push(format!("G1T_RUNNER_NAME={name}"));
237 }
238 if !settings.labels.is_empty() {
239 lines.push(format!("G1T_RUNNER_LABELS={}", settings.labels.join(",")));
240 }
241 if let Some(group) = &settings.group {
242 lines.push(format!("G1T_RUNNER_GROUP={group}"));
243 }
244 if let Some(machine) = &settings.machine {
245 lines.push(format!("G1T_RUNNER_MACHINE={machine}"));
246 }
247 if lines.iter().any(|l| l.contains('\n') || l.contains('\r')) {
248 bail!("a registration setting has a line break in it");
249 }
250 Ok(lines.join("\n") + "\n")
251}
252
253/// `docker run` for the runner in the VM.
254pub fn runner_run_args(settings: &Settings, register: bool) -> Vec<String> {
255 sysbox_run_args(settings, RUNNER_CONTAINER, STATE_DIR, register.then_some(REGISTER_ENV))
256}
257
258/// `docker run` for the runner: its image under Sysbox, `data` (a folder
259/// or a volume) as `/data`, no socket, no host network, its share of the
260/// machine, and no self-update (the launcher updates the whole sandbox).
261pub fn sysbox_run_args(settings: &Settings, name: &str, data: &str, env_file: Option<&str>) -> Vec<String> {
262 let mut args: Vec<String> = ["run", "-d", "--name", name, "--runtime=sysbox-runc", "--restart", "on-failure:5"]
263 .iter()
264 .map(|s| s.to_string())
265 .collect();
266 if settings.cpus > 0 {
267 args.extend(["--cpus".into(), settings.cpus.to_string()]);
268 }
269 if settings.memory_mib > 0 {
270 let memory = format!("{}m", settings.memory_mib);
271 args.extend(["--memory".into(), memory.clone(), "--memory-swap".into(), memory]);
272 }
273 args.extend(["-v".into(), format!("{data}:/data")]);
274 args.extend(["-e".into(), "G1T_RUNNER_NO_AUTO_UPDATE=1".into()]);
275 if let Some(file) = env_file {
276 args.extend(["--env-file".into(), file.into()]);
277 }
278 args.extend(["--label".into(), "sh.g1t.sandbox=runner".into()]);
279 args.push(settings.runner_image.clone());
280 args
281}
282
283/// Reads a seed disk: JSON, then zeros.
284pub fn read_seed(bytes: &[u8]) -> Result<Settings> {
285 let end = bytes.iter().position(|b| *b == 0).unwrap_or(bytes.len());
286 serde_json::from_slice(&bytes[..end]).context("the seed disk does not read")
287}
288
289/// A seed disk's bytes: the settings' JSON, padded to `size`.
290pub fn write_seed(settings: &Settings, size: usize) -> Result<Vec<u8>> {
291 let mut bytes = serde_json::to_vec(settings)?;
292 if bytes.len() >= size {
293 bail!("the sandbox's settings do not fit its seed disk");
294 }
295 bytes.resize(size, 0);
296 Ok(bytes)
297}
298
299/// The last status line on a VM's console.
300pub fn status_from_console(log: &str) -> Option<GuestStatus> {
301 log.lines().rev().find_map(|line| line.split_once(STATUS_PREFIX).and_then(|(_, json)| serde_json::from_str(json.trim()).ok()))
302}
303
304fn read_json<T: for<'a> Deserialize<'a>>(path: &str) -> Option<T> {
305 std::fs::read(path).ok().and_then(|bytes| serde_json::from_slice(&bytes).ok())
306}
307
308fn write_private(path: &str, bytes: &[u8]) -> Result<()> {
309 let temp = format!("{path}.new");
310 std::fs::write(&temp, bytes).with_context(|| format!("could not write {temp}"))?;
311 #[cfg(unix)]
312 {
313 use std::os::unix::fs::PermissionsExt;
314 std::fs::set_permissions(&temp, std::fs::Permissions::from_mode(0o600))?;
315 }
316 std::fs::rename(&temp, path).with_context(|| format!("could not write {path}"))?;
317 Ok(())
318}
319
320fn registered() -> bool {
321 Path::new(STATE_DIR).join("config.json").exists()
322}
323
324/// `g1t-runner sandbox guest <command>`.
325pub fn main(args: &[String]) -> Result<()> {
326 let command = args.first().map(String::as_str).unwrap_or("");
327 let value = |name: &str| args.windows(2).find(|w| w[0] == name).map(|w| w[1].clone());
328 match command {
329 "configure" => configure(&read_stdin()?),
330 "status" => {
331 println!("{}", serde_json::to_string(&current_status())?);
332 Ok(())
333 }
334 "export-state" => {
335 use std::io::Write;
336 let bytes = std::fs::read(Path::new(STATE_DIR).join("config.json")).context("the sandbox's runner is not registered")?;
337 std::io::stdout().write_all(&bytes)?;
338 Ok(())
339 }
340 "import-state" => {
341 std::fs::create_dir_all(STATE_DIR)?;
342 let bytes = read_stdin()?;
343 serde_json::from_slice::<Value>(&bytes).context("that is not a runner's configuration")?;
344 write_private(&format!("{STATE_DIR}/config.json"), &bytes)
345 }
346 "load-image" => {
347 std::fs::create_dir_all(ROOT)?;
348 let bytes = read_stdin()?;
349 write_private(IMAGE_TAR, &bytes)
350 }
351 "check" => {
352 let found = check();
353 if found.is_empty() {
354 println!("Nothing of the computer's reaches this sandbox.");
355 Ok(())
356 } else {
357 bail!("this sandbox can reach the computer:\n - {}", found.join("\n - "))
358 }
359 }
360 "stop" => stop(if args.iter().any(|a| a == "--now") { "now" } else { "drain" }, value("--wait").and_then(|w| w.parse().ok())),
361 "run" => run(value("--seed"), value("--state-disk"), value("--console")),
362 _ => bail!("guest takes configure, run, status, stop, check, export-state, import-state or load-image"),
363 }
364}
365
366fn read_stdin() -> Result<Vec<u8>> {
367 use std::io::Read;
368 let mut bytes = Vec::new();
369 std::io::stdin().read_to_end(&mut bytes)?;
370 // Windows' tools may put a byte-order mark before JSON.
371 if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
372 bytes.drain(..3);
373 }
374 Ok(bytes)
375}
376
377/// Keeps what the launcher sent, over what was there: a token only until
378/// the runner has registered.
379fn configure(bytes: &[u8]) -> Result<()> {
380 let given: Settings = serde_json::from_slice(bytes).context("the sandbox's settings do not read")?;
381 std::fs::create_dir_all(STATE_DIR)?;
382 let mut settings: Settings = read_json(SETTINGS).unwrap_or_default();
383 if given.url.is_some() {
384 settings.url = given.url;
385 }
386 if given.token.is_some() && !registered() {
387 settings.token = given.token;
388 }
389 if given.name.is_some() {
390 settings.name = given.name;
391 }
392 if !given.labels.is_empty() {
393 settings.labels = given.labels;
394 }
395 if given.group.is_some() {
396 settings.group = given.group;
397 }
398 if given.machine.is_some() {
399 settings.machine = given.machine;
400 }
401 if given.cpus > 0 {
402 settings.cpus = given.cpus;
403 }
404 if given.memory_mib > 0 {
405 settings.memory_mib = given.memory_mib;
406 }
407 if !given.runner_image.is_empty() {
408 settings.runner_image = given.runner_image;
409 }
410 if !given.image_version.is_empty() {
411 settings.image_version = given.image_version;
412 }
413 write_private(SETTINGS, &serde_json::to_vec_pretty(&settings)?)
414}
415
416/// The supervisor's status, or `stopped` when it is not running.
417fn current_status() -> GuestStatus {
418 let status: Option<GuestStatus> = read_json(STATUS);
419 let alive = supervisor_alive();
420 match status {
421 Some(status) if alive || status.state == "error" || status.state == "stopped" => status,
422 Some(status) => GuestStatus { state: "stopped".into(), ..status },
423 None => GuestStatus { state: "stopped".into(), ..GuestStatus::default() },
424 }
425}
426
427fn supervisor_alive() -> bool {
428 std::fs::read_to_string(PID_FILE)
429 .ok()
430 .and_then(|p| p.trim().parse::<u32>().ok())
431 .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists())
432}
433
434/// What reaches the computer from here, now.
435pub fn check() -> Vec<String> {
436 let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
437 let mut sockets = Vec::new();
438 // `/var/run` is `/run`.
439 for dir in ["/mnt", "/run", "/tmp"] {
440 find_sockets(Path::new(dir), 3, &mut sockets);
441 }
442 let sockets: Vec<String> = sockets.into_iter().filter(|s| !own_socket(s)).collect();
443 let on_wsl = on_wsl();
444 // With interop off, WSL starts no interop server for the distribution's
445 // processes, and `WSL_INTEROP` names none.
446 let interop = on_wsl && std::env::var_os("WSL_INTEROP").is_some();
447 let conf = on_wsl.then(|| std::fs::read_to_string("/etc/wsl.conf").unwrap_or_default());
448 problems(&mountinfo, &sockets, interop, conf.as_deref())
449}
450
451/// The sandbox's own engine's sockets, and the containerd and Sysbox it
452/// runs.
453pub fn own_socket(path: &str) -> bool {
454 [RUN_DIR, "/run/docker/", "/run/containerd/", "/run/sysbox/"].iter().any(|own| path.starts_with(own))
455}
456
457#[cfg_attr(not(unix), allow(clippy::only_used_in_recursion))]
458fn find_sockets(dir: &Path, depth: u32, found: &mut Vec<String>) {
459 let Ok(entries) = std::fs::read_dir(dir) else { return };
460 for entry in entries.flatten() {
461 let Ok(kind) = entry.file_type() else { continue };
462 let path = entry.path();
463 #[cfg(unix)]
464 {
465 use std::os::unix::fs::FileTypeExt;
466 if kind.is_socket() {
467 let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
468 if ["docker.sock", "containerd.sock", "podman.sock", "docker.proxy.sock", "backend.sock", "filesystem.sock"].contains(&name) {
469 found.push(path.display().to_string());
470 }
471 continue;
472 }
473 }
474 if kind.is_dir() && depth > 0 {
475 find_sockets(&path, depth - 1, found);
476 }
477 }
478}
479
480fn on_wsl() -> bool {
481 std::env::var_os("WSL_DISTRO_NAME").is_some() || Path::new("/proc/sys/fs/binfmt_misc/WSLInterop").exists() || Path::new("/run/WSL").exists()
482}
483
484/// Asks the supervisor to stop (`drain`: once the runner's job is done),
485/// and waits up to `wait` seconds for it, or forever.
486fn stop(mode: &str, wait: Option<u64>) -> Result<()> {
487 if !supervisor_alive() {
488 return Ok(());
489 }
490 std::fs::write(STOP_FILE, mode)?;
491 let until = wait.map(|w| std::time::Instant::now() + std::time::Duration::from_secs(w));
492 while supervisor_alive() {
493 if until.is_some_and(|u| std::time::Instant::now() > u) {
494 bail!("the sandbox is still stopping");
495 }
496 std::thread::sleep(std::time::Duration::from_millis(500));
497 }
498 Ok(())
499}
500
501#[cfg(not(target_os = "linux"))]
502fn run(_seed: Option<String>, _state_disk: Option<String>, _console: Option<String>) -> Result<()> {
503 bail!("the sandbox's supervisor runs inside its Linux VM")
504}
505
506#[cfg(target_os = "linux")]
507fn run(seed: Option<String>, state_disk: Option<String>, console: Option<String>) -> Result<()> {
508 supervisor::run(seed, state_disk, console)
509}
510
511#[cfg(target_os = "linux")]
512mod supervisor {
513 use std::path::Path;
514 use std::process::{Child, Command, Stdio};
515 use std::sync::atomic::Ordering;
516 use std::time::{Duration, Instant};
517
518 use anyhow::{Context, Result, bail};
519
520 use super::*;
521 use crate::selfhosted::{STOP, g1t_now};
522
523 extern "C" fn on_term(_: libc::c_int) {
524 STOP.store(true, Ordering::SeqCst);
525 }
526
527 fn now() -> String {
528 g1t_now(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0))
529 }
530
531 /// A clean environment: WSL's session points `XDG_RUNTIME_DIR` and
532 /// the display variables into /mnt/wslg, which is gone.
533 fn clean(command: &mut Command) -> &mut Command {
534 command.env_clear().env("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin").env("HOME", "/root")
535 }
536
537 fn docker() -> Command {
538 let mut command = Command::new("docker");
539 clean(&mut command).env("DOCKER_HOST", format!("unix://{DOCKER_SOCKET}"));
540 command
541 }
542
543 fn ok(command: &mut Command) -> bool {
544 command.stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())
545 }
546
547 struct Say {
548 console: Option<String>,
549 version: Option<String>,
550 }
551
552 impl Say {
553 fn status(&self, state: &str, runner: Option<RunnerStatus>, error: Option<String>) {
554 let status = GuestStatus { state: state.into(), runner, error, image_version: self.version.clone(), updated_at: now() };
555 let Ok(bytes) = serde_json::to_vec(&status) else { return };
556 let _ = write_private(STATUS, &bytes);
557 if let Some(console) = &self.console {
558 let _ = std::fs::OpenOptions::new()
559 .append(true)
560 .open(console)
561 .and_then(|mut c| std::io::Write::write_all(&mut c, format!("{STATUS_PREFIX}{}\n", String::from_utf8_lossy(&bytes)).as_bytes()));
562 }
563 }
564 }
565
566 fn daemon(program: &str, args: &[&str], log: &str) -> Result<Child> {
567 let out = std::fs::File::create(format!("{RUN_DIR}/{log}"))?;
568 let mut command = Command::new(program);
569 clean(&mut command)
570 .args(args)
571 .stdout(out.try_clone()?)
572 .stderr(out)
573 .stdin(Stdio::null())
574 .spawn()
575 .with_context(|| format!("could not start {program}"))
576 }
577
578 fn wait_for(what: &str, limit: Duration, mut ready: impl FnMut() -> bool) -> Result<()> {
579 let until = Instant::now() + limit;
580 while !ready() {
581 if Instant::now() > until {
582 bail!("{what} did not start");
583 }
584 std::thread::sleep(Duration::from_millis(250));
585 }
586 Ok(())
587 }
588
589 /// WSL's ways to Windows and to other distributions, taken away.
590 fn wsl_prepare() -> Result<()> {
591 // The resolvers WSL offers (its DNS tunnel, or the gateway), in a
592 // file of the sandbox's own: WSL's is in /mnt/wsl, unmounted next.
593 let offered = ["/mnt/wsl/resolv.conf", "/etc/resolv.conf"].iter().find_map(|p| std::fs::read_to_string(p).ok()).unwrap_or_default();
594 let route = std::fs::read_to_string("/proc/net/route").unwrap_or_default();
595 let _ = std::fs::remove_file("/etc/resolv.conf");
596 std::fs::write("/etc/resolv.conf", resolv_conf(&offered, gateway(&route)))?;
597 for point in WSL_CHANNELS {
598 let _ = Command::new("umount").args(["-R", "-l", point]).stdout(Stdio::null()).stderr(Stdio::null()).status();
599 }
600 // The GPU's paravirtual device: a way to Windows' driver.
601 let _ = std::fs::remove_file("/dev/dxg");
602 Ok(())
603 }
604
605 fn mount_state_disk(device: &str) -> Result<()> {
606 std::fs::create_dir_all(STATE_DIR)?;
607 let blank = !Command::new("blkid").arg(device).output().is_ok_and(|o| o.status.success() && !o.stdout.is_empty());
608 if blank && !ok(Command::new("mkfs.ext4").args(["-q", "-L", "g1t-state", device])) {
609 bail!("could not format the state disk {device}");
610 }
611 if !ok(Command::new("mount").args([device, STATE_DIR])) {
612 bail!("could not mount the state disk {device}");
613 }
614 Ok(())
615 }
616
617 fn read_seed_device(device: &str) -> Result<Settings> {
618 // The launcher may have rewritten it since boot: read past the cache.
619 let _ = Command::new("blockdev").args(["--flushbufs", device]).status();
620 let bytes = std::fs::read(device).with_context(|| format!("could not read the seed disk {device}"))?;
621 read_seed(&bytes)
622 }
623
624 fn firewall() -> Result<()> {
625 let route = std::fs::read_to_string("/proc/net/route").unwrap_or_default();
626 let resolv = std::fs::read_to_string("/etc/resolv.conf").unwrap_or_default();
627 let rules = firewall_rules(gateway(&route), &nameservers(&resolv));
628 let _ = Command::new("iptables").args(["-N", "DOCKER-USER"]).stderr(Stdio::null()).status();
629 // Only these rules, however many times the sandbox has started.
630 let _ = Command::new("iptables").args(["-F", "DOCKER-USER"]).status();
631 // Inserted last first, so they end up in order at the chain's top.
632 for (at, rule) in rules.iter().enumerate() {
633 let position = (at + 1).to_string();
634 let mut args = vec!["-I", "DOCKER-USER", position.as_str()];
635 args.extend(rule.iter().map(String::as_str));
636 if !ok(Command::new("iptables").args(&args)) {
637 bail!("could not add the rule that keeps containers off the computer: iptables {}", args.join(" "));
638 }
639 }
640 Ok(())
641 }
642
643 fn container_state() -> Option<(bool, i32)> {
644 let out = docker().args(["inspect", "-f", "{{.State.Running}} {{.State.ExitCode}}", RUNNER_CONTAINER]).output().ok()?;
645 if !out.status.success() {
646 return None;
647 }
648 let text = String::from_utf8_lossy(&out.stdout);
649 let mut parts = text.split_whitespace();
650 Some((parts.next() == Some("true"), parts.next().and_then(|c| c.parse().ok()).unwrap_or(0)))
651 }
652
653 /// The runner's last words: its last line of output.
654 fn logs_tail() -> String {
655 docker()
656 .args(["logs", "--tail", "1", RUNNER_CONTAINER])
657 .output()
658 .map(|o| format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)).trim().to_owned())
659 .unwrap_or_default()
660 .trim_start_matches("g1t-runner: ")
661 .to_owned()
662 }
663
664 fn start_runner(settings: &Settings) -> Result<()> {
665 if let Some((true, _)) = container_state() {
666 return Ok(());
667 }
668 let _ = docker().args(["rm", "-f", RUNNER_CONTAINER]).stdout(Stdio::null()).stderr(Stdio::null()).status();
669 // What an earlier runner said is not what this one says.
670 let _ = std::fs::remove_file(format!("{STATE_DIR}/status.json"));
671 let register = !registered();
672 if register {
673 write_private(REGISTER_ENV, register_env(settings)?.as_bytes())?;
674 }
675 let out = docker().args(runner_run_args(settings, register)).output().context("could not run docker")?;
676 if !out.status.success() {
677 bail!("could not start the runner: {}", String::from_utf8_lossy(&out.stderr).trim());
678 }
679 Ok(())
680 }
681
682 pub fn run(seed: Option<String>, state_disk: Option<String>, console: Option<String>) -> Result<()> {
683 unsafe {
684 libc::signal(libc::SIGTERM, on_term as *const () as libc::sighandler_t);
685 libc::signal(libc::SIGINT, on_term as *const () as libc::sighandler_t);
686 }
687 std::fs::create_dir_all(RUN_DIR)?;
688 std::fs::create_dir_all(ROOT)?;
689 std::fs::write(PID_FILE, std::process::id().to_string())?;
690 let _ = std::fs::remove_file(STOP_FILE);
691 let version = std::fs::read_to_string("/etc/g1t-sandbox-version").ok().map(|v| v.trim().to_owned());
692 let say = Say { console, version };
693 say.status("starting", None, None);
694 let outcome = supervise(&say, seed, state_disk);
695 if let Err(error) = &outcome {
696 say.status("error", None, Some(format!("{error:#}")));
697 }
698 let _ = std::fs::remove_file(PID_FILE);
699 outcome
700 }
701
702 fn supervise(say: &Say, seed: Option<String>, state_disk: Option<String>) -> Result<()> {
703 if on_wsl() {
704 wsl_prepare()?;
705 }
706 if let Some(device) = &state_disk {
707 // A VM's system disk was grown to the size asked for: its file
708 // system grows to fill it.
709 let _ = Command::new("resize2fs").arg("/dev/vda").stdout(Stdio::null()).stderr(Stdio::null()).status();
710 mount_state_disk(device)?;
711 }
712 std::fs::create_dir_all(STATE_DIR)?;
713 if let Some(device) = &seed {
714 let settings = read_seed_device(device)?;
715 configure(&serde_json::to_vec(&settings)?)?;
716 }
717 let found = check();
718 if !found.is_empty() {
719 bail!("this sandbox can reach the computer, so it will not run jobs:\n - {}", found.join("\n - "));
720 }
721 let settings: Settings = read_json(SETTINGS).context("the sandbox has no settings: run `g1t-runner sandbox up` on the computer")?;
722 if settings.runner_image.is_empty() {
723 bail!("the sandbox's settings name no runner image");
724 }
725
726 let mut daemons: Vec<Child> = Vec::new();
727 let _ = std::fs::remove_file("/run/sysbox/sysmgr.sock");
728 daemons.push(daemon("sysbox-mgr", &[], "sysbox-mgr.log")?);
729 wait_for("Sysbox's manager", Duration::from_secs(30), || Path::new("/run/sysbox/sysmgr.sock").exists())?;
730 daemons.push(daemon("sysbox-fs", &[], "sysbox-fs.log")?);
731 wait_for("Sysbox's file system", Duration::from_secs(30), || Path::new("/run/sysbox/sysfs.sock").exists())?;
732 std::fs::write("/etc/docker/daemon.json", serde_json::to_vec_pretty(&daemon_json())?)?;
733 daemons.push(daemon("dockerd", &[], "dockerd.log")?);
734 let outcome = (|| -> Result<()> {
735 wait_for("The sandbox's Docker Engine", Duration::from_secs(60), || ok(docker().args(["version"])))?;
736 firewall()?;
737 if Path::new(IMAGE_TAR).exists() {
738 let loaded = docker().args(["load", "-i", IMAGE_TAR]).stdout(Stdio::null()).status();
739 let _ = std::fs::remove_file(IMAGE_TAR);
740 if !loaded.is_ok_and(|s| s.success()) {
741 bail!("could not load the runner image the launcher gave");
742 }
743 }
744 if !ok(docker().args(["image", "inspect", &settings.runner_image])) && !ok(docker().args(["pull", "-q", &settings.runner_image])) {
745 bail!("could not fetch the runner's image {}", settings.runner_image);
746 }
747 start_runner(&settings)?;
748 watch(say)
749 })();
750 // Everything stops with the supervisor; the runner first, so it can
751 // say how its job ended.
752 let _ = docker().args(["stop", "-t", "60", RUNNER_CONTAINER]).stdout(Stdio::null()).stderr(Stdio::null()).status();
753 for child in daemons.iter_mut().rev() {
754 unsafe {
755 libc::kill(child.id() as libc::pid_t, libc::SIGTERM);
756 }
757 let until = Instant::now() + Duration::from_secs(20);
758 while matches!(child.try_wait(), Ok(None)) && Instant::now() < until {
759 std::thread::sleep(Duration::from_millis(200));
760 }
761 let _ = child.kill();
762 let _ = child.wait();
763 }
764 if outcome.is_ok() {
765 say.status("stopped", None, None);
766 }
767 outcome
768 }
769
770 /// Says how the runner is until asked to stop, then stops it.
771 fn watch(say: &Say) -> Result<()> {
772 let data = Path::new(STATE_DIR);
773 let mut asked: Option<String> = None;
774 let mut tried_again = Instant::now();
775 loop {
776 if asked.is_none() {
777 let mode = std::fs::read_to_string(STOP_FILE).ok().map(|m| m.trim().to_owned());
778 asked = mode.or_else(|| STOP.load(Ordering::SeqCst).then(|| read_json::<Settings>(SETTINGS).and_then(|s| s.stop).unwrap_or_else(|| "now".into())));
779 if let Some(mode) = &asked {
780 // The runner reads these in its folder (selfhosted/mod.rs).
781 std::fs::write(data.join(if mode == "drain" { "drain" } else { "stop" }), "")?;
782 }
783 }
784 if registered() && Path::new(REGISTER_ENV).exists() {
785 // Registered: the token is no longer kept.
786 let _ = std::fs::remove_file(REGISTER_ENV);
787 if let Some(mut settings) = read_json::<Settings>(SETTINGS) {
788 settings.token = None;
789 let _ = write_private(SETTINGS, &serde_json::to_vec_pretty(&settings)?);
790 }
791 }
792 let runner: Option<RunnerStatus> = read_json(&format!("{STATE_DIR}/status.json"));
793 match container_state() {
794 Some((true, _)) => {
795 let state = if asked.is_some() { "stopping" } else { "running" };
796 say.status(state, runner, None);
797 }
798 Some((false, code)) if asked.is_some() || code == 0 => {
799 let _ = std::fs::remove_file(data.join("stop"));
800 let _ = std::fs::remove_file(data.join("drain"));
801 return Ok(());
802 }
803 Some((false, code)) => {
804 say.status("error", runner, Some(format!("The runner stopped ({code}): {}", logs_tail())));
805 if tried_again.elapsed() > Duration::from_secs(30) {
806 tried_again = Instant::now();
807 let _ = docker().args(["start", RUNNER_CONTAINER]).stdout(Stdio::null()).stderr(Stdio::null()).status();
808 }
809 }
810 None => bail!("the runner's container is gone"),
811 }
812 std::thread::sleep(Duration::from_secs(2));
813 }
814 }
815}
816
817#[cfg(test)]
818mod tests {
819 use super::*;
820
821 const WSL_MOUNTINFO: &str = "\
822353 361 0:29 / /usr/lib/modules/6.6.87.2-microsoft-standard-WSL2 rw,nosuid,nodev,noatime - overlay none rw,lowerdir=/modules
823354 361 0:32 / /mnt/wsl rw,relatime shared:1 - tmpfs none rw
824355 354 0:61 /guest-services /mnt/wsl/docker-desktop/shared-sockets/guest-services rw,nosuid,nodev shared:18 - tmpfs none rw,mode=755
825358 361 0:34 / /usr/lib/wsl/drivers ro,nosuid,nodev,noatime - 9p drivers ro,aname=drivers;fmask=222;dmask=222
826361 343 8:96 / / rw,relatime - ext4 /dev/sdg rw,discard,errors=remount-ro,data=ordered
827362 361 0:87 / /mnt/wslg rw,relatime shared:32 - tmpfs none rw
828389 361 0:104 / /usr/lib/wsl/lib rw,nosuid,nodev,noatime - overlay none rw,lowerdir=/gpu_lib_packaged:/gpu_lib_inbox
829393 361 0:108 / /proc rw,nosuid,nodev,noexec,noatime shared:46 - proc proc rw
830411 361 0:87 /.X11-unix /tmp/.X11-unix ro,relatime shared:32 - tmpfs none rw
831";
832
833 const CLEAN: &str = "\
834361 343 8:96 / / rw,relatime - ext4 /dev/sdg rw,discard,errors=remount-ro,data=ordered
835393 361 0:108 / /proc rw,nosuid,nodev,noexec,noatime shared:46 - proc proc rw
836395 361 0:110 / /run rw,nosuid,nodev shared:48 - tmpfs none rw,mode=755
837";
838
839 #[test]
840 fn what_wsl_mounts_is_found_and_a_cleaned_distribution_is_clean() {
841 let found = host_mounts(WSL_MOUNTINFO);
842 for point in ["/mnt/wsl ", "/mnt/wsl/docker-desktop", "/usr/lib/wsl/drivers", "/mnt/wslg", "/usr/lib/wsl/lib", "/tmp/.X11-unix"] {
843 assert!(found.iter().any(|f| f.starts_with(point)), "{point} in {found:?}");
844 }
845 assert!(!found.iter().any(|f| f.starts_with("/usr/lib/modules")));
846 assert!(host_mounts(CLEAN).is_empty());
847 let drive = "500 361 0:90 / /mnt/c rw,noatime - 9p C:\\ rw,aname=drvfs;path=C:\\";
848 assert_eq!(host_mounts(drive), vec!["/mnt/c (9p)".to_owned()]);
849 assert_eq!(host_mounts("501 361 0:91 / /work rw - virtiofs share rw").len(), 1);
850 }
851
852 #[test]
853 fn the_sandbox_refuses_what_reaches_the_computer() {
854 assert!(problems(CLEAN, &[], false, Some(WSL_CONF)).is_empty());
855 assert!(problems(CLEAN, &[], false, None).is_empty());
856 let all = problems(WSL_MOUNTINFO, &["/mnt/wsl/docker-desktop/shared-sockets/host-services/docker.proxy.sock".into()], true, Some("[automount]\nenabled=true\n"));
857 assert!(all.iter().any(|p| p.contains("docker.proxy.sock")));
858 assert!(all.iter().any(|p| p.contains("interop is on")));
859 assert!(all.iter().any(|p| p.contains("/etc/wsl.conf")));
860 }
861
862 #[test]
863 fn the_image_turns_off_every_way_to_windows() {
864 let conf = WSL_CONF.replace(' ', "");
865 for setting in ["[automount]\nenabled=false", "mountFsTab=false", "[interop]\nenabled=false", "appendWindowsPath=false", "[user]\ndefault=sandbox"] {
866 assert!(conf.contains(setting), "{setting}");
867 }
868 // What opens the distribution without naming a user is not root;
869 // the launcher names root on every call (wsl.rs).
870 assert!(!conf.contains("default=root"));
871 let dockerfile = include_str!("../../../../../deploy/sandbox/Dockerfile");
872 assert!(dockerfile.contains("useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin sandbox"));
873 assert!(dockerfile.contains("passwd --lock sandbox"));
874 }
875
876 #[test]
877 fn the_gateway_and_resolvers_read() {
878 let route = "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n\
879eth0\t00000000\t01E01EAC\t0003\t0\t0\t0\t00000000\t0\t0\t0\n\
880eth0\t00E01EAC\t00000000\t0001\t0\t0\t0\t00F0FFFF\t0\t0\t0\n";
881 assert_eq!(gateway(route), Some(Ipv4Addr::new(172, 30, 224, 1)));
882 assert_eq!(gateway("Iface\tDestination\n"), None);
883 assert_eq!(nameservers("# x\nnameserver 10.255.255.254\nsearch lan\nnameserver ::1\n"), vec![Ipv4Addr::new(10, 255, 255, 254)]);
884 assert_eq!(resolv_conf("nameserver 10.255.255.254\n", None), "nameserver 10.255.255.254\n");
885 assert_eq!(resolv_conf("", Some(Ipv4Addr::new(172, 30, 224, 1))), "nameserver 172.30.224.1\n");
886 }
887
888 #[test]
889 fn containers_get_dns_and_nothing_else_of_the_computer() {
890 let gw = Ipv4Addr::new(172, 30, 224, 1);
891 let rules = firewall_rules(Some(gw), &[Ipv4Addr::new(10, 255, 255, 254), Ipv4Addr::new(1, 1, 1, 1), Ipv4Addr::LOCALHOST]);
892 let text: Vec<String> = rules.iter().map(|r| r.join(" ")).collect();
893 assert_eq!(text[0], "-d 10.255.255.254 -p udp --dport 53 -j ACCEPT");
894 assert!(text.contains(&"-d 1.1.1.1 -p tcp --dport 53 -j ACCEPT".to_owned()));
895 let first_drop = text.iter().position(|r| r.ends_with("DROP")).unwrap();
896 assert!(text[..first_drop].iter().all(|r| r.ends_with("ACCEPT")), "accepts come first: {text:?}");
897 assert!(text.contains(&"-d 172.30.224.1 -j DROP".to_owned()));
898 assert!(text.contains(&"-d 10.255.255.254 -j DROP".to_owned()));
899 assert!(!text.contains(&"-d 1.1.1.1 -j DROP".to_owned()), "a public resolver is the internet");
900 assert!(!text.iter().any(|r| r.contains("127.0.0.1")));
901 assert_eq!(text.last().unwrap(), "-d 169.254.0.0/16 -j DROP");
902 }
903
904 #[test]
905 fn the_runner_runs_under_sysbox_with_no_socket_and_no_self_update() {
906 let settings = Settings { cpus: 4, memory_mib: 8192, runner_image: "g1t.sh/flagon-io/g1t-runner@sha256:ab".into(), ..Settings::default() };
907 let args = runner_run_args(&settings, false).join(" ");
908 assert!(args.starts_with("run -d --name g1t-runner --runtime=sysbox-runc"));
909 assert!(args.contains("--cpus 4 --memory 8192m --memory-swap 8192m"));
910 assert!(args.contains("-v /var/lib/g1t-sandbox/state:/data"));
911 assert!(args.contains("G1T_RUNNER_NO_AUTO_UPDATE=1"));
912 assert!(!args.contains("docker.sock") && !args.contains("--privileged") && !args.contains("--network host"));
913 assert!(!args.contains("--env-file"));
914 assert!(args.ends_with("g1t.sh/flagon-io/g1t-runner@sha256:ab"));
915 assert!(runner_run_args(&settings, true).join(" ").contains("--env-file /var/lib/g1t-sandbox/register.env"));
916 }
917
918 #[test]
919 fn registration_goes_by_file_and_refuses_line_breaks() {
920 let mut settings = Settings {
921 url: Some("https://g1t.sh".into()),
922 token: Some("g1trt_x".into()),
923 name: Some("box-linux".into()),
924 labels: vec!["gpu".into()],
925 ..Settings::default()
926 };
927 let env = register_env(&settings).unwrap();
928 assert_eq!(env, "G1T_RUNNER_URL=https://g1t.sh\nG1T_RUNNER_TOKEN=g1trt_x\nG1T_RUNNER_NAME=box-linux\nG1T_RUNNER_LABELS=gpu\n");
929 // The computer's fingerprint, which the runner in the VM can't work out.
930 settings.machine = Some("ab".repeat(32));
931 assert!(register_env(&settings).unwrap().ends_with(&format!("G1T_RUNNER_MACHINE={}\n", "ab".repeat(32))));
932 settings.name = Some("a\nG1T_RUNNER_URL=https://evil".into());
933 assert!(register_env(&settings).is_err());
934 assert!(register_env(&Settings::default()).is_err());
935 }
936
937 #[test]
938 fn a_seed_disk_and_a_console_line_round_trip() {
939 let settings = Settings { runner_image: "img".into(), cpus: 2, stop: Some("drain".into()), ..Settings::default() };
940 let bytes = write_seed(&settings, 4096).unwrap();
941 assert_eq!(bytes.len(), 4096);
942 assert_eq!(read_seed(&bytes).unwrap(), settings);
943 assert!(write_seed(&settings, 8).is_err());
944 let log = format!("[ 1.0] booting\n{STATUS_PREFIX}{{\"state\":\"starting\"}}\nnoise\n{STATUS_PREFIX}{{\"state\":\"running\",\"updated_at\":\"t\"}}\n");
945 assert_eq!(status_from_console(&log).unwrap().state, "running");
946 assert!(status_from_console("nothing").is_none());
947 }
948
949 #[test]
950 fn the_vm_engine_has_sysbox_and_its_own_socket() {
951 let config = daemon_json();
952 assert_eq!(config["hosts"][0], "unix:///run/g1t-sandbox/docker.sock");
953 assert_eq!(config["runtimes"]["sysbox-runc"]["path"], "/usr/local/bin/sysbox-runc");
954 }
955}