Skip to content
Sign up

flagon-io/g1t

public
0
454 linesCodeBlameRaw
1//! macOS, and Linux without Sysbox: the sandbox is a small Linux VM booted
2//! from the release's signed kernel, initrd and system disk.
3//!
4//! - **macOS: vfkit**, a small command line program over Apple's
5//! Virtualization framework (`brew install vfkit`; the one Podman and
6//! CRC use). Chosen over Lima, which shares the home folder unless told
7//! otherwise and brings its own guest agent, and Apple's `container`,
8//! which needs macOS 26 and runs one VM per container rather than one
9//! for the runner. vfkit attaches only the devices named on its command
10//! line, so the sandbox gets disks, a NAT network card, a console and a
11//! random number source: no shared folder, no vsock, no display, no
12//! clipboard.
13//! - **Linux: QEMU with KVM**, the same VM: virtio-mmio devices on a
14//! `microvm` (x64) or `virt` (arm64) machine, user-mode networking with
15//! nothing forwarded in, and QEMU's seccomp sandbox on.
16//!
17//! The VM has three disks: its system (replaced by an update), the
18//! runner's state (kept, so an update does not register again; formatted
19//! by the guest the first time) and a seed the launcher writes, holding the
20//! settings (guest.rs). It says how it is on its console, which the
21//! launcher reads from a file; the launcher stops it through the
22//! hypervisor's control socket, a Unix socket in the state folder, never a
23//! network port.
24//!
25//! **Untested:** these command builders are tested here, but neither has
26//! been run on a Mac or a Linux machine with KVM yet.
27
28use std::path::{Path, PathBuf};
29use std::process::{Command, Stdio};
30use std::time::{Duration, Instant};
31
32use anyhow::{Context, Result, bail};
33
34use super::guest::{self, GuestStatus, Settings};
35use super::image::{self, Local};
36use super::{Backend, Resources, StopMode, process_alive};
37use crate::selfhosted::native::Availability;
38
39/// The seed disk's size: the settings' JSON, then zeros.
40pub const SEED_BYTES: usize = 1 << 20;
41/// The state disk's size: the runner's registration and status.
42pub const STATE_BYTES: u64 = 256 << 20;
43/// What the kernel is told: its console, and its system disk.
44pub const CMDLINE: &str = "console=hvc0 root=/dev/vda rw quiet";
45
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47pub enum Hypervisor {
48 Vfkit,
49 Qemu,
50}
51
52pub struct DiskVm {
53 pub hypervisor: Hypervisor,
54 pub dir: PathBuf,
55}
56
57/// The VM's files, in the state folder's `vm`.
58#[derive(Clone, Debug)]
59pub struct Paths {
60 pub kernel: PathBuf,
61 pub initrd: PathBuf,
62 pub system: PathBuf,
63 pub state: PathBuf,
64 pub seed: PathBuf,
65 pub console: PathBuf,
66 pub control: PathBuf,
67 pub pid: PathBuf,
68}
69
70impl Paths {
71 pub fn in_dir(vm: &Path) -> Paths {
72 Paths {
73 kernel: vm.join("kernel"),
74 initrd: vm.join("initrd"),
75 system: vm.join("system.raw"),
76 state: vm.join("state.img"),
77 seed: vm.join("seed.img"),
78 console: vm.join("console.log"),
79 control: vm.join("control.sock"),
80 pid: vm.join("vm.pid"),
81 }
82 }
83}
84
85/// vfkit's arguments: the VM's disks, NAT, its console to a file, and its
86/// control socket. Nothing shared with the Mac.
87pub fn vfkit_args(paths: &Paths, resources: &Resources) -> Vec<String> {
88 let p = |path: &Path| path.display().to_string();
89 vec![
90 "--cpus".into(),
91 resources.cpus.to_string(),
92 "--memory".into(),
93 resources.memory_mib.to_string(),
94 "--bootloader".into(),
95 format!("linux,kernel={},initrd={},cmdline=\"{CMDLINE}\"", p(&paths.kernel), p(&paths.initrd)),
96 "--device".into(),
97 format!("virtio-blk,path={}", p(&paths.system)),
98 "--device".into(),
99 format!("virtio-blk,path={}", p(&paths.state)),
100 "--device".into(),
101 format!("virtio-blk,path={}", p(&paths.seed)),
102 "--device".into(),
103 "virtio-net,nat".into(),
104 "--device".into(),
105 format!("virtio-serial,logFilePath={}", p(&paths.console)),
106 "--device".into(),
107 "virtio-rng".into(),
108 "--restful-uri".into(),
109 format!("unix://{}", p(&paths.control)),
110 ]
111}
112
113/// QEMU's arguments, for `qemu-system-x86_64` (`arch` x64) or
114/// `qemu-system-aarch64`: KVM, virtio-mmio devices, user networking with
115/// nothing forwarded in, the console to a file, QMP on a Unix socket, and
116/// QEMU's own seccomp sandbox.
117pub fn qemu_args(arch: &str, paths: &Paths, resources: &Resources) -> Vec<String> {
118 let p = |path: &Path| path.display().to_string();
119 let machine = if arch == "arm64" { "virt" } else { "microvm" };
120 let mut args: Vec<String> = vec![
121 "-machine".into(),
122 machine.into(),
123 "-accel".into(),
124 "kvm".into(),
125 "-cpu".into(),
126 "host".into(),
127 "-smp".into(),
128 resources.cpus.to_string(),
129 "-m".into(),
130 format!("{}M", resources.memory_mib),
131 "-nodefaults".into(),
132 "-no-user-config".into(),
133 "-nographic".into(),
134 "-kernel".into(),
135 p(&paths.kernel),
136 "-initrd".into(),
137 p(&paths.initrd),
138 "-append".into(),
139 CMDLINE.into(),
140 ];
141 for (id, path) in [("system", &paths.system), ("state", &paths.state), ("seed", &paths.seed)] {
142 args.extend(["-drive".into(), format!("id={id},file={},format=raw,if=none", p(path)), "-device".into(), format!("virtio-blk-device,drive={id}")]);
143 }
144 args.extend([
145 "-netdev".into(),
146 "user,id=net0".into(),
147 "-device".into(),
148 "virtio-net-device,netdev=net0".into(),
149 "-chardev".into(),
150 format!("file,id=console,path={}", p(&paths.console)),
151 "-device".into(),
152 "virtio-serial-device".into(),
153 "-device".into(),
154 "virtconsole,chardev=console".into(),
155 "-object".into(),
156 "rng-random,id=rng0,filename=/dev/urandom".into(),
157 "-device".into(),
158 "virtio-rng-device,rng=rng0".into(),
159 "-qmp".into(),
160 format!("unix:{},server=on,wait=off", p(&paths.control)),
161 "-pidfile".into(),
162 p(&paths.pid),
163 "-sandbox".into(),
164 "on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny".into(),
165 ]);
166 args
167}
168
169fn qemu_program() -> &'static str {
170 if image::arch() == "arm64" { "qemu-system-aarch64" } else { "qemu-system-x86_64" }
171}
172
173fn answers(program: &str) -> bool {
174 Command::new(program).arg("--version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())
175}
176
177/// What a request to stop the VM is, over its control socket: vfkit's
178/// REST call, or QMP's handshake and an ACPI power button.
179pub fn stop_request(hypervisor: Hypervisor) -> Vec<u8> {
180 match hypervisor {
181 Hypervisor::Vfkit => {
182 let body = r#"{"state":"Stop"}"#;
183 format!("POST /vm/state HTTP/1.1\r\nHost: vfkit\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).into_bytes()
184 }
185 Hypervisor::Qemu => b"{\"execute\":\"qmp_capabilities\"}\n{\"execute\":\"system_powerdown\"}\n".to_vec(),
186 }
187}
188
189impl DiskVm {
190 fn paths(&self) -> Paths {
191 Paths::in_dir(&self.dir.join("vm"))
192 }
193
194 fn pid(&self) -> Option<u32> {
195 std::fs::read_to_string(self.paths().pid).ok().and_then(|p| p.trim().parse().ok())
196 }
197
198 fn resources(&self) -> Resources {
199 std::fs::read(self.dir.join("vm").join("resources.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
200 }
201
202 #[cfg(unix)]
203 fn send_stop(&self) -> Result<()> {
204 use std::io::{Read, Write};
205 let mut socket = std::os::unix::net::UnixStream::connect(self.paths().control).context("could not reach the VM's control socket")?;
206 socket.set_read_timeout(Some(Duration::from_secs(5)))?;
207 socket.write_all(&stop_request(self.hypervisor))?;
208 let mut answer = [0u8; 512];
209 let _ = socket.read(&mut answer);
210 Ok(())
211 }
212
213 #[cfg(not(unix))]
214 fn send_stop(&self) -> Result<()> {
215 bail!("this VM runs on macOS and Linux")
216 }
217}
218
219impl Backend for DiskVm {
220 fn name(&self) -> &'static str {
221 match self.hypervisor {
222 Hypervisor::Vfkit => "vfkit",
223 Hypervisor::Qemu => "qemu",
224 }
225 }
226
227 fn image_keys(&self) -> Vec<String> {
228 ["kernel", "initrd", "disk"].iter().map(|part| format!("vm-{}-{part}", image::arch())).collect()
229 }
230
231 fn availability(&self) -> Availability {
232 match self.hypervisor {
233 Hypervisor::Vfkit if !answers("vfkit") => Availability::no(
234 "vfkit, which runs the sandbox's VM with Apple's Virtualization framework, is not installed.",
235 "Install it with `brew install vfkit` (macOS 13 or later), then start the sandbox again.",
236 ),
237 Hypervisor::Qemu if !Path::new("/dev/kvm").exists() => Availability::no(
238 "This machine has neither Sysbox nor KVM, so there is nothing to sandbox the runner with.",
239 "Install Sysbox 0.7 or later (https://github.com/nestybox/sysbox), or turn on virtualization and install QEMU so /dev/kvm exists.",
240 ),
241 Hypervisor::Qemu if std::fs::OpenOptions::new().read(true).write(true).open("/dev/kvm").is_err() => {
242 Availability::no("This user may not use /dev/kvm.", "Add this user to the kvm group, then log in again.")
243 }
244 Hypervisor::Qemu if !answers(qemu_program()) => Availability::no(
245 format!("{} is not installed.", qemu_program()),
246 "Install QEMU (qemu-system on Debian and Ubuntu, qemu-kvm on Fedora), or Sysbox 0.7 or later.",
247 ),
248 _ => Availability::yes(),
249 }
250 }
251
252 fn exists(&self) -> bool {
253 self.paths().system.exists()
254 }
255
256 fn running(&self) -> bool {
257 self.pid().is_some_and(process_alive)
258 }
259
260 fn create(&self, local: &Local, resources: &Resources) -> Result<()> {
261 let paths = self.paths();
262 std::fs::create_dir_all(self.dir.join("vm"))?;
263 let keys = self.image_keys();
264 let file = |part: usize| local.files.get(&keys[part]).with_context(|| format!("no {} image", keys[part]));
265 std::fs::copy(file(0)?, &paths.kernel)?;
266 std::fs::copy(file(1)?, &paths.initrd)?;
267 // The system disk comes compressed; it grows to the disk size asked
268 // for, and the guest grows its file system to fill it.
269 let mut source = flate2::read::GzDecoder::new(std::fs::File::open(file(2)?)?);
270 let mut system = std::fs::File::create(&paths.system)?;
271 std::io::copy(&mut source, &mut system).context("could not unpack the sandbox's system disk")?;
272 let wanted = resources.disk_gib << 30;
273 if system.metadata()?.len() < wanted {
274 system.set_len(wanted)?;
275 }
276 if !paths.state.exists() {
277 std::fs::File::create(&paths.state)?.set_len(STATE_BYTES)?;
278 }
279 std::fs::write(self.dir.join("vm").join("resources.json"), serde_json::to_vec(resources)?)?;
280 Ok(())
281 }
282
283 fn configure(&self, settings: &Settings) -> Result<()> {
284 let paths = self.paths();
285 let mut merged = std::fs::read(&paths.seed).ok().and_then(|b| guest::read_seed(&b).ok()).unwrap_or_default();
286 let given = settings.clone();
287 merged.url = given.url.or(merged.url);
288 merged.token = given.token.or(merged.token);
289 merged.name = given.name.or(merged.name);
290 if !given.labels.is_empty() {
291 merged.labels = given.labels;
292 }
293 merged.group = given.group.or(merged.group);
294 if given.cpus > 0 {
295 merged.cpus = given.cpus;
296 }
297 if given.memory_mib > 0 {
298 merged.memory_mib = given.memory_mib;
299 }
300 if !given.runner_image.is_empty() {
301 merged.runner_image = given.runner_image;
302 }
303 if !given.image_version.is_empty() {
304 merged.image_version = given.image_version;
305 }
306 merged.stop = given.stop.or(merged.stop);
307 super::write_private(&paths.seed, &guest::write_seed(&merged, SEED_BYTES)?)
308 }
309
310 fn load_image(&self, _tar: &Path) -> Result<()> {
311 bail!("--runner-image-file is for the WSL sandbox; a VM fetches the runner's image by digest")
312 }
313
314 fn start(&self) -> Result<()> {
315 if self.running() {
316 return Ok(());
317 }
318 let paths = self.paths();
319 let resources = self.resources();
320 let _ = std::fs::remove_file(&paths.control);
321 let _ = std::fs::remove_file(&paths.console);
322 let (program, args) = match self.hypervisor {
323 Hypervisor::Vfkit => ("vfkit", vfkit_args(&paths, &resources)),
324 Hypervisor::Qemu => (qemu_program(), qemu_args(image::arch(), &paths, &resources)),
325 };
326 let log = std::fs::File::create(self.dir.join("sandbox.log"))?;
327 let mut command = Command::new(program);
328 command.args(&args).stdin(Stdio::null()).stdout(log.try_clone()?).stderr(log);
329 let pid = super::spawn_detached(command)?;
330 std::fs::write(&paths.pid, pid.to_string())?;
331 Ok(())
332 }
333
334 fn guest_status(&self) -> Option<GuestStatus> {
335 let log = std::fs::read(self.paths().console).ok()?;
336 // The console's tail is enough: statuses come every few seconds.
337 let tail = &log[log.len().saturating_sub(64 * 1024)..];
338 guest::status_from_console(&String::from_utf8_lossy(tail))
339 }
340
341 fn stop(&self, mode: StopMode) -> Result<()> {
342 if !self.running() {
343 return Ok(());
344 }
345 self.configure(&Settings { stop: Some(if mode == StopMode::Drain { "drain" } else { "now" }.into()), ..Settings::default() })?;
346 self.send_stop()?;
347 let limit = (mode == StopMode::Now).then(|| Instant::now() + Duration::from_secs(150));
348 while self.running() {
349 if limit.is_some_and(|l| Instant::now() > l) {
350 if let Some(pid) = self.pid() {
351 let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status();
352 }
353 break;
354 }
355 std::thread::sleep(Duration::from_secs(1));
356 }
357 Ok(())
358 }
359
360 fn export_state(&self) -> Result<Option<Vec<u8>>> {
361 // The state disk is kept across updates.
362 Ok(None)
363 }
364
365 fn import_state(&self, _bytes: &[u8]) -> Result<()> {
366 Ok(())
367 }
368
369 fn remove_system(&self) -> Result<()> {
370 self.stop(StopMode::Now)?;
371 let paths = self.paths();
372 for path in [&paths.kernel, &paths.initrd, &paths.system, &paths.console] {
373 let _ = std::fs::remove_file(path);
374 }
375 Ok(())
376 }
377
378 fn remove(&self) -> Result<()> {
379 self.stop(StopMode::Now)?;
380 let _ = std::fs::remove_dir_all(self.dir.join("vm"));
381 Ok(())
382 }
383}
384
385#[cfg(test)]
386mod tests {
387 use super::*;
388
389 fn resources() -> Resources {
390 Resources { cpus: 4, memory_mib: 8192, disk_gib: 64 }
391 }
392
393 /// As `Paths::in_dir("/s/vm")` makes them on a Mac or Linux.
394 fn test_paths() -> Paths {
395 let p = |name: &str| PathBuf::from(format!("/s/vm/{name}"));
396 Paths {
397 kernel: p("kernel"),
398 initrd: p("initrd"),
399 system: p("system.raw"),
400 state: p("state.img"),
401 seed: p("seed.img"),
402 console: p("console.log"),
403 control: p("control.sock"),
404 pid: p("vm.pid"),
405 }
406 }
407
408 #[test]
409 fn vfkit_shares_nothing_with_the_mac() {
410 let paths = test_paths();
411 let args = vfkit_args(&paths, &resources());
412 let text = args.join(" ");
413 assert!(text.contains("--cpus 4 --memory 8192"));
414 assert!(text.contains("linux,kernel=/s/vm/kernel,initrd=/s/vm/initrd,cmdline=\"console=hvc0 root=/dev/vda rw quiet\""));
415 assert_eq!(args.iter().filter(|a| a.starts_with("virtio-blk,")).count(), 3);
416 assert!(text.contains("virtio-net,nat"));
417 assert!(text.contains("virtio-serial,logFilePath=/s/vm/console.log"));
418 assert!(text.contains("--restful-uri unix:///s/vm/control.sock"));
419 for shared in ["virtio-fs", "virtio-vsock", "virtio-gpu", "virtio-input", "rosetta", "tcp://"] {
420 assert!(!text.contains(shared), "{shared}");
421 }
422 }
423
424 #[test]
425 fn qemu_forwards_nothing_in_and_sandboxes_itself() {
426 let paths = test_paths();
427 let text = qemu_args("x64", &paths, &resources()).join(" ");
428 assert!(text.starts_with("-machine microvm -accel kvm -cpu host -smp 4 -m 8192M -nodefaults"));
429 assert!(text.contains("-netdev user,id=net0 "));
430 assert!(!text.contains("hostfwd") && !text.contains("-virtfs") && !text.contains("tcp:"));
431 assert!(text.contains("-qmp unix:/s/vm/control.sock,server=on,wait=off"));
432 assert!(text.contains("-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"));
433 assert_eq!(text.matches("virtio-blk-device").count(), 3);
434 assert!(qemu_args("arm64", &paths, &resources()).join(" ").starts_with("-machine virt "));
435 }
436
437 #[test]
438 fn stopping_is_a_power_button_not_a_pulled_plug() {
439 let vfkit = String::from_utf8(stop_request(Hypervisor::Vfkit)).unwrap();
440 assert!(vfkit.starts_with("POST /vm/state HTTP/1.1\r\n"));
441 assert!(vfkit.ends_with(r#"{"state":"Stop"}"#));
442 assert!(vfkit.contains("Content-Length: 16\r\n"));
443 let qmp = String::from_utf8(stop_request(Hypervisor::Qemu)).unwrap();
444 assert!(qmp.contains("qmp_capabilities") && qmp.contains("system_powerdown"));
445 }
446
447 #[test]
448 fn a_vm_needs_its_three_files() {
449 let vm = DiskVm { hypervisor: Hypervisor::Vfkit, dir: PathBuf::from("/s") };
450 let keys = vm.image_keys();
451 assert_eq!(keys.len(), 3);
452 assert!(keys[2].ends_with("-disk") && keys[0].starts_with("vm-"));
453 }
454}