Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
506 linesCodeBlameRaw
1//! macOS, and Linux without Sysbox: the sandbox is a small Linux VM booted
2//! from the release's signed kernel, initrd and system disk.
3//!
4//! - **macOS: vfkit**, a small command line program over Apple's
5//! Virtualization framework (`brew install vfkit`; the one Podman and
6//! CRC use). Chosen over Lima, which shares the home folder unless told
7//! otherwise and brings its own guest agent, and Apple's `container`,
8//! which needs macOS 26 and runs one VM per container rather than one
9//! for the runner. vfkit attaches only the devices named on its command
10//! line, so the sandbox gets disks, a NAT network card, a console and a
11//! random number source: no shared folder, no vsock, no display, no
12//! clipboard.
13//! - **Linux: QEMU with KVM**, the same VM: virtio-mmio devices on a
14//! `microvm` (x64) or `virt` (arm64) machine, user-mode networking with
15//! nothing forwarded in, and QEMU's seccomp sandbox on.
16//!
17//! The VM has three disks: its system (replaced by an update), the
18//! runner's state (kept, so an update does not register again; formatted
19//! by the guest the first time) and a seed the launcher writes, holding the
20//! settings (guest.rs). It says how it is on its console, which the
21//! launcher reads from a file; the launcher stops it through the
22//! hypervisor's control socket, a Unix socket in the state folder, never a
23//! network port.
24//!
25//! **Untested:** these command builders are tested here, but neither has
26//! been run on a Mac or a Linux machine with KVM yet.
27
28use std::path::{Path, PathBuf};
29use std::process::{Command, Stdio};
30use std::time::{Duration, Instant};
31
32use anyhow::{Context, Result, bail};
33
34use super::guest::{self, GuestStatus, Settings};
35use super::image::{self, Local};
36use super::{Backend, Resources, StopMode, process_alive};
37use crate::selfhosted::native::Availability;
38use crate::selfhosted::needs::{self, KvmFacts};
39
40/// The seed disk's size: the settings' JSON, then zeros.
41pub const SEED_BYTES: usize = 1 << 20;
42/// The state disk's size: the runner's registration and status.
43pub const STATE_BYTES: u64 = 256 << 20;
44/// What the kernel is told: its console, and its system disk.
45pub const CMDLINE: &str = "console=hvc0 root=/dev/vda rw quiet";
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum Hypervisor {
49 Vfkit,
50 Qemu,
51}
52
53pub struct DiskVm {
54 pub hypervisor: Hypervisor,
55 pub dir: PathBuf,
56}
57
58/// The VM's files, in the state folder's `vm`.
59#[derive(Clone, Debug)]
60pub struct Paths {
61 pub kernel: PathBuf,
62 pub initrd: PathBuf,
63 pub system: PathBuf,
64 pub state: PathBuf,
65 pub seed: PathBuf,
66 pub console: PathBuf,
67 pub control: PathBuf,
68 pub pid: PathBuf,
69}
70
71impl Paths {
72 pub fn in_dir(vm: &Path) -> Paths {
73 Paths {
74 kernel: vm.join("kernel"),
75 initrd: vm.join("initrd"),
76 system: vm.join("system.raw"),
77 state: vm.join("state.img"),
78 seed: vm.join("seed.img"),
79 console: vm.join("console.log"),
80 control: vm.join("control.sock"),
81 pid: vm.join("vm.pid"),
82 }
83 }
84}
85
86/// vfkit's arguments: the VM's disks, NAT, its console to a file, and its
87/// control socket. Nothing shared with the Mac.
88pub fn vfkit_args(paths: &Paths, resources: &Resources) -> Vec<String> {
89 let p = |path: &Path| path.display().to_string();
90 vec![
91 "--cpus".into(),
92 resources.cpus.to_string(),
93 "--memory".into(),
94 resources.memory_mib.to_string(),
95 "--bootloader".into(),
96 format!("linux,kernel={},initrd={},cmdline=\"{CMDLINE}\"", p(&paths.kernel), p(&paths.initrd)),
97 "--device".into(),
98 format!("virtio-blk,path={}", p(&paths.system)),
99 "--device".into(),
100 format!("virtio-blk,path={}", p(&paths.state)),
101 "--device".into(),
102 format!("virtio-blk,path={}", p(&paths.seed)),
103 "--device".into(),
104 "virtio-net,nat".into(),
105 "--device".into(),
106 format!("virtio-serial,logFilePath={}", p(&paths.console)),
107 "--device".into(),
108 "virtio-rng".into(),
109 "--restful-uri".into(),
110 format!("unix://{}", p(&paths.control)),
111 ]
112}
113
114/// QEMU's arguments, for `qemu-system-x86_64` (`arch` x64) or
115/// `qemu-system-aarch64`: KVM, virtio-mmio devices, user networking with
116/// nothing forwarded in, the console to a file, QMP on a Unix socket, and
117/// QEMU's own seccomp sandbox.
118pub fn qemu_args(arch: &str, paths: &Paths, resources: &Resources) -> Vec<String> {
119 let p = |path: &Path| path.display().to_string();
120 let machine = if arch == "arm64" { "virt" } else { "microvm" };
121 let mut args: Vec<String> = vec![
122 "-machine".into(),
123 machine.into(),
124 "-accel".into(),
125 "kvm".into(),
126 "-cpu".into(),
127 "host".into(),
128 "-smp".into(),
129 resources.cpus.to_string(),
130 "-m".into(),
131 format!("{}M", resources.memory_mib),
132 "-nodefaults".into(),
133 "-no-user-config".into(),
134 "-nographic".into(),
135 "-kernel".into(),
136 p(&paths.kernel),
137 "-initrd".into(),
138 p(&paths.initrd),
139 "-append".into(),
140 CMDLINE.into(),
141 ];
142 for (id, path) in [("system", &paths.system), ("state", &paths.state), ("seed", &paths.seed)] {
143 args.extend(["-drive".into(), format!("id={id},file={},format=raw,if=none", p(path)), "-device".into(), format!("virtio-blk-device,drive={id}")]);
144 }
145 args.extend([
146 "-netdev".into(),
147 "user,id=net0".into(),
148 "-device".into(),
149 "virtio-net-device,netdev=net0".into(),
150 "-chardev".into(),
151 format!("file,id=console,path={}", p(&paths.console)),
152 "-device".into(),
153 "virtio-serial-device".into(),
154 "-device".into(),
155 "virtconsole,chardev=console".into(),
156 "-object".into(),
157 "rng-random,id=rng0,filename=/dev/urandom".into(),
158 "-device".into(),
159 "virtio-rng-device,rng=rng0".into(),
160 "-qmp".into(),
161 format!("unix:{},server=on,wait=off", p(&paths.control)),
162 "-pidfile".into(),
163 p(&paths.pid),
164 "-sandbox".into(),
165 "on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny".into(),
166 ]);
167 args
168}
169
170fn qemu_program() -> &'static str {
171 if image::arch() == "arm64" { "qemu-system-aarch64" } else { "qemu-system-x86_64" }
172}
173
174/// vfkit: on the PATH, else where Homebrew puts it, since a program the
175/// desktop app starts on a Mac has only the system's folders on its PATH.
176fn vfkit_program() -> String {
177 if answers("vfkit") {
178 return "vfkit".into();
179 }
180 needs::HOMEBREW_BINS
181 .iter()
182 .map(|bin| Path::new(bin).join("vfkit"))
183 .find(|path| path.exists())
184 .map_or_else(|| "vfkit".into(), |path| path.display().to_string())
185}
186
187fn answers(program: &str) -> bool {
188 Command::new(program).arg("--version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())
189}
190
191/// What a request to stop the VM is, over its control socket: vfkit's
192/// REST call, or QMP's handshake and an ACPI power button.
193pub fn stop_request(hypervisor: Hypervisor) -> Vec<u8> {
194 match hypervisor {
195 Hypervisor::Vfkit => {
196 let body = r#"{"state":"Stop"}"#;
197 format!("POST /vm/state HTTP/1.1\r\nHost: vfkit\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).into_bytes()
198 }
199 Hypervisor::Qemu => b"{\"execute\":\"qmp_capabilities\"}\n{\"execute\":\"system_powerdown\"}\n".to_vec(),
200 }
201}
202
203impl DiskVm {
204 fn paths(&self) -> Paths {
205 Paths::in_dir(&self.dir.join("vm"))
206 }
207
208 /// Takes the registration token out of the seed disk, once the runner
209 /// has registered with it. The guest reads the seed only as it boots,
210 /// and a token it lacks then is one it no longer needs.
211 fn forget_token(&self) {
212 let seed = self.paths().seed;
213 let Some(mut settings) = std::fs::read(&seed).ok().and_then(|bytes| guest::read_seed(&bytes).ok()) else { return };
214 if settings.token.take().is_some()
215 && let Ok(bytes) = guest::write_seed(&settings, SEED_BYTES)
216 {
217 let _ = super::write_private(&seed, &bytes);
218 }
219 }
220
221 fn pid(&self) -> Option<u32> {
222 std::fs::read_to_string(self.paths().pid).ok().and_then(|p| p.trim().parse().ok())
223 }
224
225 fn resources(&self) -> Resources {
226 std::fs::read(self.dir.join("vm").join("resources.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
227 }
228
229 #[cfg(unix)]
230 fn send_stop(&self) -> Result<()> {
231 use std::io::{Read, Write};
232 let mut socket = std::os::unix::net::UnixStream::connect(self.paths().control).context("could not reach the VM's control socket")?;
233 socket.set_read_timeout(Some(Duration::from_secs(5)))?;
234 socket.write_all(&stop_request(self.hypervisor))?;
235 let mut answer = [0u8; 512];
236 let _ = socket.read(&mut answer);
237 Ok(())
238 }
239
240 #[cfg(not(unix))]
241 fn send_stop(&self) -> Result<()> {
242 bail!("this VM runs on macOS and Linux")
243 }
244}
245
246impl Backend for DiskVm {
247 fn name(&self) -> &'static str {
248 match self.hypervisor {
249 Hypervisor::Vfkit => "vfkit",
250 Hypervisor::Qemu => "qemu",
251 }
252 }
253
254 fn image_keys(&self) -> Vec<String> {
255 ["kernel", "initrd", "disk"].iter().map(|part| format!("vm-{}-{part}", image::arch())).collect()
256 }
257
258 fn availability(&self) -> Availability {
259 let needs = match self.hypervisor {
260 Hypervisor::Vfkit => {
261 let macos = Command::new("sw_vers").arg("-productVersion").output().ok().and_then(|o| needs::macos_major(&String::from_utf8_lossy(&o.stdout)));
262 let homebrew = answers("brew") || needs::HOMEBREW_BINS.iter().any(|bin| Path::new(bin).join("brew").exists());
263 needs::vfkit_needs(macos, answers(&vfkit_program()), homebrew)
264 }
265 Hypervisor::Qemu => {
266 let device = Path::new("/dev/kvm").exists();
267 let (cpu, guest) = needs::cpu_flags(&std::fs::read_to_string("/proc/cpuinfo").unwrap_or_default());
268 needs::kvm_needs(&KvmFacts {
269 device,
270 usable: device && std::fs::OpenOptions::new().read(true).write(true).open("/dev/kvm").is_ok(),
271 cpu,
272 guest,
273 qemu: answers(qemu_program()),
274 })
275 }
276 };
277 Availability::needing(needs, "linux")
278 }
279
280 fn exists(&self) -> bool {
281 self.paths().system.exists()
282 }
283
284 fn running(&self) -> bool {
285 self.pid().is_some_and(process_alive)
286 }
287
288 fn create(&self, local: &Local, resources: &Resources) -> Result<()> {
289 let paths = self.paths();
290 std::fs::create_dir_all(self.dir.join("vm"))?;
291 let keys = self.image_keys();
292 let file = |part: usize| local.files.get(&keys[part]).with_context(|| format!("no {} image", keys[part]));
293 std::fs::copy(file(0)?, &paths.kernel)?;
294 std::fs::copy(file(1)?, &paths.initrd)?;
295 // The system disk comes compressed; it grows to the disk size asked
296 // for, and the guest grows its file system to fill it.
297 let mut source = flate2::read::GzDecoder::new(std::fs::File::open(file(2)?)?);
298 let mut system = std::fs::File::create(&paths.system)?;
299 std::io::copy(&mut source, &mut system).context("could not unpack the sandbox's system disk")?;
300 let wanted = resources.disk_gib << 30;
301 if system.metadata()?.len() < wanted {
302 system.set_len(wanted)?;
303 }
304 if !paths.state.exists() {
305 std::fs::File::create(&paths.state)?.set_len(STATE_BYTES)?;
306 }
307 std::fs::write(self.dir.join("vm").join("resources.json"), serde_json::to_vec(resources)?)?;
308 Ok(())
309 }
310
311 fn configure(&self, settings: &Settings) -> Result<()> {
312 let paths = self.paths();
313 let mut merged = std::fs::read(&paths.seed).ok().and_then(|b| guest::read_seed(&b).ok()).unwrap_or_default();
314 let given = settings.clone();
315 merged.url = given.url.or(merged.url);
316 merged.token = given.token.or(merged.token);
317 merged.name = given.name.or(merged.name);
318 if !given.labels.is_empty() {
319 merged.labels = given.labels;
320 }
321 merged.group = given.group.or(merged.group);
322 if given.cpus > 0 {
323 merged.cpus = given.cpus;
324 }
325 if given.memory_mib > 0 {
326 merged.memory_mib = given.memory_mib;
327 }
328 if !given.runner_image.is_empty() {
329 merged.runner_image = given.runner_image;
330 }
331 if !given.image_version.is_empty() {
332 merged.image_version = given.image_version;
333 }
334 merged.stop = given.stop.or(merged.stop);
335 super::write_private(&paths.seed, &guest::write_seed(&merged, SEED_BYTES)?)
336 }
337
338 fn load_image(&self, _tar: &Path) -> Result<()> {
339 bail!("--runner-image-file is for the WSL sandbox; a VM fetches the runner's image by digest")
340 }
341
342 fn start(&self) -> Result<()> {
343 if self.running() {
344 return Ok(());
345 }
346 let paths = self.paths();
347 let resources = self.resources();
348 let _ = std::fs::remove_file(&paths.control);
349 let _ = std::fs::remove_file(&paths.console);
350 let (program, args) = match self.hypervisor {
351 Hypervisor::Vfkit => (vfkit_program(), vfkit_args(&paths, &resources)),
352 Hypervisor::Qemu => (qemu_program().to_owned(), qemu_args(image::arch(), &paths, &resources)),
353 };
354 let log = std::fs::File::create(self.dir.join("sandbox.log"))?;
355 let mut command = Command::new(program);
356 command.args(&args).stdin(Stdio::null()).stdout(log.try_clone()?).stderr(log);
357 let pid = super::spawn_detached(command)?;
358 std::fs::write(&paths.pid, pid.to_string())?;
359 Ok(())
360 }
361
362 fn guest_status(&self) -> Option<GuestStatus> {
363 let log = std::fs::read(self.paths().console).ok()?;
364 // The console's tail is enough: statuses come every few seconds.
365 let tail = &log[log.len().saturating_sub(64 * 1024)..];
366 let status = guest::status_from_console(&String::from_utf8_lossy(tail))?;
367 // Registered: the token is no longer kept here either, as the
368 // guest no longer keeps its own copy (guest.rs).
369 if status.runner.is_some() {
370 self.forget_token();
371 }
372 Some(status)
373 }
374
375 fn stop(&self, mode: StopMode) -> Result<()> {
376 if !self.running() {
377 return Ok(());
378 }
379 self.configure(&Settings { stop: Some(if mode == StopMode::Drain { "drain" } else { "now" }.into()), ..Settings::default() })?;
380 self.send_stop()?;
381 let limit = (mode == StopMode::Now).then(|| Instant::now() + Duration::from_secs(150));
382 while self.running() {
383 if limit.is_some_and(|l| Instant::now() > l) {
384 if let Some(pid) = self.pid() {
385 let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status();
386 }
387 break;
388 }
389 std::thread::sleep(Duration::from_secs(1));
390 }
391 Ok(())
392 }
393
394 fn export_state(&self) -> Result<Option<Vec<u8>>> {
395 // The state disk is kept across updates.
396 Ok(None)
397 }
398
399 fn import_state(&self, _bytes: &[u8]) -> Result<()> {
400 Ok(())
401 }
402
403 fn remove_system(&self) -> Result<()> {
404 self.stop(StopMode::Now)?;
405 let paths = self.paths();
406 for path in [&paths.kernel, &paths.initrd, &paths.system, &paths.console] {
407 let _ = std::fs::remove_file(path);
408 }
409 Ok(())
410 }
411
412 fn remove(&self) -> Result<()> {
413 self.stop(StopMode::Now)?;
414 let _ = std::fs::remove_dir_all(self.dir.join("vm"));
415 Ok(())
416 }
417}
418
419#[cfg(test)]
420mod tests {
421 use super::*;
422
423 fn resources() -> Resources {
424 Resources { cpus: 4, memory_mib: 8192, disk_gib: 64 }
425 }
426
427 /// As `Paths::in_dir("/s/vm")` makes them on a Mac or Linux.
428 fn test_paths() -> Paths {
429 let p = |name: &str| PathBuf::from(format!("/s/vm/{name}"));
430 Paths {
431 kernel: p("kernel"),
432 initrd: p("initrd"),
433 system: p("system.raw"),
434 state: p("state.img"),
435 seed: p("seed.img"),
436 console: p("console.log"),
437 control: p("control.sock"),
438 pid: p("vm.pid"),
439 }
440 }
441
442 #[test]
443 fn vfkit_shares_nothing_with_the_mac() {
444 let paths = test_paths();
445 let args = vfkit_args(&paths, &resources());
446 let text = args.join(" ");
447 assert!(text.contains("--cpus 4 --memory 8192"));
448 assert!(text.contains("linux,kernel=/s/vm/kernel,initrd=/s/vm/initrd,cmdline=\"console=hvc0 root=/dev/vda rw quiet\""));
449 assert_eq!(args.iter().filter(|a| a.starts_with("virtio-blk,")).count(), 3);
450 assert!(text.contains("virtio-net,nat"));
451 assert!(text.contains("virtio-serial,logFilePath=/s/vm/console.log"));
452 assert!(text.contains("--restful-uri unix:///s/vm/control.sock"));
453 for shared in ["virtio-fs", "virtio-vsock", "virtio-gpu", "virtio-input", "rosetta", "tcp://"] {
454 assert!(!text.contains(shared), "{shared}");
455 }
456 }
457
458 #[test]
459 fn qemu_forwards_nothing_in_and_sandboxes_itself() {
460 let paths = test_paths();
461 let text = qemu_args("x64", &paths, &resources()).join(" ");
462 assert!(text.starts_with("-machine microvm -accel kvm -cpu host -smp 4 -m 8192M -nodefaults"));
463 assert!(text.contains("-netdev user,id=net0 "));
464 assert!(!text.contains("hostfwd") && !text.contains("-virtfs") && !text.contains("tcp:"));
465 assert!(text.contains("-qmp unix:/s/vm/control.sock,server=on,wait=off"));
466 assert!(text.contains("-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"));
467 assert_eq!(text.matches("virtio-blk-device").count(), 3);
468 assert!(qemu_args("arm64", &paths, &resources()).join(" ").starts_with("-machine virt "));
469 }
470
471 #[test]
472 fn stopping_is_a_power_button_not_a_pulled_plug() {
473 let vfkit = String::from_utf8(stop_request(Hypervisor::Vfkit)).unwrap();
474 assert!(vfkit.starts_with("POST /vm/state HTTP/1.1\r\n"));
475 assert!(vfkit.ends_with(r#"{"state":"Stop"}"#));
476 assert!(vfkit.contains("Content-Length: 16\r\n"));
477 let qmp = String::from_utf8(stop_request(Hypervisor::Qemu)).unwrap();
478 assert!(qmp.contains("qmp_capabilities") && qmp.contains("system_powerdown"));
479 }
480
481 #[test]
482 fn the_seed_lets_go_of_the_token_once_the_runner_registered() {
483 let dir = std::env::temp_dir().join(format!("g1t-vm-seed-{}", std::process::id()));
484 let vm = DiskVm { hypervisor: Hypervisor::Qemu, dir: dir.clone() };
485 std::fs::create_dir_all(dir.join("vm")).unwrap();
486 let settings = Settings { url: Some("https://g1t.sh".into()), token: Some("g1trt_x".into()), runner_image: "img@sha256:ab".into(), ..Settings::default() };
487 std::fs::write(vm.paths().seed, guest::write_seed(&settings, SEED_BYTES).unwrap()).unwrap();
488 // The console says the runner is registered and running.
489 let status = GuestStatus { state: "running".into(), runner: Some(crate::selfhosted::RunnerStatus::default()), ..GuestStatus::default() };
490 std::fs::write(vm.paths().console, format!("boot\n{}{}\n", guest::STATUS_PREFIX, serde_json::to_string(&status).unwrap())).unwrap();
491 assert!(vm.guest_status().is_some_and(|s| s.runner.is_some()));
492 let kept = guest::read_seed(&std::fs::read(vm.paths().seed).unwrap()).unwrap();
493 assert_eq!(kept.token, None);
494 assert_eq!(kept.url.as_deref(), Some("https://g1t.sh"));
495 assert_eq!(kept.runner_image, "img@sha256:ab");
496 let _ = std::fs::remove_dir_all(&dir);
497 }
498
499 #[test]
500 fn a_vm_needs_its_three_files() {
501 let vm = DiskVm { hypervisor: Hypervisor::Vfkit, dir: PathBuf::from("/s") };
502 let keys = vm.image_keys();
503 assert_eq!(keys.len(), 3);
504 assert!(keys[2].ends_with("-disk") && keys[0].starts_with("vm-"));
505 }
506}