Skip to content
Sign up

flagon-io/g1t

public
0

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
491 linesCodeBlameRaw
1//! macOS, and Linux without Sysbox: the sandbox is a small Linux VM booted
2//! from the release's signed kernel, initrd and system disk.
3//!
4//! - **macOS: vfkit**, a small command line program over Apple's
5//! Virtualization framework (`brew install vfkit`; the one Podman and
6//! CRC use). Chosen over Lima, which shares the home folder unless told
7//! otherwise and brings its own guest agent, and Apple's `container`,
8//! which needs macOS 26 and runs one VM per container rather than one
9//! for the runner. vfkit attaches only the devices named on its command
10//! line, so the sandbox gets disks, a NAT network card, a console and a
11//! random number source: no shared folder, no vsock, no display, no
12//! clipboard.
13//! - **Linux: QEMU with KVM**, the same VM: virtio-mmio devices on a
14//! `microvm` (x64) or `virt` (arm64) machine, user-mode networking with
15//! nothing forwarded in, and QEMU's seccomp sandbox on.
16//!
17//! The VM has three disks: its system (replaced by an update), the
18//! runner's state (kept, so an update does not register again; formatted
19//! by the guest the first time) and a seed the launcher writes, holding the
20//! settings (guest.rs). It says how it is on its console, which the
21//! launcher reads from a file; the launcher stops it through the
22//! hypervisor's control socket, a Unix socket in the state folder, never a
23//! network port.
24//!
25//! **Untested:** these command builders are tested here, but neither has
26//! been run on a Mac or a Linux machine with KVM yet.
27
28use std::path::{Path, PathBuf};
29use std::process::{Command, Stdio};
30use std::time::{Duration, Instant};
31
32use anyhow::{Context, Result, bail};
33
34use super::guest::{self, GuestStatus, Settings};
35use super::image::{self, Local};
36use super::{Backend, Resources, StopMode, process_alive};
37use crate::selfhosted::native::Availability;
38
39/// The seed disk's size: the settings' JSON, then zeros.
40pub const SEED_BYTES: usize = 1 << 20;
41/// The state disk's size: the runner's registration and status.
42pub const STATE_BYTES: u64 = 256 << 20;
43/// What the kernel is told: its console, and its system disk.
44pub const CMDLINE: &str = "console=hvc0 root=/dev/vda rw quiet";
45
46#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47pub enum Hypervisor {
48 Vfkit,
49 Qemu,
50}
51
52pub struct DiskVm {
53 pub hypervisor: Hypervisor,
54 pub dir: PathBuf,
55}
56
57/// The VM's files, in the state folder's `vm`.
58#[derive(Clone, Debug)]
59pub struct Paths {
60 pub kernel: PathBuf,
61 pub initrd: PathBuf,
62 pub system: PathBuf,
63 pub state: PathBuf,
64 pub seed: PathBuf,
65 pub console: PathBuf,
66 pub control: PathBuf,
67 pub pid: PathBuf,
68}
69
70impl Paths {
71 pub fn in_dir(vm: &Path) -> Paths {
72 Paths {
73 kernel: vm.join("kernel"),
74 initrd: vm.join("initrd"),
75 system: vm.join("system.raw"),
76 state: vm.join("state.img"),
77 seed: vm.join("seed.img"),
78 console: vm.join("console.log"),
79 control: vm.join("control.sock"),
80 pid: vm.join("vm.pid"),
81 }
82 }
83}
84
85/// vfkit's arguments: the VM's disks, NAT, its console to a file, and its
86/// control socket. Nothing shared with the Mac.
87pub fn vfkit_args(paths: &Paths, resources: &Resources) -> Vec<String> {
88 let p = |path: &Path| path.display().to_string();
89 vec![
90 "--cpus".into(),
91 resources.cpus.to_string(),
92 "--memory".into(),
93 resources.memory_mib.to_string(),
94 "--bootloader".into(),
95 format!("linux,kernel={},initrd={},cmdline=\"{CMDLINE}\"", p(&paths.kernel), p(&paths.initrd)),
96 "--device".into(),
97 format!("virtio-blk,path={}", p(&paths.system)),
98 "--device".into(),
99 format!("virtio-blk,path={}", p(&paths.state)),
100 "--device".into(),
101 format!("virtio-blk,path={}", p(&paths.seed)),
102 "--device".into(),
103 "virtio-net,nat".into(),
104 "--device".into(),
105 format!("virtio-serial,logFilePath={}", p(&paths.console)),
106 "--device".into(),
107 "virtio-rng".into(),
108 "--restful-uri".into(),
109 format!("unix://{}", p(&paths.control)),
110 ]
111}
112
113/// QEMU's arguments, for `qemu-system-x86_64` (`arch` x64) or
114/// `qemu-system-aarch64`: KVM, virtio-mmio devices, user networking with
115/// nothing forwarded in, the console to a file, QMP on a Unix socket, and
116/// QEMU's own seccomp sandbox.
117pub fn qemu_args(arch: &str, paths: &Paths, resources: &Resources) -> Vec<String> {
118 let p = |path: &Path| path.display().to_string();
119 let machine = if arch == "arm64" { "virt" } else { "microvm" };
120 let mut args: Vec<String> = vec![
121 "-machine".into(),
122 machine.into(),
123 "-accel".into(),
124 "kvm".into(),
125 "-cpu".into(),
126 "host".into(),
127 "-smp".into(),
128 resources.cpus.to_string(),
129 "-m".into(),
130 format!("{}M", resources.memory_mib),
131 "-nodefaults".into(),
132 "-no-user-config".into(),
133 "-nographic".into(),
134 "-kernel".into(),
135 p(&paths.kernel),
136 "-initrd".into(),
137 p(&paths.initrd),
138 "-append".into(),
139 CMDLINE.into(),
140 ];
141 for (id, path) in [("system", &paths.system), ("state", &paths.state), ("seed", &paths.seed)] {
142 args.extend(["-drive".into(), format!("id={id},file={},format=raw,if=none", p(path)), "-device".into(), format!("virtio-blk-device,drive={id}")]);
143 }
144 args.extend([
145 "-netdev".into(),
146 "user,id=net0".into(),
147 "-device".into(),
148 "virtio-net-device,netdev=net0".into(),
149 "-chardev".into(),
150 format!("file,id=console,path={}", p(&paths.console)),
151 "-device".into(),
152 "virtio-serial-device".into(),
153 "-device".into(),
154 "virtconsole,chardev=console".into(),
155 "-object".into(),
156 "rng-random,id=rng0,filename=/dev/urandom".into(),
157 "-device".into(),
158 "virtio-rng-device,rng=rng0".into(),
159 "-qmp".into(),
160 format!("unix:{},server=on,wait=off", p(&paths.control)),
161 "-pidfile".into(),
162 p(&paths.pid),
163 "-sandbox".into(),
164 "on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny".into(),
165 ]);
166 args
167}
168
169fn qemu_program() -> &'static str {
170 if image::arch() == "arm64" { "qemu-system-aarch64" } else { "qemu-system-x86_64" }
171}
172
173fn answers(program: &str) -> bool {
174 Command::new(program).arg("--version").stdout(Stdio::null()).stderr(Stdio::null()).status().is_ok_and(|s| s.success())
175}
176
177/// What a request to stop the VM is, over its control socket: vfkit's
178/// REST call, or QMP's handshake and an ACPI power button.
179pub fn stop_request(hypervisor: Hypervisor) -> Vec<u8> {
180 match hypervisor {
181 Hypervisor::Vfkit => {
182 let body = r#"{"state":"Stop"}"#;
183 format!("POST /vm/state HTTP/1.1\r\nHost: vfkit\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).into_bytes()
184 }
185 Hypervisor::Qemu => b"{\"execute\":\"qmp_capabilities\"}\n{\"execute\":\"system_powerdown\"}\n".to_vec(),
186 }
187}
188
189impl DiskVm {
190 fn paths(&self) -> Paths {
191 Paths::in_dir(&self.dir.join("vm"))
192 }
193
194 /// Takes the registration token out of the seed disk, once the runner
195 /// has registered with it. The guest reads the seed only as it boots,
196 /// and a token it lacks then is one it no longer needs.
197 fn forget_token(&self) {
198 let seed = self.paths().seed;
199 let Some(mut settings) = std::fs::read(&seed).ok().and_then(|bytes| guest::read_seed(&bytes).ok()) else { return };
200 if settings.token.take().is_some()
201 && let Ok(bytes) = guest::write_seed(&settings, SEED_BYTES)
202 {
203 let _ = super::write_private(&seed, &bytes);
204 }
205 }
206
207 fn pid(&self) -> Option<u32> {
208 std::fs::read_to_string(self.paths().pid).ok().and_then(|p| p.trim().parse().ok())
209 }
210
211 fn resources(&self) -> Resources {
212 std::fs::read(self.dir.join("vm").join("resources.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
213 }
214
215 #[cfg(unix)]
216 fn send_stop(&self) -> Result<()> {
217 use std::io::{Read, Write};
218 let mut socket = std::os::unix::net::UnixStream::connect(self.paths().control).context("could not reach the VM's control socket")?;
219 socket.set_read_timeout(Some(Duration::from_secs(5)))?;
220 socket.write_all(&stop_request(self.hypervisor))?;
221 let mut answer = [0u8; 512];
222 let _ = socket.read(&mut answer);
223 Ok(())
224 }
225
226 #[cfg(not(unix))]
227 fn send_stop(&self) -> Result<()> {
228 bail!("this VM runs on macOS and Linux")
229 }
230}
231
232impl Backend for DiskVm {
233 fn name(&self) -> &'static str {
234 match self.hypervisor {
235 Hypervisor::Vfkit => "vfkit",
236 Hypervisor::Qemu => "qemu",
237 }
238 }
239
240 fn image_keys(&self) -> Vec<String> {
241 ["kernel", "initrd", "disk"].iter().map(|part| format!("vm-{}-{part}", image::arch())).collect()
242 }
243
244 fn availability(&self) -> Availability {
245 match self.hypervisor {
246 Hypervisor::Vfkit if !answers("vfkit") => Availability::no(
247 "vfkit, which runs the sandbox's VM with Apple's Virtualization framework, is not installed.",
248 "Install it with `brew install vfkit` (macOS 13 or later), then start the sandbox again.",
249 ),
250 Hypervisor::Qemu if !Path::new("/dev/kvm").exists() => Availability::no(
251 "This machine has neither Sysbox nor KVM, so there is nothing to sandbox the runner with.",
252 "Install Sysbox 0.7 or later (https://github.com/nestybox/sysbox), or turn on virtualization and install QEMU so /dev/kvm exists.",
253 ),
254 Hypervisor::Qemu if std::fs::OpenOptions::new().read(true).write(true).open("/dev/kvm").is_err() => {
255 Availability::no("This user may not use /dev/kvm.", "Add this user to the kvm group, then log in again.")
256 }
257 Hypervisor::Qemu if !answers(qemu_program()) => Availability::no(
258 format!("{} is not installed.", qemu_program()),
259 "Install QEMU (qemu-system on Debian and Ubuntu, qemu-kvm on Fedora), or Sysbox 0.7 or later.",
260 ),
261 _ => Availability::yes(),
262 }
263 }
264
265 fn exists(&self) -> bool {
266 self.paths().system.exists()
267 }
268
269 fn running(&self) -> bool {
270 self.pid().is_some_and(process_alive)
271 }
272
273 fn create(&self, local: &Local, resources: &Resources) -> Result<()> {
274 let paths = self.paths();
275 std::fs::create_dir_all(self.dir.join("vm"))?;
276 let keys = self.image_keys();
277 let file = |part: usize| local.files.get(&keys[part]).with_context(|| format!("no {} image", keys[part]));
278 std::fs::copy(file(0)?, &paths.kernel)?;
279 std::fs::copy(file(1)?, &paths.initrd)?;
280 // The system disk comes compressed; it grows to the disk size asked
281 // for, and the guest grows its file system to fill it.
282 let mut source = flate2::read::GzDecoder::new(std::fs::File::open(file(2)?)?);
283 let mut system = std::fs::File::create(&paths.system)?;
284 std::io::copy(&mut source, &mut system).context("could not unpack the sandbox's system disk")?;
285 let wanted = resources.disk_gib << 30;
286 if system.metadata()?.len() < wanted {
287 system.set_len(wanted)?;
288 }
289 if !paths.state.exists() {
290 std::fs::File::create(&paths.state)?.set_len(STATE_BYTES)?;
291 }
292 std::fs::write(self.dir.join("vm").join("resources.json"), serde_json::to_vec(resources)?)?;
293 Ok(())
294 }
295
296 fn configure(&self, settings: &Settings) -> Result<()> {
297 let paths = self.paths();
298 let mut merged = std::fs::read(&paths.seed).ok().and_then(|b| guest::read_seed(&b).ok()).unwrap_or_default();
299 let given = settings.clone();
300 merged.url = given.url.or(merged.url);
301 merged.token = given.token.or(merged.token);
302 merged.name = given.name.or(merged.name);
303 if !given.labels.is_empty() {
304 merged.labels = given.labels;
305 }
306 merged.group = given.group.or(merged.group);
307 if given.cpus > 0 {
308 merged.cpus = given.cpus;
309 }
310 if given.memory_mib > 0 {
311 merged.memory_mib = given.memory_mib;
312 }
313 if !given.runner_image.is_empty() {
314 merged.runner_image = given.runner_image;
315 }
316 if !given.image_version.is_empty() {
317 merged.image_version = given.image_version;
318 }
319 merged.stop = given.stop.or(merged.stop);
320 super::write_private(&paths.seed, &guest::write_seed(&merged, SEED_BYTES)?)
321 }
322
323 fn load_image(&self, _tar: &Path) -> Result<()> {
324 bail!("--runner-image-file is for the WSL sandbox; a VM fetches the runner's image by digest")
325 }
326
327 fn start(&self) -> Result<()> {
328 if self.running() {
329 return Ok(());
330 }
331 let paths = self.paths();
332 let resources = self.resources();
333 let _ = std::fs::remove_file(&paths.control);
334 let _ = std::fs::remove_file(&paths.console);
335 let (program, args) = match self.hypervisor {
336 Hypervisor::Vfkit => ("vfkit", vfkit_args(&paths, &resources)),
337 Hypervisor::Qemu => (qemu_program(), qemu_args(image::arch(), &paths, &resources)),
338 };
339 let log = std::fs::File::create(self.dir.join("sandbox.log"))?;
340 let mut command = Command::new(program);
341 command.args(&args).stdin(Stdio::null()).stdout(log.try_clone()?).stderr(log);
342 let pid = super::spawn_detached(command)?;
343 std::fs::write(&paths.pid, pid.to_string())?;
344 Ok(())
345 }
346
347 fn guest_status(&self) -> Option<GuestStatus> {
348 let log = std::fs::read(self.paths().console).ok()?;
349 // The console's tail is enough: statuses come every few seconds.
350 let tail = &log[log.len().saturating_sub(64 * 1024)..];
351 let status = guest::status_from_console(&String::from_utf8_lossy(tail))?;
352 // Registered: the token is no longer kept here either, as the
353 // guest no longer keeps its own copy (guest.rs).
354 if status.runner.is_some() {
355 self.forget_token();
356 }
357 Some(status)
358 }
359
360 fn stop(&self, mode: StopMode) -> Result<()> {
361 if !self.running() {
362 return Ok(());
363 }
364 self.configure(&Settings { stop: Some(if mode == StopMode::Drain { "drain" } else { "now" }.into()), ..Settings::default() })?;
365 self.send_stop()?;
366 let limit = (mode == StopMode::Now).then(|| Instant::now() + Duration::from_secs(150));
367 while self.running() {
368 if limit.is_some_and(|l| Instant::now() > l) {
369 if let Some(pid) = self.pid() {
370 let _ = Command::new("kill").args(["-KILL", &pid.to_string()]).status();
371 }
372 break;
373 }
374 std::thread::sleep(Duration::from_secs(1));
375 }
376 Ok(())
377 }
378
379 fn export_state(&self) -> Result<Option<Vec<u8>>> {
380 // The state disk is kept across updates.
381 Ok(None)
382 }
383
384 fn import_state(&self, _bytes: &[u8]) -> Result<()> {
385 Ok(())
386 }
387
388 fn remove_system(&self) -> Result<()> {
389 self.stop(StopMode::Now)?;
390 let paths = self.paths();
391 for path in [&paths.kernel, &paths.initrd, &paths.system, &paths.console] {
392 let _ = std::fs::remove_file(path);
393 }
394 Ok(())
395 }
396
397 fn remove(&self) -> Result<()> {
398 self.stop(StopMode::Now)?;
399 let _ = std::fs::remove_dir_all(self.dir.join("vm"));
400 Ok(())
401 }
402}
403
404#[cfg(test)]
405mod tests {
406 use super::*;
407
408 fn resources() -> Resources {
409 Resources { cpus: 4, memory_mib: 8192, disk_gib: 64 }
410 }
411
412 /// As `Paths::in_dir("/s/vm")` makes them on a Mac or Linux.
413 fn test_paths() -> Paths {
414 let p = |name: &str| PathBuf::from(format!("/s/vm/{name}"));
415 Paths {
416 kernel: p("kernel"),
417 initrd: p("initrd"),
418 system: p("system.raw"),
419 state: p("state.img"),
420 seed: p("seed.img"),
421 console: p("console.log"),
422 control: p("control.sock"),
423 pid: p("vm.pid"),
424 }
425 }
426
427 #[test]
428 fn vfkit_shares_nothing_with_the_mac() {
429 let paths = test_paths();
430 let args = vfkit_args(&paths, &resources());
431 let text = args.join(" ");
432 assert!(text.contains("--cpus 4 --memory 8192"));
433 assert!(text.contains("linux,kernel=/s/vm/kernel,initrd=/s/vm/initrd,cmdline=\"console=hvc0 root=/dev/vda rw quiet\""));
434 assert_eq!(args.iter().filter(|a| a.starts_with("virtio-blk,")).count(), 3);
435 assert!(text.contains("virtio-net,nat"));
436 assert!(text.contains("virtio-serial,logFilePath=/s/vm/console.log"));
437 assert!(text.contains("--restful-uri unix:///s/vm/control.sock"));
438 for shared in ["virtio-fs", "virtio-vsock", "virtio-gpu", "virtio-input", "rosetta", "tcp://"] {
439 assert!(!text.contains(shared), "{shared}");
440 }
441 }
442
443 #[test]
444 fn qemu_forwards_nothing_in_and_sandboxes_itself() {
445 let paths = test_paths();
446 let text = qemu_args("x64", &paths, &resources()).join(" ");
447 assert!(text.starts_with("-machine microvm -accel kvm -cpu host -smp 4 -m 8192M -nodefaults"));
448 assert!(text.contains("-netdev user,id=net0 "));
449 assert!(!text.contains("hostfwd") && !text.contains("-virtfs") && !text.contains("tcp:"));
450 assert!(text.contains("-qmp unix:/s/vm/control.sock,server=on,wait=off"));
451 assert!(text.contains("-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny"));
452 assert_eq!(text.matches("virtio-blk-device").count(), 3);
453 assert!(qemu_args("arm64", &paths, &resources()).join(" ").starts_with("-machine virt "));
454 }
455
456 #[test]
457 fn stopping_is_a_power_button_not_a_pulled_plug() {
458 let vfkit = String::from_utf8(stop_request(Hypervisor::Vfkit)).unwrap();
459 assert!(vfkit.starts_with("POST /vm/state HTTP/1.1\r\n"));
460 assert!(vfkit.ends_with(r#"{"state":"Stop"}"#));
461 assert!(vfkit.contains("Content-Length: 16\r\n"));
462 let qmp = String::from_utf8(stop_request(Hypervisor::Qemu)).unwrap();
463 assert!(qmp.contains("qmp_capabilities") && qmp.contains("system_powerdown"));
464 }
465
466 #[test]
467 fn the_seed_lets_go_of_the_token_once_the_runner_registered() {
468 let dir = std::env::temp_dir().join(format!("g1t-vm-seed-{}", std::process::id()));
469 let vm = DiskVm { hypervisor: Hypervisor::Qemu, dir: dir.clone() };
470 std::fs::create_dir_all(dir.join("vm")).unwrap();
471 let settings = Settings { url: Some("https://g1t.sh".into()), token: Some("g1trt_x".into()), runner_image: "img@sha256:ab".into(), ..Settings::default() };
472 std::fs::write(vm.paths().seed, guest::write_seed(&settings, SEED_BYTES).unwrap()).unwrap();
473 // The console says the runner is registered and running.
474 let status = GuestStatus { state: "running".into(), runner: Some(crate::selfhosted::RunnerStatus::default()), ..GuestStatus::default() };
475 std::fs::write(vm.paths().console, format!("boot\n{}{}\n", guest::STATUS_PREFIX, serde_json::to_string(&status).unwrap())).unwrap();
476 assert!(vm.guest_status().is_some_and(|s| s.runner.is_some()));
477 let kept = guest::read_seed(&std::fs::read(vm.paths().seed).unwrap()).unwrap();
478 assert_eq!(kept.token, None);
479 assert_eq!(kept.url.as_deref(), Some("https://g1t.sh"));
480 assert_eq!(kept.runner_image, "img@sha256:ab");
481 let _ = std::fs::remove_dir_all(&dir);
482 }
483
484 #[test]
485 fn a_vm_needs_its_three_files() {
486 let vm = DiskVm { hypervisor: Hypervisor::Vfkit, dir: PathBuf::from("/s") };
487 let keys = vm.image_keys();
488 assert_eq!(keys.len(), 3);
489 assert!(keys[2].ends_with("-disk") && keys[0].starts_with("vm-"));
490 }
491}