Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge the runner sandbox: g1t-runner sandbox up makes a person's own computer a self-hosted runner that nothing a job runs can get out of. Linux jobs run in a Linux VM of the runner's own (a WSL 2 distribution with no host mounts, interop or Docker socket on Windows; vfkit on macOS; Sysbox, or QEMU and KVM, on Linux), each in a fresh container under Sysbox inside it; on Windows 11 Pro and up, Windows jobs each run in a Windows Sandbox discarded after the job. Runners report their isolation, and the Runners page shows it. | 1 | //! The sandbox's images: fetched from the release they belong to, checked |
| 2 | //! against the signed manifest, and kept by version in the state folder. | |
| 3 | //! | |
| 4 | //! The trust model is the runner's own (update.rs): `latest.json` is only | |
| 5 | //! believed when its Ed25519 signature is g1t's release key's, built into | |
| 6 | //! this program, and a file is only used when its SHA-256 is the one the | |
| 7 | //! signed manifest names. A file already on disk is checked again before | |
| 8 | //! each use. The runner's image inside the sandbox is named by digest in | |
| 9 | //! the same manifest, so the engine that pulls it checks it too. | |
| 10 | //! | |
| 11 | //! A build without the release key (a developer's) cannot check anything, | |
| 12 | //! so it fetches nothing: `--image-file` gives it a local image instead, | |
| 13 | //! and it says loudly that it is unchecked. | |
| 14 | ||
| 15 | use std::collections::BTreeMap; | |
| 16 | use std::io::{Read, Write}; | |
| 17 | use std::path::{Path, PathBuf}; | |
| 18 | ||
| 19 | use anyhow::{Context, Result, anyhow, bail}; | |
| 20 | use sha2::{Digest, Sha256}; | |
| 21 | ||
| 22 | use crate::selfhosted::update::{self, SandboxFile}; | |
| 23 | ||
| 24 | /// The images a sandbox needs, on disk and checked. | |
| 25 | #[derive(Clone, Debug, Default)] | |
| 26 | pub struct Local { | |
| 27 | pub version: String, | |
| 28 | /// By kind (`wsl-x64`, `vm-arm64-disk`…). | |
| 29 | pub files: BTreeMap<String, PathBuf>, | |
| 30 | /// The runner's image, by digest. | |
| 31 | pub runner_image: Option<String>, | |
| 32 | } | |
| 33 | ||
| 34 | /// This machine's architecture, as releases name it. | |
| 35 | pub fn arch() -> &'static str { | |
| 36 | if std::env::consts::ARCH == "aarch64" { "arm64" } else { "x64" } | |
| 37 | } | |
| 38 | ||
| 39 | /// SHA-256 of a file, read in pieces. | |
| 40 | pub fn sha256_file(path: &Path) -> Result<String> { | |
| 41 | let mut file = std::fs::File::open(path).with_context(|| format!("could not open {}", path.display()))?; | |
| 42 | let mut hasher = Sha256::new(); | |
| 43 | let mut buffer = vec![0u8; 1 << 20]; | |
| 44 | loop { | |
| 45 | let read = file.read(&mut buffer)?; | |
| 46 | if read == 0 { | |
| 47 | break; | |
| 48 | } | |
| 49 | hasher.update(&buffer[..read]); | |
| 50 | } | |
| 51 | Ok(hex::encode(hasher.finalize())) | |
| 52 | } | |
| 53 | ||
| 54 | /// Whether a release's sandbox has every file of `keys`, and the newest | |
| 55 | /// signed release's version. | |
| 56 | pub fn latest(site: &str) -> Result<update::Manifest> { | |
| 57 | update::latest_signed_at(site)?.ok_or_else(|| { | |
| 58 | anyhow!("this build of g1t-runner has no release key, so it cannot check the sandbox's images: pass --image-file with an image you built (scripts/sandbox-image.mjs)") | |
| 59 | }) | |
| 60 | } | |
| 61 | ||
| 62 | /// Fetches (or finds) and checks the files of `keys` from the signed | |
| 63 | /// release `manifest`. | |
| 64 | pub fn obtain(site: &str, manifest: &update::Manifest, keys: &[String], images: &Path, progress: &dyn Fn(&str)) -> Result<Local> { | |
| 65 | let sandbox = manifest.sandbox.as_ref().ok_or_else(|| anyhow!("release {} has no sandbox images", manifest.version))?; | |
| 66 | let dir = images.join(&manifest.version); | |
| 67 | std::fs::create_dir_all(&dir).with_context(|| format!("could not make {}", dir.display()))?; | |
| 68 | let mut files = BTreeMap::new(); | |
| 69 | for key in keys { | |
| 70 | let file = sandbox.files.get(key).ok_or_else(|| anyhow!("release {} has no sandbox image {key} for this machine", manifest.version))?; | |
| 71 | let path = dir.join(safe_name(&file.name)?); | |
| 72 | if !(path.exists() && sha256_file(&path)? == file.sha256) { | |
| 73 | progress(&format!("Fetching the sandbox's {key} image ({} MB)", file.size.div_ceil(1_000_000))); | |
| 74 | download_checked(&format!("{}/{}/{}", update::downloads_at(site), manifest.version, file.name), &path, file)?; | |
| 75 | } | |
| 76 | files.insert(key.clone(), path); | |
| 77 | } | |
| 78 | Ok(Local { version: manifest.version.clone(), files, runner_image: sandbox.runner_image.clone() }) | |
| 79 | } | |
| 80 | ||
| 81 | /// A file name from a manifest, which must not name a folder. | |
| 82 | fn safe_name(name: &str) -> Result<&str> { | |
| 83 | if name.is_empty() || name.contains(['/', '\\']) || name.starts_with('.') { | |
| 84 | bail!("the release names a sandbox image {name:?}, which is not a file name"); | |
| 85 | } | |
| 86 | Ok(name) | |
| 87 | } | |
| 88 | ||
| 89 | /// Downloads to `path`, hashing as it goes; the file only takes its name | |
| 90 | /// once its SHA-256 is the manifest's. | |
| 91 | fn download_checked(url: &str, path: &Path, want: &SandboxFile) -> Result<()> { | |
| 92 | let response = ureq::get(url).timeout(std::time::Duration::from_secs(3600)).call().map_err(|e| anyhow!("could not fetch {url}: {e}"))?; | |
| 93 | let part = path.with_extension("part"); | |
| 94 | let mut out = std::fs::File::create(&part).with_context(|| format!("could not write {}", part.display()))?; | |
| 95 | let mut reader = response.into_reader(); | |
| 96 | let mut hasher = Sha256::new(); | |
| 97 | let mut buffer = vec![0u8; 1 << 20]; | |
| 98 | loop { | |
| 99 | let read = reader.read(&mut buffer).with_context(|| format!("the download of {url} broke off"))?; | |
| 100 | if read == 0 { | |
| 101 | break; | |
| 102 | } | |
| 103 | hasher.update(&buffer[..read]); | |
| 104 | out.write_all(&buffer[..read])?; | |
| 105 | } | |
| 106 | out.sync_all()?; | |
| 107 | drop(out); | |
| 108 | if hex::encode(hasher.finalize()) != want.sha256 { | |
| 109 | let _ = std::fs::remove_file(&part); | |
| 110 | bail!("the sandbox image {}'s SHA-256 is not the release's; not using it", want.name); | |
| 111 | } | |
| 112 | std::fs::rename(&part, path)?; | |
| 113 | Ok(()) | |
| 114 | } | |
| 115 | ||
| 116 | /// A local image (`--image-file`), unchecked: for a build of the sandbox | |
| 117 | /// that is not released. | |
| 118 | pub fn local(keys: &[String], given: &Path, runner_image: Option<String>) -> Result<Local> { | |
| 119 | if !given.exists() { | |
| 120 | bail!("{} does not exist", given.display()); | |
| 121 | } | |
| 122 | let mut files = BTreeMap::new(); | |
| 123 | if keys.len() == 1 { | |
| 124 | files.insert(keys[0].clone(), given.to_path_buf()); | |
| 125 | } else { | |
| 126 | // A folder holding the VM's files, named as a release names them. | |
| 127 | for key in keys { | |
| 128 | let suffix = key.rsplit('-').next().unwrap_or(key); | |
| 129 | let found = std::fs::read_dir(given)? | |
| 130 | .flatten() | |
| 131 | .map(|e| e.path()) | |
| 132 | .find(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.contains(suffix))) | |
| 133 | .ok_or_else(|| anyhow!("{} has no {key} file", given.display()))?; | |
| 134 | files.insert(key.clone(), found); | |
| 135 | } | |
| 136 | } | |
| 137 | Ok(Local { version: "local".into(), files, runner_image }) | |
| 138 | } | |
| 139 | ||
| 140 | /// Removes every version but `keep`. | |
| 141 | pub fn prune(images: &Path, keep: &str) { | |
| 142 | let Ok(entries) = std::fs::read_dir(images) else { return }; | |
| 143 | for entry in entries.flatten() { | |
| 144 | if entry.file_name().to_string_lossy() != keep { | |
| 145 | let _ = std::fs::remove_dir_all(entry.path()); | |
| 146 | } | |
| 147 | } | |
| 148 | } | |
| 149 | ||
| 150 | #[cfg(test)] | |
| 151 | mod tests { | |
| 152 | use super::*; | |
| 153 | ||
| 154 | #[test] | |
| 155 | fn a_files_hash_is_read_in_pieces() { | |
| 156 | let dir = std::env::temp_dir().join(format!("g1t-sandbox-image-{}", std::process::id())); | |
| 157 | std::fs::create_dir_all(&dir).unwrap(); | |
| 158 | let path = dir.join("x"); | |
| 159 | std::fs::write(&path, b"abc").unwrap(); | |
| 160 | assert_eq!(sha256_file(&path).unwrap(), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); | |
| 161 | let _ = std::fs::remove_dir_all(&dir); | |
| 162 | } | |
| 163 | ||
| 164 | #[test] | |
| 165 | fn a_manifest_cannot_name_a_folder() { | |
| 166 | assert!(safe_name("g1t-sandbox-wsl-x64.tar.gz").is_ok()); | |
| 167 | for bad in ["../x", "a/b", "a\\b", ".hidden", ""] { | |
| 168 | assert!(safe_name(bad).is_err(), "{bad}"); | |
| 169 | } | |
| 170 | } | |
| 171 | ||
| 172 | #[test] | |
| 173 | fn a_release_without_the_sandbox_is_refused() { | |
| 174 | let manifest: update::Manifest = serde_json::from_str(r#"{"version":"0.2.0","files":{}}"#).unwrap(); | |
| 175 | let refused = obtain("https://g1t.sh", &manifest, &["wsl-x64".into()], Path::new("."), &|_| {}).unwrap_err(); | |
| 176 | assert!(refused.to_string().contains("no sandbox images")); | |
| 177 | let manifest: update::Manifest = | |
| 178 | serde_json::from_str(r#"{"version":"0.3.0","files":{},"sandbox":{"runner_image":"r@sha256:1","files":{"wsl-x64":{"name":"w.tar.gz","sha256":"00"}}}}"#).unwrap(); | |
| 179 | let refused = obtain("https://g1t.sh", &manifest, &["vm-arm64-disk".into()], &std::env::temp_dir(), &|_| {}).unwrap_err(); | |
| 180 | assert!(refused.to_string().contains("no sandbox image vm-arm64-disk")); | |
| 181 | } | |
| 182 | ||
| 183 | #[test] | |
| 184 | fn a_local_vm_image_is_found_by_its_parts() { | |
| 185 | let dir = std::env::temp_dir().join(format!("g1t-sandbox-local-{}", std::process::id())); | |
| 186 | std::fs::create_dir_all(&dir).unwrap(); | |
| 187 | for name in ["g1t-sandbox-vm-arm64-kernel", "g1t-sandbox-vm-arm64-initrd", "g1t-sandbox-vm-arm64-disk.raw.gz"] { | |
| 188 | std::fs::write(dir.join(name), b"x").unwrap(); | |
| 189 | } | |
| 190 | let keys: Vec<String> = ["kernel", "initrd", "disk"].iter().map(|k| format!("vm-arm64-{k}")).collect(); | |
| 191 | let found = local(&keys, &dir, None).unwrap(); | |
| 192 | assert!(found.files["vm-arm64-disk"].to_string_lossy().ends_with("disk.raw.gz")); | |
| 193 | let _ = std::fs::remove_dir_all(&dir); | |
| 194 | } | |
| 195 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.