| 1 | //! `g1t-runner` on your own machine: a self-hosted runner for g1t. |
| 2 | //! |
| 3 | //! ```text |
| 4 | //! g1t-runner register --url https://g1t.sh --token g1trt_… [--name build-01] |
| 5 | //! [--labels gpu,cuda] [--group Default] [--ephemeral] [--work-dir DIR] |
| 6 | //! [--no-docker] [--image IMAGE] [--agent-image IMAGE] [--replace] |
| 7 | //! [--memory 8g] [--cpus 4] [--pids-limit 4096] [--no-job-docker] |
| 8 | //! g1t-runner run [--once] |
| 9 | //! g1t-runner service install|start|stop|status|uninstall |
| 10 | //! g1t-runner remove |
| 11 | //! g1t-runner update |
| 12 | //! g1t-runner version |
| 13 | //! ``` |
| 14 | //! |
| 15 | //! It only ever calls out, to `api.g1t.sh` over HTTPS: it polls for work |
| 16 | //! (a poll waits up to 20 seconds for some, and is the runner's heartbeat), |
| 17 | //! runs what it is given with the same harness g1t's sandboxes run (this |
| 18 | //! program in another mode, see `exec`), and says how it ended. Nothing |
| 19 | //! listens on the machine. Every command takes `--dir` (default |
| 20 | //! `~/.g1t-runner`, or `G1T_RUNNER_DIR`), where its configuration is kept. |
| 21 | //! |
| 22 | //! How it keeps jobs from the machine is `sandbox`: in its image under |
| 23 | //! Sysbox (`G1T_RUNNER_SANDBOX=nested`) it runs its own Docker Engine |
| 24 | //! (`engine`) and refuses to start if it can reach the machine's. |
| 25 | |
| 26 | mod api; |
| 27 | mod config; |
| 28 | #[cfg(unix)] |
| 29 | mod engine; |
| 30 | mod exec; |
| 31 | mod launcher; |
| 32 | mod native; |
| 33 | mod sandbox; |
| 34 | mod service; |
| 35 | mod update; |
| 36 | |
| 37 | use std::path::{Path, PathBuf}; |
| 38 | use std::sync::atomic::{AtomicBool, Ordering}; |
| 39 | use std::time::{Duration, Instant}; |
| 40 | |
| 41 | use anyhow::{Context, Result, bail}; |
| 42 | use serde_json::json; |
| 43 | |
| 44 | use api::{Api, Failure}; |
| 45 | use config::Config; |
| 46 | |
| 47 | pub const VERSION: &str = env!("CARGO_PKG_VERSION"); |
| 48 | |
| 49 | /// Set to stop the loop between polls: a Windows service's stop request. |
| 50 | pub static STOP: AtomicBool = AtomicBool::new(false); |
| 51 | |
| 52 | /// How long a busy runner waits between heartbeats. |
| 53 | const HEARTBEAT: Duration = Duration::from_secs(10); |
| 54 | /// How long an idle poll waits for work. |
| 55 | const IDLE_WAIT_MS: u64 = 20_000; |
| 56 | /// How often an idle runner checks for a new version. |
| 57 | const UPDATE_EVERY: Duration = Duration::from_secs(6 * 60 * 60); |
| 58 | |
| 59 | const COMMANDS: &[&str] = &["register", "register-and-run", "run", "service", "sandbox", "remove", "update", "version", "--version", "help", "--help", "-h"]; |
| 60 | |
| 61 | /// Whether `args` (without the program) are a command of this one, rather |
| 62 | /// than the harness's `MODE`. |
| 63 | pub fn is_command(args: &[String]) -> bool { |
| 64 | args.first().is_some_and(|first| COMMANDS.contains(&first.as_str())) |
| 65 | } |
| 66 | |
| 67 | pub fn log(message: &str) { |
| 68 | let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0); |
| 69 | println!("{} {message}", g1t_now(now)); |
| 70 | } |
| 71 | |
| 72 | /// RFC 3339 for the log, without pulling in a date library. |
| 73 | fn g1t_now(ms: u64) -> String { |
| 74 | let seconds = ms / 1000; |
| 75 | let (days, rest) = (seconds / 86_400, seconds % 86_400); |
| 76 | let z = days as i64 + 719_468; |
| 77 | let era = z.div_euclid(146_097); |
| 78 | let doe = z.rem_euclid(146_097); |
| 79 | let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; |
| 80 | let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); |
| 81 | let mp = (5 * doy + 2) / 153; |
| 82 | let day = doy - (153 * mp + 2) / 5 + 1; |
| 83 | let month = if mp < 10 { mp + 3 } else { mp - 9 }; |
| 84 | let year = yoe + era * 400 + i64::from(month <= 2); |
| 85 | format!("{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", rest / 3600, rest % 3600 / 60, rest % 60) |
| 86 | } |
| 87 | |
| 88 | /// `--name value` and `--flag` options. |
| 89 | struct Options { |
| 90 | values: Vec<(String, String)>, |
| 91 | flags: Vec<String>, |
| 92 | rest: Vec<String>, |
| 93 | } |
| 94 | |
| 95 | impl Options { |
| 96 | fn parse(args: &[String], takes_value: &[&str]) -> Result<Options> { |
| 97 | let mut options = Options { values: Vec::new(), flags: Vec::new(), rest: Vec::new() }; |
| 98 | let mut i = 0; |
| 99 | while i < args.len() { |
| 100 | let arg = &args[i]; |
| 101 | if let Some(name) = arg.strip_prefix("--") { |
| 102 | if let Some((name, value)) = name.split_once('=') { |
| 103 | options.values.push((name.to_owned(), value.to_owned())); |
| 104 | } else if takes_value.contains(&name) { |
| 105 | let Some(value) = args.get(i + 1) else { bail!("--{name} needs a value") }; |
| 106 | options.values.push((name.to_owned(), value.clone())); |
| 107 | i += 1; |
| 108 | } else { |
| 109 | options.flags.push(name.to_owned()); |
| 110 | } |
| 111 | } else { |
| 112 | options.rest.push(arg.clone()); |
| 113 | } |
| 114 | i += 1; |
| 115 | } |
| 116 | Ok(options) |
| 117 | } |
| 118 | |
| 119 | fn value(&self, name: &str) -> Option<&str> { |
| 120 | self.values.iter().rev().find(|(n, _)| n == name).map(|(_, v)| v.as_str()) |
| 121 | } |
| 122 | |
| 123 | fn flag(&self, name: &str) -> bool { |
| 124 | self.flags.iter().any(|f| f == name) |
| 125 | } |
| 126 | |
| 127 | /// For `register-and-run` in a container: each option may come from |
| 128 | /// `G1T_RUNNER_<NAME>` instead (`G1T_RUNNER_TOKEN`, `G1T_RUNNER_LABELS`, |
| 129 | /// `G1T_RUNNER_EPHEMERAL=1`), so a Kubernetes secret can hold the token. |
| 130 | fn with_environment(mut self) -> Options { |
| 131 | let env_name = |name: &str| format!("G1T_RUNNER_{}", name.to_ascii_uppercase().replace('-', "_")); |
| 132 | for name in VALUE_OPTIONS { |
| 133 | if self.value(name).is_none() |
| 134 | && let Ok(value) = std::env::var(env_name(name)) |
| 135 | && !value.is_empty() |
| 136 | { |
| 137 | self.values.push((name.to_owned(), value)); |
| 138 | } |
| 139 | } |
| 140 | for name in ["ephemeral", "no-docker", "replace", "no-auto-update", "no-job-docker", "native"] { |
| 141 | if !self.flag(name) && std::env::var(env_name(name)).is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes")) { |
| 142 | self.flags.push(name.to_owned()); |
| 143 | } |
| 144 | } |
| 145 | self |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | /// The options that take a value. |
| 150 | const VALUE_OPTIONS: [&str; 14] = |
| 151 | ["url", "api", "token", "name", "labels", "group", "work-dir", "image", "agent-image", "harness", "memory", "cpus", "pids-limit", "job-docker-image"]; |
| 152 | |
| 153 | const HELP: &str = "g1t-runner: a self-hosted runner for g1t. |
| 154 | |
| 155 | g1t-runner register --url https://g1t.sh --token <registration token> |
| 156 | [--name NAME] [--labels a,b] [--group GROUP] [--ephemeral] |
| 157 | [--work-dir DIR] [--no-docker] [--image IMAGE] [--agent-image IMAGE] |
| 158 | [--api URL] [--harness PATH] [--replace] [--no-auto-update] |
| 159 | [--memory 8g] [--cpus 4] [--pids-limit 4096] [--no-job-docker] |
| 160 | [--job-docker-image IMAGE] [--native] |
| 161 | g1t-runner run [--once] poll for work and run it |
| 162 | g1t-runner sandbox up|down|status|update [--json] |
| 163 | on your own computer: a Linux VM of the |
| 164 | runner's own for Linux jobs, and Windows |
| 165 | Sandbox for Windows jobs; see |
| 166 | `g1t-runner sandbox help` |
| 167 | g1t-runner register-and-run ... both, for containers; options may be |
| 168 | G1T_RUNNER_URL, _TOKEN, _LABELS, _EPHEMERAL… |
| 169 | g1t-runner service install|start|stop|status|uninstall |
| 170 | g1t-runner remove unregister this runner |
| 171 | g1t-runner update update to the newest release |
| 172 | g1t-runner version |
| 173 | |
| 174 | In a workflow step (the g1t/deploy and g1t/detect actions use these): |
| 175 | g1t-runner deploy build this checkout and put it on g1t.page |
| 176 | g1t-runner detect [--root-dir DIR] what that build would find |
| 177 | |
| 178 | Every command takes --dir DIR (default ~/.g1t-runner, or G1T_RUNNER_DIR). |
| 179 | Run it as a container under Sysbox so neither it nor its jobs reach the |
| 180 | machine: docker run --runtime=sysbox-runc … g1t.sh/flagon-io/g1t-runner, |
| 181 | or on a desktop, g1t-runner sandbox up. --native runs jobs native to this |
| 182 | OS, each in a throwaway VM (Windows Sandbox). --no-docker (host mode) is |
| 183 | deprecated. |
| 184 | Make a registration token under Settings, Runners on g1t.sh. |
| 185 | Docs: https://docs.g1t.sh/guides/self-hosted-runners/"; |
| 186 | |
| 187 | /// Runs a command. Returns the process's exit code. |
| 188 | pub fn main(args: Vec<String>) -> i32 { |
| 189 | // Options of its own, and output the desktop app reads. |
| 190 | if args[0] == "sandbox" { |
| 191 | return launcher::main(&args[1..]); |
| 192 | } |
| 193 | let mut takes = VALUE_OPTIONS.to_vec(); |
| 194 | takes.push("dir"); |
| 195 | let options = match Options::parse(&args[1..], &takes) { |
| 196 | Ok(options) => options, |
| 197 | Err(error) => { |
| 198 | eprintln!("g1t-runner: {error:#}"); |
| 199 | return 2; |
| 200 | } |
| 201 | }; |
| 202 | let folder = config::folder(options.value("dir")); |
| 203 | let outcome = match args[0].as_str() { |
| 204 | "register" => register(&options, &folder), |
| 205 | // A container's one command: register unless it already is, then run. |
| 206 | "register-and-run" => { |
| 207 | let options = options.with_environment(); |
| 208 | let ready = if config::load(&folder).is_ok() { Ok(()) } else { register(&options, &folder) }; |
| 209 | ready.and_then(|()| run(&folder, options.flag("once"))) |
| 210 | } |
| 211 | "run" => run(&folder, options.flag("once")), |
| 212 | "service" => match options.rest.first().map(String::as_str) { |
| 213 | Some(action) => config::load(&folder).and_then(|config| service::main(action, &config, &folder)), |
| 214 | None => Err(anyhow::anyhow!("service takes install, uninstall, start, stop or status")), |
| 215 | }, |
| 216 | "remove" => remove(&folder), |
| 217 | "update" => config::load(&folder).and_then(|config| match update::update(&config)? { |
| 218 | Some(version) => { |
| 219 | log(&format!("Updated to {version}. Restart the runner (or its service) to use it.")); |
| 220 | Ok(()) |
| 221 | } |
| 222 | None => { |
| 223 | log(&format!("{VERSION} is the newest release.")); |
| 224 | Ok(()) |
| 225 | } |
| 226 | }), |
| 227 | "version" | "--version" => { |
| 228 | println!("g1t-runner {VERSION} ({})", update::platform()); |
| 229 | Ok(()) |
| 230 | } |
| 231 | _ => { |
| 232 | println!("{HELP}"); |
| 233 | Ok(()) |
| 234 | } |
| 235 | }; |
| 236 | match outcome { |
| 237 | Ok(()) => 0, |
| 238 | Err(error) => { |
| 239 | eprintln!("g1t-runner: {error:#}"); |
| 240 | 1 |
| 241 | } |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | fn hostname() -> String { |
| 246 | for name in ["COMPUTERNAME", "HOSTNAME"] { |
| 247 | if let Ok(value) = std::env::var(name).map(|v| v.trim().to_owned()) |
| 248 | && !value.is_empty() |
| 249 | { |
| 250 | return value; |
| 251 | } |
| 252 | } |
| 253 | std::process::Command::new("hostname") |
| 254 | .output() |
| 255 | .ok() |
| 256 | .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_owned()) |
| 257 | .filter(|name| !name.is_empty()) |
| 258 | .unwrap_or_else(|| "runner".to_owned()) |
| 259 | } |
| 260 | |
| 261 | /// A runner's name from a machine's: what g1t accepts. |
| 262 | fn runner_name(given: &str) -> String { |
| 263 | let name: String = given |
| 264 | .chars() |
| 265 | .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') { c } else { '-' }) |
| 266 | .take(64) |
| 267 | .collect(); |
| 268 | if name.is_empty() { "runner".into() } else { name } |
| 269 | } |
| 270 | |
| 271 | fn register(options: &Options, folder: &Path) -> Result<()> { |
| 272 | let Some(url) = options.value("url") else { bail!("--url is needed: the g1t you register with, such as https://g1t.sh") }; |
| 273 | let Some(token) = options.value("token") else { bail!("--token is needed: make a registration token under Settings, Runners") }; |
| 274 | let url = url.trim_end_matches('/').to_owned(); |
| 275 | let api = options.value("api").map(|a| a.trim_end_matches('/').to_owned()).unwrap_or_else(|| config::api_for(&url)); |
| 276 | let name = runner_name(options.value("name").map(str::to_owned).unwrap_or_else(hostname).as_str()); |
| 277 | let labels: Vec<String> = options |
| 278 | .value("labels") |
| 279 | .unwrap_or_default() |
| 280 | .split(',') |
| 281 | .map(|l| l.trim().to_owned()) |
| 282 | .filter(|l| !l.is_empty()) |
| 283 | .collect(); |
| 284 | let native = options.flag("native"); |
| 285 | if native && options.flag("no-docker") { |
| 286 | bail!("--native runs each job in a VM of its own, and --no-docker runs it on this machine: pick one"); |
| 287 | } |
| 288 | let docker = !options.flag("no-docker") && !native; |
| 289 | let isolation = isolation(docker, native)?; |
| 290 | if isolation == sandbox::Isolation::Host { |
| 291 | log(sandbox::HOST_MODE_AT_START); |
| 292 | } else if !native && !sandbox::nested_requested() && !exec::docker_ready() { |
| 293 | log("Docker does not answer here. Jobs will fail until it does."); |
| 294 | } |
| 295 | let pids_limit = match options.value("pids-limit") { |
| 296 | Some(text) => Some(text.parse::<u32>().ok().filter(|n| *n > 0).with_context(|| format!("--pids-limit takes a number of processes, not {text}"))?), |
| 297 | None => None, |
| 298 | }; |
| 299 | let existing = config::load(folder).ok(); |
| 300 | if existing.is_some() && !options.flag("replace") { |
| 301 | bail!("{} already holds a registered runner: run `g1t-runner remove` first, or register with --replace or another --dir", folder.display()); |
| 302 | } |
| 303 | // The OS jobs run on: in Docker that is Linux, whatever the machine |
| 304 | // (Docker Desktop runs Linux containers on macOS and Windows). A |
| 305 | // runner says it runs jobs native to this OS only when it sandboxes |
| 306 | // them (--native), or when the person chose host mode (--no-docker). |
| 307 | let os = match std::env::consts::OS { |
| 308 | _ if docker => "linux", |
| 309 | "macos" => "macos", |
| 310 | "windows" => "windows", |
| 311 | _ => "linux", |
| 312 | }; |
| 313 | let arch = if std::env::consts::ARCH == "aarch64" { "arm64" } else { "x64" }; |
| 314 | let registered = Api::register( |
| 315 | &api, |
| 316 | &json!({ |
| 317 | "token": token, |
| 318 | "name": name, |
| 319 | "labels": labels, |
| 320 | "os": os, |
| 321 | "arch": arch, |
| 322 | "version": VERSION, |
| 323 | "ephemeral": options.flag("ephemeral"), |
| 324 | "group": options.value("group"), |
| 325 | "replace": options.flag("replace"), |
| 326 | "isolation": isolation.as_str(), |
| 327 | }), |
| 328 | )?; |
| 329 | let work_dir = options.value("work-dir").map(PathBuf::from).unwrap_or_else(|| folder.join("work")); |
| 330 | let config = Config { |
| 331 | url, |
| 332 | api, |
| 333 | runner: registered.runner.id.clone(), |
| 334 | name: registered.runner.name.clone(), |
| 335 | credential: registered.credential, |
| 336 | workspace: registered.runner.workspace.clone(), |
| 337 | repo: registered.runner.repo.clone(), |
| 338 | group: registered.runner.group.clone(), |
| 339 | labels: registered.runner.labels.clone(), |
| 340 | ephemeral: options.flag("ephemeral"), |
| 341 | work_dir, |
| 342 | docker, |
| 343 | image: options.value("image").map(str::to_owned), |
| 344 | agent_image: options.value("agent-image").map(str::to_owned), |
| 345 | harness: options.value("harness").map(PathBuf::from), |
| 346 | auto_update: !options.flag("no-auto-update"), |
| 347 | memory: options.value("memory").map(str::to_owned), |
| 348 | cpus: options.value("cpus").map(str::to_owned), |
| 349 | pids_limit, |
| 350 | job_docker: !options.flag("no-job-docker"), |
| 351 | job_docker_image: options.value("job-docker-image").map(str::to_owned), |
| 352 | native, |
| 353 | }; |
| 354 | config::save(folder, &config)?; |
| 355 | let place = match &config.repo { |
| 356 | Some(repo) => format!("for {repo}"), |
| 357 | None => format!("in {}{}", config.workspace, config.group.as_ref().map(|g| format!(", group {g}")).unwrap_or_default()), |
| 358 | }; |
| 359 | log(&format!("Registered {} {place}, with labels {}.", config.name, config.labels.join(", "))); |
| 360 | log("Start it with `g1t-runner run`, or `g1t-runner service install` to keep it running."); |
| 361 | Ok(()) |
| 362 | } |
| 363 | |
| 364 | /// How this runner keeps work from its machine; refuses a nested runner |
| 365 | /// that can reach the machine (sandbox.rs). |
| 366 | fn isolation(docker: bool, native: bool) -> Result<sandbox::Isolation> { |
| 367 | use sandbox::{Isolation, SandboxDriver}; |
| 368 | if native { |
| 369 | let Some(driver) = sandbox::native_driver() else { |
| 370 | bail!("--native: there is no sandbox for jobs native to {}; Linux jobs run in containers", std::env::consts::OS); |
| 371 | }; |
| 372 | return driver.preflight().with_context(|| format!("--native needs {}", driver.name())); |
| 373 | } |
| 374 | if !docker { |
| 375 | // What would sandbox jobs native to this OS instead. |
| 376 | if let Some(driver) = sandbox::native_driver() { |
| 377 | match driver.preflight() { |
| 378 | Ok(_) => log(&format!("This machine can run {} jobs sandboxed, each in {}: register with --native instead of --no-docker.", std::env::consts::OS, driver.name())), |
| 379 | Err(why) => log(&format!("Sandboxed {} jobs, in {}: {why:#}", std::env::consts::OS, driver.name())), |
| 380 | } |
| 381 | } |
| 382 | return Ok(Isolation::Host); |
| 383 | } |
| 384 | if !sandbox::nested_requested() { |
| 385 | return sandbox::SharedDocker.preflight(); |
| 386 | } |
| 387 | if !cfg!(target_os = "linux") { |
| 388 | bail!("{}=nested is for the runner's Linux image", sandbox::SANDBOX_ENV); |
| 389 | } |
| 390 | let isolation = sandbox::NestedDocker { allow_unsafe: sandbox::unsafe_allowed() }.preflight()?; |
| 391 | if isolation != Isolation::NestedContainer { |
| 392 | for problem in sandbox::Findings::read().problems() { |
| 393 | log(&format!("UNSAFE: {problem}")); |
| 394 | } |
| 395 | log(&format!("UNSAFE: {}=1 is set, so the runner starts anyway, and its jobs can reach this machine.", sandbox::UNSAFE_ENV)); |
| 396 | } |
| 397 | Ok(isolation) |
| 398 | } |
| 399 | |
| 400 | /// A nested runner's own Docker Engine, which it uses whatever else it |
| 401 | /// can see (with the override, it starts despite that). |
| 402 | fn prepare(docker: bool) -> Result<()> { |
| 403 | if !docker || !sandbox::nested_requested() { |
| 404 | return Ok(()); |
| 405 | } |
| 406 | #[cfg(unix)] |
| 407 | { |
| 408 | let version = engine::start()?; |
| 409 | log(&format!("Started this runner's own Docker Engine {version}, inside its container.")); |
| 410 | } |
| 411 | Ok(()) |
| 412 | } |
| 413 | |
| 414 | fn remove(folder: &Path) -> Result<()> { |
| 415 | let config = config::load(folder)?; |
| 416 | let api = Api { base: config.api.clone(), runner: config.runner.clone(), credential: config.credential.clone() }; |
| 417 | match api.remove() { |
| 418 | Ok(()) | Err(Failure::Refused(401, _)) | Err(Failure::Refused(404, _)) => {} |
| 419 | Err(failure) => bail!("{failure}; the runner is still registered"), |
| 420 | } |
| 421 | config::forget(folder); |
| 422 | log(&format!("Removed {}.", config.name)); |
| 423 | Ok(()) |
| 424 | } |
| 425 | |
| 426 | /// The loop, for a Windows service: its exit code. |
| 427 | #[cfg_attr(not(windows), allow(dead_code))] |
| 428 | pub fn serve_from_service() -> i32 { |
| 429 | let folder = config::folder(None); |
| 430 | let folder = std::env::args() |
| 431 | .collect::<Vec<_>>() |
| 432 | .windows(2) |
| 433 | .find(|pair| pair[0] == "--dir") |
| 434 | .map(|pair| PathBuf::from(&pair[1])) |
| 435 | .unwrap_or(folder); |
| 436 | match run(&folder, false) { |
| 437 | Ok(()) => 0, |
| 438 | Err(error) => { |
| 439 | eprintln!("g1t-runner: {error:#}"); |
| 440 | 1 |
| 441 | } |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | /// Sleeps up to `duration`, waking early when `stop` says so. |
| 446 | fn nap(duration: Duration, mut stop: impl FnMut() -> bool) { |
| 447 | let until = Instant::now() + duration; |
| 448 | while Instant::now() < until { |
| 449 | if STOP.load(Ordering::SeqCst) || stop() { |
| 450 | return; |
| 451 | } |
| 452 | std::thread::sleep(Duration::from_millis(250)); |
| 453 | } |
| 454 | } |
| 455 | |
| 456 | fn run(folder: &Path, once: bool) -> Result<()> { |
| 457 | let mut config = config::load(folder)?; |
| 458 | let mut api = Api { base: config.api.clone(), runner: config.runner.clone(), credential: config.credential.clone() }; |
| 459 | std::fs::create_dir_all(&config.work_dir).with_context(|| format!("could not make {}", config.work_dir.display()))?; |
| 460 | let isolation = isolation(config.docker, config.native)?; |
| 461 | prepare(config.docker)?; |
| 462 | match isolation { |
| 463 | sandbox::Isolation::Host => {} |
| 464 | sandbox::Isolation::Vm => native::sweep(folder), |
| 465 | _ => exec::sweep(&config), |
| 466 | } |
| 467 | // A drain or stop asked for before this start is done with. |
| 468 | let _ = std::fs::remove_file(drain_path(folder)); |
| 469 | let _ = std::fs::remove_file(folder.join("stop")); |
| 470 | log(&format!( |
| 471 | "g1t-runner {VERSION}: {} ({}) is listening for work from {}. Isolation: {}.", |
| 472 | config.name, |
| 473 | config.labels.join(", "), |
| 474 | config.url, |
| 475 | isolation.describe() |
| 476 | )); |
| 477 | if isolation == sandbox::Isolation::Host { |
| 478 | log(sandbox::HOST_MODE_AT_START); |
| 479 | } |
| 480 | let mut current: Option<exec::Running> = None; |
| 481 | let mut ran_one = false; |
| 482 | let mut backoff = Duration::from_secs(2); |
| 483 | let mut last_update_check = Instant::now().checked_sub(UPDATE_EVERY).unwrap_or_else(Instant::now); |
| 484 | let mut said_no_key = false; |
| 485 | loop { |
| 486 | // Asked to stop now (`g1t-runner sandbox down`): as a service's stop. |
| 487 | if std::fs::remove_file(folder.join("stop")).is_ok() { |
| 488 | STOP.store(true, Ordering::SeqCst); |
| 489 | } |
| 490 | if STOP.load(Ordering::SeqCst) { |
| 491 | if let Some(mut running) = current.take() { |
| 492 | log(&format!("Stopping: {} is cut short.", running.name)); |
| 493 | running.stop(); |
| 494 | let _ = api.finished(&running.id, 1, Some("The self-hosted runner was stopped.")); |
| 495 | running.clean(); |
| 496 | } |
| 497 | say_status(folder, isolation, None, "stopped"); |
| 498 | return Ok(()); |
| 499 | } |
| 500 | // What was running has ended: say how. |
| 501 | if let Some(running) = current.as_mut() |
| 502 | && let Some(code) = running.ended() |
| 503 | { |
| 504 | let mut running = current.take().expect("checked above"); |
| 505 | log(&format!("{} {} ended ({code}).", running.kind, running.name)); |
| 506 | if let Err(failure) = api.finished(&running.id, code, None) { |
| 507 | log(&format!("Could not say how it ended: {failure}")); |
| 508 | } |
| 509 | running.clean(); |
| 510 | ran_one = true; |
| 511 | if config.ephemeral || once { |
| 512 | if config.ephemeral { |
| 513 | let _ = api.remove(); |
| 514 | config::forget(folder); |
| 515 | log("The ephemeral runner ran its job and removed itself."); |
| 516 | } |
| 517 | return Ok(()); |
| 518 | } |
| 519 | } |
| 520 | // Asked to drain (`g1t-runner sandbox update`, before it swaps the |
| 521 | // sandbox's image): no new work, and once idle, stop. |
| 522 | let draining = drain_path(folder).exists(); |
| 523 | if draining && current.is_none() { |
| 524 | let _ = std::fs::remove_file(drain_path(folder)); |
| 525 | say_status(folder, isolation, None, "stopped"); |
| 526 | log("Drained: no job is running, so the runner stops here."); |
| 527 | return Ok(()); |
| 528 | } |
| 529 | let state = match (¤t, draining) { |
| 530 | (Some(_), true) => "draining", |
| 531 | (Some(_), false) => "busy", |
| 532 | (None, _) => "idle", |
| 533 | }; |
| 534 | say_status(folder, isolation, current.as_ref(), state); |
| 535 | // Idle: a newer release, now and then. |
| 536 | if current.is_none() && config.auto_update && last_update_check.elapsed() >= UPDATE_EVERY { |
| 537 | last_update_check = Instant::now(); |
| 538 | if update::can_update() { |
| 539 | match update::update(&config) { |
| 540 | Ok(Some(version)) => { |
| 541 | log(&format!("Updated to {version}; starting it.")); |
| 542 | return restart(); |
| 543 | } |
| 544 | Ok(None) => {} |
| 545 | Err(error) => log(&format!("Could not check for a new version: {error:#}")), |
| 546 | } |
| 547 | } else if !said_no_key { |
| 548 | said_no_key = true; |
| 549 | log("This build cannot check releases, so it will not update itself."); |
| 550 | } |
| 551 | } |
| 552 | let running: Vec<String> = current.iter().map(|r| r.id.clone()).collect(); |
| 553 | let wait = if current.is_some() { 0 } else { IDLE_WAIT_MS }; |
| 554 | let poll = match api.poll(&running, wait, isolation.as_str()) { |
| 555 | Ok(poll) => { |
| 556 | backoff = Duration::from_secs(2); |
| 557 | poll |
| 558 | } |
| 559 | Err(Failure::Refused(401, message)) => { |
| 560 | if let Some(mut running) = current.take() { |
| 561 | running.stop(); |
| 562 | running.clean(); |
| 563 | } |
| 564 | bail!("g1t no longer knows this runner ({message}). Register it again."); |
| 565 | } |
| 566 | Err(failure) => { |
| 567 | log(&format!("{failure}; trying again in {}s", backoff.as_secs())); |
| 568 | nap(backoff, || false); |
| 569 | backoff = (backoff * 2).min(Duration::from_secs(60)); |
| 570 | continue; |
| 571 | } |
| 572 | }; |
| 573 | if let Some(fresh) = poll.credential { |
| 574 | config.credential = fresh.clone(); |
| 575 | api.credential = fresh; |
| 576 | config::save(folder, &config)?; |
| 577 | } |
| 578 | if poll.removed { |
| 579 | config::forget(folder); |
| 580 | log("This runner was removed from g1t; stopping."); |
| 581 | return Ok(()); |
| 582 | } |
| 583 | if let Some(running) = current.as_mut() |
| 584 | && poll.cancel.contains(&running.id) |
| 585 | { |
| 586 | log(&format!("{} was cancelled; stopping it.", running.name)); |
| 587 | running.stop(); |
| 588 | } |
| 589 | if let Some(running) = current.as_mut() |
| 590 | && running.overdue() |
| 591 | { |
| 592 | log(&format!("{} ran past its time limit; stopping it.", running.name)); |
| 593 | running.stop(); |
| 594 | } |
| 595 | if let Some(work) = poll.assignment { |
| 596 | log(&format!("Took {} {} ({}).", work.kind, work.name, work.repo)); |
| 597 | match exec::start(&config, isolation, folder, &work) { |
| 598 | Ok(running) => current = Some(running), |
| 599 | Err(error) => { |
| 600 | log(&format!("Could not start it: {error:#}")); |
| 601 | let _ = api.finished(&work.id, 1, Some(&format!("The self-hosted runner {} could not start it: {error:#}", config.name))); |
| 602 | if config.ephemeral || once { |
| 603 | return Ok(()); |
| 604 | } |
| 605 | } |
| 606 | } |
| 607 | continue; |
| 608 | } |
| 609 | if once && ran_one { |
| 610 | return Ok(()); |
| 611 | } |
| 612 | // Busy: a heartbeat every few seconds until it ends. |
| 613 | if current.is_some() { |
| 614 | let mut check = || current.as_mut().is_some_and(|r| r.ended().is_some()); |
| 615 | nap(HEARTBEAT, &mut check); |
| 616 | } |
| 617 | } |
| 618 | } |
| 619 | |
| 620 | /// The file whose presence asks a runner to drain: take no new work, and |
| 621 | /// stop once idle. |
| 622 | pub fn drain_path(folder: &Path) -> PathBuf { |
| 623 | folder.join("drain") |
| 624 | } |
| 625 | |
| 626 | /// The file a runner keeps saying what it is doing, for `g1t-runner |
| 627 | /// sandbox status` (which reads it from inside the sandbox). |
| 628 | pub fn status_path(folder: &Path) -> PathBuf { |
| 629 | folder.join("status.json") |
| 630 | } |
| 631 | |
| 632 | /// What a runner says it is doing, in its status file. |
| 633 | #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] |
| 634 | pub struct RunnerStatus { |
| 635 | pub version: String, |
| 636 | pub isolation: String, |
| 637 | /// `idle`, `busy`, `draining` or `stopped`. |
| 638 | pub state: String, |
| 639 | pub job: Option<JobStatus>, |
| 640 | pub updated_at: String, |
| 641 | } |
| 642 | |
| 643 | #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize, PartialEq)] |
| 644 | pub struct JobStatus { |
| 645 | /// `workflow` or `agent`. |
| 646 | pub kind: String, |
| 647 | pub id: String, |
| 648 | pub name: String, |
| 649 | pub repo: String, |
| 650 | } |
| 651 | |
| 652 | fn say_status(folder: &Path, isolation: sandbox::Isolation, running: Option<&exec::Running>, state: &str) { |
| 653 | let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0); |
| 654 | let status = RunnerStatus { |
| 655 | version: VERSION.to_owned(), |
| 656 | isolation: isolation.as_str().to_owned(), |
| 657 | state: state.to_owned(), |
| 658 | job: running.map(|r| JobStatus { kind: r.kind.clone(), id: r.id.clone(), name: r.name.clone(), repo: r.repo.clone() }), |
| 659 | updated_at: g1t_now(now), |
| 660 | }; |
| 661 | let temp = folder.join("status.json.new"); |
| 662 | if serde_json::to_vec_pretty(&status).ok().is_some_and(|bytes| std::fs::write(&temp, bytes).is_ok()) { |
| 663 | let _ = std::fs::rename(&temp, status_path(folder)); |
| 664 | } |
| 665 | } |
| 666 | |
| 667 | /// Runs the updated program in this one's place, with the same arguments. |
| 668 | fn restart() -> Result<()> { |
| 669 | let me = std::env::current_exe()?; |
| 670 | let status = std::process::Command::new(me).args(std::env::args().skip(1)).status()?; |
| 671 | std::process::exit(status.code().unwrap_or(1)); |
| 672 | } |
| 673 | |
| 674 | #[cfg(test)] |
| 675 | mod tests { |
| 676 | use super::*; |
| 677 | |
| 678 | fn args(items: &[&str]) -> Vec<String> { |
| 679 | items.iter().map(|s| s.to_string()).collect() |
| 680 | } |
| 681 | |
| 682 | #[test] |
| 683 | fn commands_are_told_apart_from_the_harness() { |
| 684 | assert!(is_command(&args(&["register", "--url", "x"]))); |
| 685 | assert!(is_command(&args(&["run"]))); |
| 686 | assert!(!is_command(&args(&[]))); |
| 687 | assert!(!is_command(&args(&["something"]))); |
| 688 | } |
| 689 | |
| 690 | #[test] |
| 691 | fn options_read_both_ways() { |
| 692 | let options = Options::parse(&args(&["--url", "https://g1t.sh", "--labels=gpu,cuda", "--ephemeral", "install"]), &["url", "labels"]).unwrap(); |
| 693 | assert_eq!(options.value("url"), Some("https://g1t.sh")); |
| 694 | assert_eq!(options.value("labels"), Some("gpu,cuda")); |
| 695 | assert!(options.flag("ephemeral")); |
| 696 | assert_eq!(options.rest, vec!["install".to_owned()]); |
| 697 | assert!(Options::parse(&args(&["--url"]), &["url"]).is_err()); |
| 698 | } |
| 699 | |
| 700 | #[test] |
| 701 | fn a_machines_name_becomes_a_runners() { |
| 702 | assert_eq!(runner_name("Build Box #1"), "Build-Box--1"); |
| 703 | assert_eq!(runner_name(""), "runner"); |
| 704 | assert_eq!(runner_name(&"x".repeat(80)).len(), 64); |
| 705 | } |
| 706 | |
| 707 | #[test] |
| 708 | fn the_log_says_when() { |
| 709 | assert_eq!(g1t_now(1_790_918_179_123), "2026-10-02T05:16:19Z"); |
| 710 | } |
| 711 | } |