| 1 | //! Self-hosted runners: registering them, handing them work, hearing back, |
| 2 | //! and the people's side (listing, removing, groups, settings). See |
| 3 | //! `g1t_contracts::runners` for the model and the wire. |
| 4 | //! |
| 5 | //! Work reaches a runner only when it asks for it (`runner_poll`), so a |
| 6 | //! runner never needs an open port. A workflow job is taken by flipping its |
| 7 | //! row from `queued` to `in_progress` in one statement, exactly as g1t's |
| 8 | //! own sandboxes take one, and the harness on the machine then fetches it |
| 9 | //! and reports with the job's own token through the same API. Agent work |
| 10 | //! from the runner service is kept as a task with its environment sealed, |
| 11 | //! and handed over once. |
| 12 | |
| 13 | use g1t_contracts::access::Capability; |
| 14 | use g1t_contracts::audit::{AuditActor, AuditOutcome, AuditTarget, NewAuditEntry, RecordAuditArgs, Surface}; |
| 15 | use g1t_contracts::billing::{ComputeKind, RecordSandboxArgs}; |
| 16 | use g1t_contracts::repos::{Repo, RepoPath}; |
| 17 | use g1t_contracts::runners::{ |
| 18 | self as model, ACTIVITY_LIMIT, CloudJob, HandedOverTask, RunnerActivity, RunnerActivityArgs, StuckJob, StuckJobsArgs, Assignment, CancelTaskArgs, CreateRegistrationTokenArgs, DeleteRunnerGroupArgs, EnqueueTaskArgs, FinishedArgs, |
| 19 | ListRunnersArgs, Poll, PollArgs, RegisterArgs, Registered, RegistrationToken, RemoveRunnerArgs, RemoveSelfArgs, RouteArgs, Runner, |
| 20 | RunnerAuth, RunnerGroup, RunnerGroupsArgs, RunnerSettings, RunnerSettingsArgs, RunnerWork, RunnersOwner, SetRunnerGroupArgs, |
| 21 | SetRunnerSettingsArgs, Wanted, |
| 22 | }; |
| 23 | use g1t_contracts::time::rfc3339; |
| 24 | use g1t_contracts::{FailureCode, Outcome, PrincipalKind, Role, User, new_id}; |
| 25 | use g1t_kit::now_ms; |
| 26 | use g1t_secrets::{random_hex, sha256_hex}; |
| 27 | use serde::Deserialize; |
| 28 | use serde_json::{Map, Value, json}; |
| 29 | use worker::Result; |
| 30 | use worker::wasm_bindgen::JsValue; |
| 31 | |
| 32 | use crate::plan::JobRow; |
| 33 | use crate::{Actions, Count, SILENT_MS, check, fail, optional}; |
| 34 | |
| 35 | /// What one poll has learned while it looks for work, so its looks again |
| 36 | /// do not ask again: each repository's visibility, and the jobs it already |
| 37 | /// said wait for a sandboxed runner. |
| 38 | #[derive(Default)] |
| 39 | struct Looked { |
| 40 | visibilities: std::collections::HashMap<String, Option<String>>, |
| 41 | noted: std::collections::HashSet<String>, |
| 42 | } |
| 43 | |
| 44 | /// How often a poll looks again for work while it waits. |
| 45 | const POLL_EVERY_MS: u64 = 2_500; |
| 46 | /// A runner that has been offline this long is removed, as on GitHub. |
| 47 | const FORGET_OFFLINE_MS: u64 = 14 * 24 * 60 * 60 * 1000; |
| 48 | /// How often a poll writes down that a runner, or the work it holds, is |
| 49 | /// still there. Polls come every twenty seconds or so; a write each time |
| 50 | /// would be most of the service's writes for nothing. Well inside |
| 51 | /// [`model::ONLINE_WITHIN_MS`] and [`crate::SILENT_MS`]. |
| 52 | const SEEN_RESOLUTION_MS: u64 = 60 * 1000; |
| 53 | |
| 54 | /// Whether `seen` is old enough, at `at`, to be written down again. |
| 55 | fn seen_is_stale(seen: Option<&str>, at: u64) -> bool { |
| 56 | seen.is_none_or(|seen| seen < rfc3339(at.saturating_sub(SEEN_RESOLUTION_MS)).as_str()) |
| 57 | } |
| 58 | |
| 59 | #[derive(Clone, Debug, Deserialize)] |
| 60 | pub struct RunnerRow { |
| 61 | pub id: String, |
| 62 | pub workspace: String, |
| 63 | pub repo_id: Option<String>, |
| 64 | pub repo: Option<String>, |
| 65 | pub group_id: Option<String>, |
| 66 | pub name: String, |
| 67 | pub labels: String, |
| 68 | pub os: String, |
| 69 | pub arch: String, |
| 70 | pub version: String, |
| 71 | pub ephemeral: u32, |
| 72 | pub credential_hash: String, |
| 73 | pub previous_hash: Option<String>, |
| 74 | pub rotated_at: String, |
| 75 | pub work_id: Option<String>, |
| 76 | pub work_kind: Option<String>, |
| 77 | pub spent: u32, |
| 78 | pub last_seen_at: Option<String>, |
| 79 | pub created_at: String, |
| 80 | pub created_by: Option<String>, |
| 81 | /// `nested-container`, `container` or `host` (migration 0015); none |
| 82 | /// for a runner that never said. |
| 83 | #[serde(default)] |
| 84 | pub isolation: Option<String>, |
| 85 | } |
| 86 | |
| 87 | impl RunnerRow { |
| 88 | fn labels(&self) -> Vec<String> { |
| 89 | serde_json::from_str(&self.labels).unwrap_or_default() |
| 90 | } |
| 91 | |
| 92 | fn online(&self, now: u64) -> bool { |
| 93 | self.last_seen_at.as_deref().is_some_and(|seen| seen >= rfc3339(now.saturating_sub(model::ONLINE_WITHIN_MS)).as_str()) |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | #[derive(Clone, Debug, Deserialize)] |
| 98 | struct GroupRow { |
| 99 | id: String, |
| 100 | name: String, |
| 101 | is_default: u32, |
| 102 | repositories: String, |
| 103 | updated_at: String, |
| 104 | } |
| 105 | |
| 106 | impl GroupRow { |
| 107 | fn repositories(&self) -> Vec<String> { |
| 108 | serde_json::from_str(&self.repositories).unwrap_or_default() |
| 109 | } |
| 110 | |
| 111 | /// Whether `repo` (`owner/name`, or a bare name) may use the group. |
| 112 | fn allows(&self, repo: &str) -> bool { |
| 113 | let name = repo.rsplit('/').next().unwrap_or(repo); |
| 114 | let listed = self.repositories(); |
| 115 | listed.is_empty() || listed.iter().any(|r| r.eq_ignore_ascii_case(name)) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | #[derive(Clone, Debug, Deserialize)] |
| 120 | struct RegistrationRow { |
| 121 | workspace: String, |
| 122 | repo_id: Option<String>, |
| 123 | repo: Option<String>, |
| 124 | group_id: Option<String>, |
| 125 | expires_at: String, |
| 126 | } |
| 127 | |
| 128 | #[derive(Clone, Debug, Deserialize)] |
| 129 | struct SettingsRow { |
| 130 | agents: u32, |
| 131 | agent_labels: String, |
| 132 | fork_pulls: u32, |
| 133 | } |
| 134 | |
| 135 | #[derive(Clone, Debug, Deserialize)] |
| 136 | pub struct TaskRow { |
| 137 | pub id: String, |
| 138 | pub sandbox: String, |
| 139 | pub repo_id: Option<String>, |
| 140 | pub repo: String, |
| 141 | pub kind: String, |
| 142 | pub title: String, |
| 143 | pub labels: String, |
| 144 | pub env: Option<String>, |
| 145 | pub timeout_minutes: u32, |
| 146 | pub status: String, |
| 147 | pub runner_id: Option<String>, |
| 148 | pub runner_name: Option<String>, |
| 149 | pub started_at: Option<String>, |
| 150 | #[serde(default)] |
| 151 | pub seen_at: Option<String>, |
| 152 | /// The agent run it belongs to (migration 0012); none for older tasks. |
| 153 | #[serde(default)] |
| 154 | pub run_id: Option<String>, |
| 155 | } |
| 156 | |
| 157 | /// Who may do what with a set of runners: the workspace, and the |
| 158 | /// repository when they are a repository's own. |
| 159 | struct Place { |
| 160 | workspace: String, |
| 161 | repo: Option<Repo>, |
| 162 | } |
| 163 | |
| 164 | fn now() -> String { |
| 165 | rfc3339(now_ms()) |
| 166 | } |
| 167 | |
| 168 | fn valid_name(name: &str) -> std::result::Result<String, String> { |
| 169 | let name = name.trim(); |
| 170 | let ok = !name.is_empty() |
| 171 | && name.len() <= 64 |
| 172 | && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')); |
| 173 | if ok { |
| 174 | Ok(name.to_owned()) |
| 175 | } else { |
| 176 | Err("A runner's name is 1 to 64 letters, digits, `-`, `_` and `.`.".to_owned()) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | fn valid_group_name(name: &str) -> std::result::Result<String, String> { |
| 181 | let name = name.trim(); |
| 182 | if name.is_empty() || name.len() > 64 || name.chars().any(|c| c.is_control()) { |
| 183 | return Err("A group's name is 1 to 64 characters.".to_owned()); |
| 184 | } |
| 185 | Ok(name.to_owned()) |
| 186 | } |
| 187 | |
| 188 | /// What a runner's credential is checked against, and what a registration |
| 189 | /// token is found by. |
| 190 | fn hash(secret: &str) -> String { |
| 191 | sha256_hex(secret) |
| 192 | } |
| 193 | |
| 194 | /// The actor of a runner's own entries in the audit log. |
| 195 | fn runner_actor(runner: &RunnerRow) -> AuditActor { |
| 196 | AuditActor { |
| 197 | actor_kind: Some(g1t_contracts::audit::ActorKind::Runner), |
| 198 | actor: runner.name.clone(), |
| 199 | actor_id: runner.id.clone(), |
| 200 | ..AuditActor::default() |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | fn entry(actor: AuditActor, action: &str, workspace: &str, repo: Option<String>, path: Option<String>, message: String) -> NewAuditEntry { |
| 205 | NewAuditEntry { |
| 206 | actor, |
| 207 | action: action.to_owned(), |
| 208 | surface: Surface::Rest, |
| 209 | target: AuditTarget { |
| 210 | workspace: workspace.to_owned(), |
| 211 | repo, |
| 212 | path, |
| 213 | ..AuditTarget::default() |
| 214 | }, |
| 215 | outcome: AuditOutcome::Allowed, |
| 216 | rule: "runner".to_owned(), |
| 217 | result: Some("ok".to_owned()), |
| 218 | message: Some(message), |
| 219 | request_id: new_id("req", now_ms()), |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | impl Actions { |
| 224 | // --- Shared ------------------------------------------------------------ |
| 225 | |
| 226 | pub(crate) async fn audit(&self, entries: Vec<NewAuditEntry>) { |
| 227 | let recorded: Result<u32> = g1t_kit::call(&self.events, "audit_record", &RecordAuditArgs { entries }).await; |
| 228 | if let Err(error) = recorded { |
| 229 | worker::console_error!("actions: audit entries not recorded: {error}"); |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | /// Where `owner` points, if `actor` may see it (`manage` false) or |
| 234 | /// change it (`manage` true): a workspace's runners are seen by its |
| 235 | /// members and changed by its owners; a repository's own, by its |
| 236 | /// admins. Agents and workspace tokens never change them, so a |
| 237 | /// workflow's `G1T_TOKEN` cannot add a machine to run its own jobs. |
| 238 | async fn runner_place(&self, actor: &User, owner: &RunnersOwner, manage: bool) -> Result<Outcome<Place>> { |
| 239 | if actor.kind == PrincipalKind::Agent { |
| 240 | return Ok(fail(FailureCode::Forbidden, "Agents cannot see or change self-hosted runners.")); |
| 241 | } |
| 242 | if manage && actor.kind == PrincipalKind::Workspace { |
| 243 | return Ok(fail( |
| 244 | FailureCode::Forbidden, |
| 245 | "A workspace's tokens, G1T_TOKEN included, cannot change self-hosted runners. Use a person's token with the runners:admin scope.", |
| 246 | )); |
| 247 | } |
| 248 | match (&owner.repo, &owner.workspace) { |
| 249 | (Some(path), _) => { |
| 250 | let repo = check!(self.may(actor, path, Capability::ManageIntegrations).await?); |
| 251 | Ok(Outcome::Ok(Place { workspace: repo.namespace.to_lowercase(), repo: Some(repo) })) |
| 252 | } |
| 253 | (None, Some(slug)) => { |
| 254 | let slug = slug.to_lowercase(); |
| 255 | let role = actor.workspaces.iter().find(|m| m.slug.eq_ignore_ascii_case(&slug)).map(|m| m.role); |
| 256 | match role { |
| 257 | None => Ok(fail(FailureCode::NotFound, "There is no such workspace, or you are not a member of it.")), |
| 258 | // A workspace's machines and their tokens are its owners' alone, to see as well as change. |
| 259 | Some(Role::Member) => Ok(fail( |
| 260 | FailureCode::Forbidden, |
| 261 | format!("Only owners of {slug} can {} its self-hosted runners.", if manage { "change" } else { "see" }), |
| 262 | )), |
| 263 | Some(_) => Ok(Outcome::Ok(Place { workspace: slug, repo: None })), |
| 264 | } |
| 265 | } |
| 266 | (None, None) => Ok(fail(FailureCode::Invalid, "Give `repo` or `workspace`.")), |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | async fn group_rows(&self, workspace: &str) -> Result<Vec<GroupRow>> { |
| 271 | self.db |
| 272 | .prepare("SELECT * FROM runner_groups WHERE workspace = ? ORDER BY is_default DESC, name") |
| 273 | .bind(&[workspace.into()])? |
| 274 | .all() |
| 275 | .await? |
| 276 | .results::<GroupRow>() |
| 277 | } |
| 278 | |
| 279 | /// The workspace's default group, made the first time it is needed. |
| 280 | async fn default_group(&self, workspace: &str) -> Result<GroupRow> { |
| 281 | let at = now(); |
| 282 | self.db |
| 283 | .prepare( |
| 284 | "INSERT OR IGNORE INTO runner_groups (id, workspace, name, is_default, repositories, created_at, updated_at) |
| 285 | VALUES (?, ?, 'Default', 1, '[]', ?, ?)", |
| 286 | ) |
| 287 | .bind(&[new_id("rng", now_ms()).into(), workspace.into(), at.as_str().into(), at.as_str().into()])? |
| 288 | .run() |
| 289 | .await?; |
| 290 | let row = self |
| 291 | .db |
| 292 | .prepare("SELECT * FROM runner_groups WHERE workspace = ? AND is_default = 1 LIMIT 1") |
| 293 | .bind(&[workspace.into()])? |
| 294 | .first::<GroupRow>(None) |
| 295 | .await?; |
| 296 | row.ok_or_else(|| worker::Error::RustError("the default runner group could not be made".into())) |
| 297 | } |
| 298 | |
| 299 | /// A group of the workspace by id or name. |
| 300 | async fn find_group(&self, workspace: &str, wanted: &str) -> Result<Option<GroupRow>> { |
| 301 | self.default_group(workspace).await?; |
| 302 | Ok(self |
| 303 | .group_rows(workspace) |
| 304 | .await? |
| 305 | .into_iter() |
| 306 | .find(|g| g.id == wanted || g.name.eq_ignore_ascii_case(wanted.trim()))) |
| 307 | } |
| 308 | |
| 309 | async fn runner_rows(&self, workspace: &str) -> Result<Vec<RunnerRow>> { |
| 310 | self.db |
| 311 | .prepare("SELECT * FROM runners WHERE workspace = ? ORDER BY name") |
| 312 | .bind(&[workspace.into()])? |
| 313 | .all() |
| 314 | .await? |
| 315 | .results::<RunnerRow>() |
| 316 | } |
| 317 | |
| 318 | async fn runner_by_id(&self, id: &str) -> Result<Option<RunnerRow>> { |
| 319 | self.db.prepare("SELECT * FROM runners WHERE id = ?").bind(&[id.into()])?.first::<RunnerRow>(None).await |
| 320 | } |
| 321 | |
| 322 | /// What a runner is doing, for people. |
| 323 | async fn work_of(&self, row: &RunnerRow) -> Result<Option<RunnerWork>> { |
| 324 | let Some(id) = &row.work_id else { return Ok(None) }; |
| 325 | if row.work_kind.as_deref() == Some("agent") { |
| 326 | let task = self.db.prepare("SELECT * FROM runner_tasks WHERE id = ?").bind(&[id.as_str().into()])?.first::<TaskRow>(None).await?; |
| 327 | return Ok(task.filter(|t| t.status == "in_progress").map(|t| RunnerWork { |
| 328 | kind: "agent".into(), |
| 329 | id: t.id, |
| 330 | name: t.title, |
| 331 | repo: Some(t.repo), |
| 332 | run_id: t.run_id, |
| 333 | started_at: t.started_at, |
| 334 | })); |
| 335 | } |
| 336 | #[derive(Deserialize)] |
| 337 | struct Found { |
| 338 | id: String, |
| 339 | name: String, |
| 340 | run_id: String, |
| 341 | status: String, |
| 342 | started_at: Option<String>, |
| 343 | repo: String, |
| 344 | } |
| 345 | let job = self |
| 346 | .db |
| 347 | .prepare("SELECT jobs.id, jobs.name, jobs.run_id, jobs.status, jobs.started_at, runs.repo FROM jobs JOIN runs ON runs.id = jobs.run_id WHERE jobs.id = ?") |
| 348 | .bind(&[id.as_str().into()])? |
| 349 | .first::<Found>(None) |
| 350 | .await?; |
| 351 | Ok(job.filter(|j| j.status == "in_progress").map(|j| RunnerWork { |
| 352 | kind: "workflow".into(), |
| 353 | id: j.id, |
| 354 | name: j.name, |
| 355 | repo: Some(j.repo), |
| 356 | run_id: Some(j.run_id), |
| 357 | started_at: j.started_at, |
| 358 | })) |
| 359 | } |
| 360 | |
| 361 | async fn runner_view(&self, row: &RunnerRow, groups: &[GroupRow], at: u64) -> Result<Runner> { |
| 362 | let online = row.online(at); |
| 363 | let work = if online { self.work_of(row).await? } else { None }; |
| 364 | Ok(Runner { |
| 365 | id: row.id.clone(), |
| 366 | name: row.name.clone(), |
| 367 | workspace: row.workspace.clone(), |
| 368 | repo: row.repo.clone(), |
| 369 | group: row |
| 370 | .group_id |
| 371 | .as_ref() |
| 372 | .and_then(|id| groups.iter().find(|g| &g.id == id)) |
| 373 | .map(|g| g.name.clone()), |
| 374 | labels: row.labels(), |
| 375 | os: row.os.clone(), |
| 376 | arch: row.arch.clone(), |
| 377 | version: row.version.clone(), |
| 378 | ephemeral: row.ephemeral != 0, |
| 379 | status: if !online { |
| 380 | "offline" |
| 381 | } else if work.is_some() { |
| 382 | "busy" |
| 383 | } else { |
| 384 | "online" |
| 385 | } |
| 386 | .to_owned(), |
| 387 | work, |
| 388 | last_seen_at: row.last_seen_at.clone(), |
| 389 | created_at: row.created_at.clone(), |
| 390 | created_by: row.created_by.clone(), |
| 391 | isolation: row.isolation.clone(), |
| 392 | }) |
| 393 | } |
| 394 | |
| 395 | // --- People's side ----------------------------------------------------- |
| 396 | |
| 397 | /// `runners`: a workspace's runners, or a repository's: its own and |
| 398 | /// the workspace's that its group lets it use. |
| 399 | pub async fn runners(&self, a: ListRunnersArgs) -> Result<Outcome<Vec<Runner>>> { |
| 400 | let place = check!(self.runner_place(&a.actor, &a.owner, false).await?); |
| 401 | self.default_group(&place.workspace).await?; |
| 402 | let groups = self.group_rows(&place.workspace).await?; |
| 403 | let at = now_ms(); |
| 404 | let mut out = Vec::new(); |
| 405 | for row in self.runner_rows(&place.workspace).await? { |
| 406 | let shown = match &place.repo { |
| 407 | None => row.repo_id.is_none(), |
| 408 | Some(repo) => match &row.repo_id { |
| 409 | Some(id) => id == &repo.id, |
| 410 | None => groups.iter().find(|g| Some(&g.id) == row.group_id.as_ref()).is_none_or(|g| g.allows(&repo.name)), |
| 411 | }, |
| 412 | }; |
| 413 | if shown { |
| 414 | out.push(self.runner_view(&row, &groups, at).await?); |
| 415 | } |
| 416 | } |
| 417 | Ok(Outcome::Ok(out)) |
| 418 | } |
| 419 | |
| 420 | /// `create_registration_token`. |
| 421 | pub async fn create_registration_token(&self, a: CreateRegistrationTokenArgs) -> Result<Outcome<RegistrationToken>> { |
| 422 | let place = check!(self.runner_place(&a.actor, &a.owner, true).await?); |
| 423 | let made = self |
| 424 | .db |
| 425 | .prepare("SELECT COUNT(*) AS n FROM runner_registrations WHERE workspace = ? AND created_at > ?") |
| 426 | .bind(&[place.workspace.as_str().into(), rfc3339(now_ms().saturating_sub(60 * 60 * 1000)).into()])? |
| 427 | .first::<Count>(None) |
| 428 | .await? |
| 429 | .map_or(0, |c| c.n); |
| 430 | if made >= model::MAX_TOKENS_PER_HOUR { |
| 431 | return Ok(fail(FailureCode::Conflict, "Too many registration tokens in the last hour. Each lasts an hour and registers any number of runners.")); |
| 432 | } |
| 433 | let group = match (&place.repo, a.group.as_deref().map(str::trim).filter(|g| !g.is_empty())) { |
| 434 | (Some(_), Some(_)) => return Ok(fail(FailureCode::Invalid, "A repository's own runners are in no group.")), |
| 435 | (Some(_), None) => None, |
| 436 | (None, Some(wanted)) => match self.find_group(&place.workspace, wanted).await? { |
| 437 | Some(group) => Some(group), |
| 438 | None => return Ok(fail(FailureCode::NotFound, format!("There is no runner group called {wanted}."))), |
| 439 | }, |
| 440 | (None, None) => Some(self.default_group(&place.workspace).await?), |
| 441 | }; |
| 442 | let token = format!("{}{}", model::REGISTRATION_PREFIX, random_hex(24)); |
| 443 | let at = now_ms(); |
| 444 | let expires_at = rfc3339(at + model::REGISTRATION_TTL_SECONDS * 1000); |
| 445 | let repo_name = place.repo.as_ref().map(|r| format!("{}/{}", r.namespace, r.name)); |
| 446 | self.db |
| 447 | .prepare( |
| 448 | "INSERT INTO runner_registrations (token_hash, workspace, repo_id, repo, group_id, created_by, created_at, expires_at) |
| 449 | VALUES (?, ?, ?, ?, ?, ?, ?, ?)", |
| 450 | ) |
| 451 | .bind(&[ |
| 452 | hash(&token).into(), |
| 453 | place.workspace.as_str().into(), |
| 454 | optional(place.repo.as_ref().map(|r| r.id.as_str())), |
| 455 | optional(repo_name.as_deref()), |
| 456 | optional(group.as_ref().map(|g| g.id.as_str())), |
| 457 | a.actor.username.as_str().into(), |
| 458 | rfc3339(at).into(), |
| 459 | expires_at.as_str().into(), |
| 460 | ])? |
| 461 | .run() |
| 462 | .await?; |
| 463 | self.audit(vec![entry( |
| 464 | AuditActor::of(&a.actor), |
| 465 | "create_runner_registration_token", |
| 466 | &place.workspace, |
| 467 | repo_name.clone(), |
| 468 | None, |
| 469 | format!("Made a runner registration token{}.", group.as_ref().map(|g| format!(" for group {}", g.name)).unwrap_or_default()), |
| 470 | )]) |
| 471 | .await; |
| 472 | Ok(Outcome::Ok(RegistrationToken { |
| 473 | token, |
| 474 | expires_at, |
| 475 | workspace: place.workspace, |
| 476 | repo: repo_name, |
| 477 | group: group.map(|g| g.name), |
| 478 | url: crate::SITE.to_owned(), |
| 479 | })) |
| 480 | } |
| 481 | |
| 482 | /// `remove_runner`: a person removing one. |
| 483 | pub async fn remove_runner(&self, a: RemoveRunnerArgs) -> Result<Outcome<bool>> { |
| 484 | let place = check!(self.runner_place(&a.actor, &a.owner, true).await?); |
| 485 | let Some(row) = self.runner_by_id(&a.id).await? else { |
| 486 | return Ok(fail(FailureCode::NotFound, "There is no such runner.")); |
| 487 | }; |
| 488 | let ours = row.workspace == place.workspace |
| 489 | && match &place.repo { |
| 490 | Some(repo) => row.repo_id.as_deref() == Some(repo.id.as_str()), |
| 491 | None => row.repo_id.is_none(), |
| 492 | }; |
| 493 | if !ours { |
| 494 | return Ok(fail(FailureCode::NotFound, "There is no such runner.")); |
| 495 | } |
| 496 | self.forget_runner(&row, &format!("{} removed the runner {}.", a.actor.username, row.name)).await?; |
| 497 | self.audit(vec![entry( |
| 498 | AuditActor::of(&a.actor), |
| 499 | "remove_runner", |
| 500 | &row.workspace, |
| 501 | row.repo.clone(), |
| 502 | Some(row.name.clone()), |
| 503 | format!("Removed the self-hosted runner {}.", row.name), |
| 504 | )]) |
| 505 | .await; |
| 506 | Ok(Outcome::Ok(true)) |
| 507 | } |
| 508 | |
| 509 | /// Removes a runner: what it was running fails, and its credential |
| 510 | /// stops working at once. |
| 511 | async fn forget_runner(&self, row: &RunnerRow, why: &str) -> Result<()> { |
| 512 | self.db.prepare("DELETE FROM runners WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 513 | self.abandon_work(&row.id, why).await |
| 514 | } |
| 515 | |
| 516 | /// Fails whatever a runner was running. |
| 517 | async fn abandon_work(&self, runner_id: &str, why: &str) -> Result<()> { |
| 518 | let jobs = self |
| 519 | .db |
| 520 | .prepare("SELECT id FROM jobs WHERE runner_id = ? AND status = 'in_progress'") |
| 521 | .bind(&[runner_id.into()])? |
| 522 | .all() |
| 523 | .await? |
| 524 | .results::<IdRow>()?; |
| 525 | for job in jobs { |
| 526 | self.finish_job(&job.id, "failure", Some(why), None).await?; |
| 527 | } |
| 528 | let tasks = self |
| 529 | .db |
| 530 | .prepare("SELECT * FROM runner_tasks WHERE runner_id = ? AND status = 'in_progress'") |
| 531 | .bind(&[runner_id.into()])? |
| 532 | .all() |
| 533 | .await? |
| 534 | .results::<TaskRow>()?; |
| 535 | for task in tasks { |
| 536 | self.end_task(&task, 1, why).await?; |
| 537 | } |
| 538 | Ok(()) |
| 539 | } |
| 540 | |
| 541 | /// `runner_groups`. |
| 542 | pub async fn runner_groups(&self, a: RunnerGroupsArgs) -> Result<Outcome<Vec<RunnerGroup>>> { |
| 543 | let owner = RunnersOwner { repo: None, workspace: Some(a.workspace.clone()) }; |
| 544 | let place = check!(self.runner_place(&a.actor, &owner, false).await?); |
| 545 | self.default_group(&place.workspace).await?; |
| 546 | let groups = self.group_rows(&place.workspace).await?; |
| 547 | let runners = self.runner_rows(&place.workspace).await?; |
| 548 | Ok(Outcome::Ok(groups.iter().map(|g| group_view(g, &runners)).collect())) |
| 549 | } |
| 550 | |
| 551 | /// `set_runner_group`. |
| 552 | pub async fn set_runner_group(&self, a: SetRunnerGroupArgs) -> Result<Outcome<RunnerGroup>> { |
| 553 | let owner = RunnersOwner { repo: None, workspace: Some(a.workspace.clone()) }; |
| 554 | let place = check!(self.runner_place(&a.actor, &owner, true).await?); |
| 555 | self.default_group(&place.workspace).await?; |
| 556 | let groups = self.group_rows(&place.workspace).await?; |
| 557 | let name = match a.name.as_deref().map(valid_group_name).transpose() { |
| 558 | Ok(name) => name, |
| 559 | Err(problem) => return Ok(fail(FailureCode::Invalid, problem)), |
| 560 | }; |
| 561 | let repositories: Option<Vec<String>> = a.repositories.map(|list| { |
| 562 | let mut out: Vec<String> = Vec::new(); |
| 563 | for repo in list { |
| 564 | let name = repo.trim().rsplit('/').next().unwrap_or_default().to_owned(); |
| 565 | if !name.is_empty() && !out.iter().any(|r: &String| r.eq_ignore_ascii_case(&name)) { |
| 566 | out.push(name); |
| 567 | } |
| 568 | } |
| 569 | out |
| 570 | }); |
| 571 | if let Some(name) = &name |
| 572 | && groups.iter().any(|g| g.name.eq_ignore_ascii_case(name) && Some(&g.id) != a.id.as_ref()) |
| 573 | { |
| 574 | return Ok(fail(FailureCode::Conflict, format!("There is already a group called {name}."))); |
| 575 | } |
| 576 | let at = now(); |
| 577 | let id = match &a.id { |
| 578 | Some(id) => { |
| 579 | let Some(group) = groups.iter().find(|g| &g.id == id) else { |
| 580 | return Ok(fail(FailureCode::NotFound, "There is no such runner group.")); |
| 581 | }; |
| 582 | self.db |
| 583 | .prepare("UPDATE runner_groups SET name = ?, repositories = ?, updated_at = ? WHERE id = ?") |
| 584 | .bind(&[ |
| 585 | name.clone().unwrap_or(group.name.clone()).into(), |
| 586 | serde_json::to_string(&repositories.clone().unwrap_or(group.repositories()))?.into(), |
| 587 | at.as_str().into(), |
| 588 | id.as_str().into(), |
| 589 | ])? |
| 590 | .run() |
| 591 | .await?; |
| 592 | id.clone() |
| 593 | } |
| 594 | None => { |
| 595 | let Some(name) = name.clone() else { |
| 596 | return Ok(fail(FailureCode::Invalid, "A new group needs a name.")); |
| 597 | }; |
| 598 | if groups.len() >= 50 { |
| 599 | return Ok(fail(FailureCode::Conflict, "A workspace has at most 50 runner groups.")); |
| 600 | } |
| 601 | let id = new_id("rng", now_ms()); |
| 602 | self.db |
| 603 | .prepare( |
| 604 | "INSERT INTO runner_groups (id, workspace, name, is_default, repositories, created_at, updated_at) VALUES (?, ?, ?, 0, ?, ?, ?)", |
| 605 | ) |
| 606 | .bind(&[ |
| 607 | id.as_str().into(), |
| 608 | place.workspace.as_str().into(), |
| 609 | name.into(), |
| 610 | serde_json::to_string(&repositories.clone().unwrap_or_default())?.into(), |
| 611 | at.as_str().into(), |
| 612 | at.as_str().into(), |
| 613 | ])? |
| 614 | .run() |
| 615 | .await?; |
| 616 | id |
| 617 | } |
| 618 | }; |
| 619 | let groups = self.group_rows(&place.workspace).await?; |
| 620 | let runners = self.runner_rows(&place.workspace).await?; |
| 621 | let Some(group) = groups.iter().find(|g| g.id == id) else { |
| 622 | return Ok(fail(FailureCode::NotFound, "There is no such runner group.")); |
| 623 | }; |
| 624 | self.audit(vec![entry( |
| 625 | AuditActor::of(&a.actor), |
| 626 | if a.id.is_some() { "update_runner_group" } else { "create_runner_group" }, |
| 627 | &place.workspace, |
| 628 | None, |
| 629 | Some(group.name.clone()), |
| 630 | format!( |
| 631 | "Runner group {}: {}.", |
| 632 | group.name, |
| 633 | if group.repositories().is_empty() { "every repository".to_owned() } else { group.repositories().join(", ") } |
| 634 | ), |
| 635 | )]) |
| 636 | .await; |
| 637 | Ok(Outcome::Ok(group_view(group, &runners))) |
| 638 | } |
| 639 | |
| 640 | /// `delete_runner_group`. |
| 641 | pub async fn delete_runner_group(&self, a: DeleteRunnerGroupArgs) -> Result<Outcome<bool>> { |
| 642 | let owner = RunnersOwner { repo: None, workspace: Some(a.workspace.clone()) }; |
| 643 | let place = check!(self.runner_place(&a.actor, &owner, true).await?); |
| 644 | let default = self.default_group(&place.workspace).await?; |
| 645 | let Some(group) = self.group_rows(&place.workspace).await?.into_iter().find(|g| g.id == a.id) else { |
| 646 | return Ok(fail(FailureCode::NotFound, "There is no such runner group.")); |
| 647 | }; |
| 648 | if group.is_default != 0 { |
| 649 | return Ok(fail(FailureCode::Conflict, "The default group cannot be deleted.")); |
| 650 | } |
| 651 | self.db |
| 652 | .batch(vec![ |
| 653 | self.db.prepare("UPDATE runners SET group_id = ? WHERE group_id = ?").bind(&[default.id.as_str().into(), group.id.as_str().into()])?, |
| 654 | self.db.prepare("DELETE FROM runner_groups WHERE id = ?").bind(&[group.id.as_str().into()])?, |
| 655 | ]) |
| 656 | .await?; |
| 657 | self.audit(vec![entry( |
| 658 | AuditActor::of(&a.actor), |
| 659 | "delete_runner_group", |
| 660 | &place.workspace, |
| 661 | None, |
| 662 | Some(group.name.clone()), |
| 663 | format!("Deleted the runner group {}; its runners joined {}.", group.name, default.name), |
| 664 | )]) |
| 665 | .await; |
| 666 | Ok(Outcome::Ok(true)) |
| 667 | } |
| 668 | |
| 669 | async fn settings_row(&self, owner: &str) -> Result<Option<SettingsRow>> { |
| 670 | self.db.prepare("SELECT * FROM runner_settings WHERE owner = ?").bind(&[owner.into()])?.first::<SettingsRow>(None).await |
| 671 | } |
| 672 | |
| 673 | fn settings_of(row: Option<SettingsRow>, inherited: bool) -> RunnerSettings { |
| 674 | match row { |
| 675 | Some(row) => RunnerSettings { |
| 676 | agents_on_self_hosted: row.agents != 0, |
| 677 | agent_labels: serde_json::from_str(&row.agent_labels).unwrap_or_else(|_| vec![model::SELF_HOSTED.to_owned()]), |
| 678 | fork_pull_requests: row.fork_pulls != 0, |
| 679 | inherited, |
| 680 | }, |
| 681 | None => RunnerSettings { |
| 682 | agent_labels: vec![model::SELF_HOSTED.to_owned()], |
| 683 | inherited, |
| 684 | ..RunnerSettings::default() |
| 685 | }, |
| 686 | } |
| 687 | } |
| 688 | |
| 689 | /// The settings that apply in a workspace, or in one of its |
| 690 | /// repositories: the repository's own if it has them. |
| 691 | pub async fn effective_runner_settings(&self, workspace: &str, repo_id: Option<&str>) -> Result<RunnerSettings> { |
| 692 | if let Some(repo_id) = repo_id |
| 693 | && let Some(own) = self.settings_row(repo_id).await? |
| 694 | { |
| 695 | return Ok(Self::settings_of(Some(own), false)); |
| 696 | } |
| 697 | let inherited = repo_id.is_some(); |
| 698 | Ok(Self::settings_of(self.settings_row(&workspace.to_lowercase()).await?, inherited)) |
| 699 | } |
| 700 | |
| 701 | /// `runner_settings`. |
| 702 | pub async fn runner_settings(&self, a: RunnerSettingsArgs) -> Result<Outcome<RunnerSettings>> { |
| 703 | let place = check!(self.runner_place(&a.actor, &a.owner, false).await?); |
| 704 | Ok(Outcome::Ok(self.effective_runner_settings(&place.workspace, place.repo.as_ref().map(|r| r.id.as_str())).await?)) |
| 705 | } |
| 706 | |
| 707 | /// `set_runner_settings`. |
| 708 | pub async fn set_runner_settings(&self, a: SetRunnerSettingsArgs) -> Result<Outcome<RunnerSettings>> { |
| 709 | let place = check!(self.runner_place(&a.actor, &a.owner, true).await?); |
| 710 | let (owner, scope) = match &place.repo { |
| 711 | Some(repo) => (repo.id.clone(), "repository"), |
| 712 | None => (place.workspace.clone(), "workspace"), |
| 713 | }; |
| 714 | if a.inherit && place.repo.is_some() { |
| 715 | self.db.prepare("DELETE FROM runner_settings WHERE owner = ?").bind(&[owner.as_str().into()])?.run().await?; |
| 716 | } else { |
| 717 | let current = self.effective_runner_settings(&place.workspace, place.repo.as_ref().map(|r| r.id.as_str())).await?; |
| 718 | let labels = match &a.agent_labels { |
| 719 | Some(given) => match model::runner_labels(given, model::SELF_HOSTED, model::SELF_HOSTED) { |
| 720 | Ok(labels) => labels, |
| 721 | Err(problem) => return Ok(fail(FailureCode::Invalid, problem)), |
| 722 | }, |
| 723 | None => current.agent_labels.clone(), |
| 724 | }; |
| 725 | let agents = a.agents_on_self_hosted.unwrap_or(current.agents_on_self_hosted); |
| 726 | let forks = a.fork_pull_requests.unwrap_or(current.fork_pull_requests); |
| 727 | self.db |
| 728 | .prepare( |
| 729 | "INSERT INTO runner_settings (owner, scope, agents, agent_labels, fork_pulls, updated_at, updated_by) |
| 730 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) |
| 731 | ON CONFLICT (owner) DO UPDATE SET agents = ?3, agent_labels = ?4, fork_pulls = ?5, updated_at = ?6, updated_by = ?7", |
| 732 | ) |
| 733 | .bind(&[ |
| 734 | owner.as_str().into(), |
| 735 | scope.into(), |
| 736 | u32::from(agents).into(), |
| 737 | serde_json::to_string(&labels)?.into(), |
| 738 | u32::from(forks).into(), |
| 739 | now().into(), |
| 740 | a.actor.username.as_str().into(), |
| 741 | ])? |
| 742 | .run() |
| 743 | .await?; |
| 744 | } |
| 745 | let settings = self.effective_runner_settings(&place.workspace, place.repo.as_ref().map(|r| r.id.as_str())).await?; |
| 746 | self.audit(vec![entry( |
| 747 | AuditActor::of(&a.actor), |
| 748 | "update_runner_settings", |
| 749 | &place.workspace, |
| 750 | place.repo.as_ref().map(|r| format!("{}/{}", r.namespace, r.name)), |
| 751 | None, |
| 752 | format!( |
| 753 | "Agents on self-hosted runners: {}; labels {}; pull requests from forks: {}.", |
| 754 | if settings.agents_on_self_hosted { "on" } else { "off" }, |
| 755 | settings.agent_labels.join(", "), |
| 756 | if settings.fork_pull_requests { "allowed" } else { "not allowed" } |
| 757 | ), |
| 758 | )]) |
| 759 | .await; |
| 760 | Ok(Outcome::Ok(settings)) |
| 761 | } |
| 762 | |
| 763 | // --- The runner's side --------------------------------------------------- |
| 764 | |
| 765 | /// `runner_register`. |
| 766 | pub async fn runner_register(&self, a: RegisterArgs) -> Result<Outcome<Registered>> { |
| 767 | let refused = || fail(FailureCode::Unauthenticated, "That registration token is not valid, or it expired. Make a new one under Settings, Runners."); |
| 768 | if !a.token.starts_with(model::REGISTRATION_PREFIX) { |
| 769 | return Ok(refused()); |
| 770 | } |
| 771 | let registration = self |
| 772 | .db |
| 773 | .prepare("SELECT * FROM runner_registrations WHERE token_hash = ?") |
| 774 | .bind(&[hash(&a.token).into()])? |
| 775 | .first::<RegistrationRow>(None) |
| 776 | .await?; |
| 777 | let at = now_ms(); |
| 778 | let Some(registration) = registration.filter(|r| r.expires_at.as_str() > rfc3339(at).as_str()) else { |
| 779 | return Ok(refused()); |
| 780 | }; |
| 781 | let name = match valid_name(&a.name) { |
| 782 | Ok(name) => name, |
| 783 | Err(problem) => return Ok(fail(FailureCode::Invalid, problem)), |
| 784 | }; |
| 785 | let (Some(os), Some(arch)) = (model::os_label(&a.os), model::arch_label(&a.arch)) else { |
| 786 | return Ok(fail(FailureCode::Invalid, "A runner's os is linux, macos or windows, and its arch x64 or arm64.")); |
| 787 | }; |
| 788 | let labels = match model::runner_labels(&a.labels, os, arch) { |
| 789 | Ok(labels) => labels, |
| 790 | Err(problem) => return Ok(fail(FailureCode::Invalid, problem)), |
| 791 | }; |
| 792 | let workspace = registration.workspace.clone(); |
| 793 | // Abuse: so many machines, so fast, are not a team's. |
| 794 | let recent = self |
| 795 | .db |
| 796 | .prepare("SELECT COUNT(*) AS n FROM runners WHERE workspace = ? AND created_at > ?") |
| 797 | .bind(&[workspace.as_str().into(), rfc3339(at.saturating_sub(60 * 1000)).into()])? |
| 798 | .first::<Count>(None) |
| 799 | .await? |
| 800 | .map_or(0, |c| c.n); |
| 801 | if recent >= model::MAX_REGISTRATIONS_PER_MINUTE { |
| 802 | return Ok(fail(FailureCode::Conflict, "Too many runners registered in the last minute. Wait a moment and try again.")); |
| 803 | } |
| 804 | let total = self |
| 805 | .db |
| 806 | .prepare("SELECT COUNT(*) AS n FROM runners WHERE workspace = ?") |
| 807 | .bind(&[workspace.as_str().into()])? |
| 808 | .first::<Count>(None) |
| 809 | .await? |
| 810 | .map_or(0, |c| c.n); |
| 811 | // A group: the one asked for, the token's, or the default. |
| 812 | let group = if registration.repo_id.is_some() { |
| 813 | None |
| 814 | } else { |
| 815 | match a.group.as_deref().map(str::trim).filter(|g| !g.is_empty()) { |
| 816 | Some(wanted) => match self.find_group(&workspace, wanted).await? { |
| 817 | Some(group) => Some(group.id), |
| 818 | None => return Ok(fail(FailureCode::NotFound, format!("There is no runner group called {wanted}."))), |
| 819 | }, |
| 820 | None => match registration.group_id.clone() { |
| 821 | Some(id) => Some(id), |
| 822 | None => Some(self.default_group(&workspace).await?.id), |
| 823 | }, |
| 824 | } |
| 825 | }; |
| 826 | let existing = self |
| 827 | .db |
| 828 | .prepare("SELECT * FROM runners WHERE workspace = ? AND COALESCE(repo_id, '') = ? AND lower(name) = lower(?)") |
| 829 | .bind(&[workspace.as_str().into(), registration.repo_id.clone().unwrap_or_default().into(), name.as_str().into()])? |
| 830 | .first::<RunnerRow>(None) |
| 831 | .await?; |
| 832 | if let Some(existing) = &existing { |
| 833 | if !a.replace { |
| 834 | return Ok(fail( |
| 835 | FailureCode::Conflict, |
| 836 | format!("A runner called {name} is already registered here. Register with --replace to take its place, or choose another --name."), |
| 837 | )); |
| 838 | } |
| 839 | self.forget_runner(existing, &format!("The runner {name} was replaced by a new registration.")).await?; |
| 840 | } else if total >= model::MAX_RUNNERS { |
| 841 | return Ok(fail(FailureCode::Conflict, format!("A workspace has at most {} self-hosted runners.", model::MAX_RUNNERS))); |
| 842 | } |
| 843 | // How it keeps work from its machine, if it said (older runners do not). |
| 844 | let isolation = a.isolation.as_deref().filter(|level| model::ISOLATIONS.contains(level)); |
| 845 | let id = new_id("rnr", at); |
| 846 | let credential = format!("{}{}", model::CREDENTIAL_PREFIX, random_hex(32)); |
| 847 | let created = rfc3339(at); |
| 848 | self.db |
| 849 | .prepare( |
| 850 | "INSERT INTO runners (id, workspace, repo_id, repo, group_id, name, labels, os, arch, version, ephemeral, credential_hash, |
| 851 | rotated_at, last_seen_at, created_at, created_by, isolation) |
| 852 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", |
| 853 | ) |
| 854 | .bind(&[ |
| 855 | id.as_str().into(), |
| 856 | workspace.as_str().into(), |
| 857 | optional(registration.repo_id.as_deref()), |
| 858 | optional(registration.repo.as_deref()), |
| 859 | optional(group.as_deref()), |
| 860 | name.as_str().into(), |
| 861 | serde_json::to_string(&labels)?.into(), |
| 862 | os.into(), |
| 863 | arch.into(), |
| 864 | a.version.chars().take(40).collect::<String>().into(), |
| 865 | u32::from(a.ephemeral).into(), |
| 866 | hash(&credential).into(), |
| 867 | created.as_str().into(), |
| 868 | created.as_str().into(), |
| 869 | created.as_str().into(), |
| 870 | JsValue::NULL, |
| 871 | optional(isolation), |
| 872 | ])? |
| 873 | .run() |
| 874 | .await?; |
| 875 | self.db |
| 876 | .prepare("UPDATE runner_registrations SET used = used + 1 WHERE token_hash = ?") |
| 877 | .bind(&[hash(&a.token).into()])? |
| 878 | .run() |
| 879 | .await?; |
| 880 | let row = self.runner_by_id(&id).await?.ok_or_else(|| worker::Error::RustError("the runner was not kept".into()))?; |
| 881 | self.audit(vec![entry( |
| 882 | runner_actor(&row), |
| 883 | "register_runner", |
| 884 | &workspace, |
| 885 | row.repo.clone(), |
| 886 | Some(name.clone()), |
| 887 | format!("Registered the self-hosted runner {name} ({os}, {arch}; labels {}).", labels.join(", ")), |
| 888 | )]) |
| 889 | .await; |
| 890 | let groups = self.group_rows(&workspace).await?; |
| 891 | Ok(Outcome::Ok(Registered { |
| 892 | runner: self.runner_view(&row, &groups, at).await?, |
| 893 | credential, |
| 894 | })) |
| 895 | } |
| 896 | |
| 897 | /// The runner a call is from, if its credential is that runner's (or |
| 898 | /// the one before a rotation, until the new one is used). |
| 899 | async fn authenticated(&self, auth: &RunnerAuth) -> Result<Option<(RunnerRow, bool)>> { |
| 900 | if !auth.credential.starts_with(model::CREDENTIAL_PREFIX) { |
| 901 | return Ok(None); |
| 902 | } |
| 903 | let Some(row) = self.runner_by_id(&auth.runner).await? else { return Ok(None) }; |
| 904 | let given = hash(&auth.credential); |
| 905 | if g1t_secrets::same(&row.credential_hash, &given) { |
| 906 | if row.previous_hash.is_some() { |
| 907 | self.db.prepare("UPDATE runners SET previous_hash = NULL WHERE id = ?").bind(&[row.id.as_str().into()])?.run().await?; |
| 908 | } |
| 909 | return Ok(Some((row, true))); |
| 910 | } |
| 911 | if row.previous_hash.as_deref().is_some_and(|previous| g1t_secrets::same(previous, &given)) { |
| 912 | return Ok(Some((row, false))); |
| 913 | } |
| 914 | Ok(None) |
| 915 | } |
| 916 | |
| 917 | fn unknown_runner<T>() -> Outcome<T> { |
| 918 | fail(FailureCode::Unauthenticated, "This runner is not registered, or its credential is not valid. Register it again.") |
| 919 | } |
| 920 | |
| 921 | /// `runner_poll`. |
| 922 | pub async fn runner_poll(&self, a: PollArgs) -> Result<Outcome<Poll>> { |
| 923 | let Some((row, current)) = self.authenticated(&a.auth).await? else { |
| 924 | return Ok(Self::unknown_runner()); |
| 925 | }; |
| 926 | let at = now_ms(); |
| 927 | let seen = rfc3339(at); |
| 928 | // A new credential every day, handed over on a poll that used the |
| 929 | // current one. |
| 930 | let mut credential = None; |
| 931 | let rotate_after = rfc3339(at.saturating_sub(model::ROTATE_AFTER_MS)); |
| 932 | if current && row.rotated_at.as_str() < rotate_after.as_str() { |
| 933 | let fresh = format!("{}{}", model::CREDENTIAL_PREFIX, random_hex(32)); |
| 934 | self.db |
| 935 | .prepare("UPDATE runners SET previous_hash = credential_hash, credential_hash = ?, rotated_at = ? WHERE id = ?") |
| 936 | .bind(&[hash(&fresh).into(), seen.as_str().into(), row.id.as_str().into()])? |
| 937 | .run() |
| 938 | .await?; |
| 939 | credential = Some(fresh); |
| 940 | } |
| 941 | // What it holds that it should stop, and the heartbeat for the rest. |
| 942 | let mut cancel = Vec::new(); |
| 943 | let mut active: Option<(String, &'static str)> = None; |
| 944 | for id in a.running.iter().take(10) { |
| 945 | if let Some(job) = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[id.as_str().into()])?.first::<JobRow>(None).await? { |
| 946 | if job.status == "in_progress" && job.runner_id.as_deref() == Some(row.id.as_str()) { |
| 947 | if seen_is_stale(job.seen_at.as_deref(), at) { |
| 948 | self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[seen.as_str().into(), id.as_str().into()])?.run().await?; |
| 949 | } |
| 950 | active = Some((id.clone(), "workflow")); |
| 951 | } else { |
| 952 | cancel.push(id.clone()); |
| 953 | } |
| 954 | continue; |
| 955 | } |
| 956 | match self.db.prepare("SELECT * FROM runner_tasks WHERE id = ?").bind(&[id.as_str().into()])?.first::<TaskRow>(None).await? { |
| 957 | Some(task) if task.status == "in_progress" && task.runner_id.as_deref() == Some(row.id.as_str()) => { |
| 958 | if seen_is_stale(task.seen_at.as_deref(), at) { |
| 959 | self.db.prepare("UPDATE runner_tasks SET seen_at = ? WHERE id = ?").bind(&[seen.as_str().into(), id.as_str().into()])?.run().await?; |
| 960 | } |
| 961 | active = Some((id.clone(), "agent")); |
| 962 | } |
| 963 | _ => cancel.push(id.clone()), |
| 964 | } |
| 965 | } |
| 966 | // Written when something changed, or once a minute to keep it online. |
| 967 | let version: String = if a.version.is_empty() { row.version.clone() } else { a.version.chars().take(40).collect() }; |
| 968 | let work_id = active.as_ref().map(|(id, _)| id.as_str()); |
| 969 | let work_kind = active.as_ref().map(|(_, kind)| *kind); |
| 970 | // How it is isolated, as it says now: a runner re-registered or |
| 971 | // restarted with or without Docker says so on its next poll. |
| 972 | let isolation = a |
| 973 | .isolation |
| 974 | .as_deref() |
| 975 | .filter(|level| model::ISOLATIONS.contains(level)) |
| 976 | .map(str::to_owned) |
| 977 | .or_else(|| row.isolation.clone()); |
| 978 | let changed = version != row.version |
| 979 | || work_id != row.work_id.as_deref() |
| 980 | || work_kind != row.work_kind.as_deref() |
| 981 | || isolation != row.isolation; |
| 982 | if changed || seen_is_stale(row.last_seen_at.as_deref(), at) { |
| 983 | self.db |
| 984 | .prepare("UPDATE runners SET last_seen_at = ?, version = ?, work_id = ?, work_kind = ?, isolation = ? WHERE id = ?") |
| 985 | .bind(&[ |
| 986 | seen.as_str().into(), |
| 987 | version.into(), |
| 988 | optional(work_id), |
| 989 | optional(work_kind), |
| 990 | optional(isolation.as_deref()), |
| 991 | row.id.as_str().into(), |
| 992 | ])? |
| 993 | .run() |
| 994 | .await?; |
| 995 | } |
| 996 | let row = RunnerRow { isolation, ..row }; |
| 997 | let mut poll = Poll { assignment: None, cancel, credential, removed: false }; |
| 998 | // Busy, or an ephemeral runner that has had its one job. |
| 999 | if active.is_some() || !a.running.is_empty() { |
| 1000 | return Ok(Outcome::Ok(poll)); |
| 1001 | } |
| 1002 | if row.ephemeral != 0 && row.spent != 0 { |
| 1003 | self.forget_runner(&row, "The ephemeral runner finished its job.").await?; |
| 1004 | poll.removed = true; |
| 1005 | return Ok(Outcome::Ok(poll)); |
| 1006 | } |
| 1007 | let groups = self.group_rows(&row.workspace).await?; |
| 1008 | let group = row.group_id.as_ref().and_then(|id| groups.iter().find(|g| &g.id == id)).cloned(); |
| 1009 | let deadline = at + a.wait_ms.min(model::MAX_POLL_WAIT_MS); |
| 1010 | let mut looked = Looked::default(); |
| 1011 | loop { |
| 1012 | if let Some(assignment) = self.claim(&row, group.as_ref(), &mut looked).await? { |
| 1013 | poll.assignment = Some(assignment); |
| 1014 | return Ok(Outcome::Ok(poll)); |
| 1015 | } |
| 1016 | if now_ms() + POLL_EVERY_MS > deadline { |
| 1017 | return Ok(Outcome::Ok(poll)); |
| 1018 | } |
| 1019 | worker::Delay::from(std::time::Duration::from_millis(POLL_EVERY_MS)).await; |
| 1020 | } |
| 1021 | } |
| 1022 | |
| 1023 | /// Takes the oldest work this runner can do, if there is any. |
| 1024 | async fn claim(&self, runner: &RunnerRow, group: Option<&GroupRow>, looked: &mut Looked) -> Result<Option<Assignment>> { |
| 1025 | let labels = runner.labels(); |
| 1026 | let group_name = group.map(|g| g.name.clone()); |
| 1027 | // Workflow jobs. |
| 1028 | let mut sql = "SELECT jobs.*, runs.repo AS run_repo FROM jobs JOIN runs ON runs.id = jobs.run_id |
| 1029 | WHERE jobs.status = 'queued' AND jobs.labels IS NOT NULL AND lower(jobs.namespace) = ?" |
| 1030 | .to_owned(); |
| 1031 | let mut binds: Vec<JsValue> = vec![runner.workspace.as_str().into()]; |
| 1032 | if let Some(repo_id) = &runner.repo_id { |
| 1033 | sql.push_str(" AND jobs.repo_id = ?"); |
| 1034 | binds.push(repo_id.as_str().into()); |
| 1035 | } |
| 1036 | // One job of a concurrency group runs at a time. |
| 1037 | sql.push_str( |
| 1038 | " AND (jobs.concurrency_group IS NULL OR NOT EXISTS (SELECT 1 FROM jobs o WHERE o.repo_id = jobs.repo_id |
| 1039 | AND o.concurrency_group = jobs.concurrency_group AND o.status IN ('in_progress', 'calling')))", |
| 1040 | ); |
| 1041 | sql.push_str(" ORDER BY jobs.rowid LIMIT 50"); |
| 1042 | #[derive(Deserialize)] |
| 1043 | struct Queued { |
| 1044 | #[serde(flatten)] |
| 1045 | job: JobRow, |
| 1046 | run_repo: String, |
| 1047 | } |
| 1048 | let queued = self.db.prepare(sql).bind(&binds)?.all().await?.results::<Queued>()?; |
| 1049 | for Queued { job, run_repo } in queued { |
| 1050 | let stored: Vec<String> = job.labels.as_deref().and_then(|l| serde_json::from_str(l).ok()).unwrap_or_default(); |
| 1051 | let wanted = Wanted::from_stored(&stored); |
| 1052 | if !wanted.matches(&labels, group_name.as_deref()) { |
| 1053 | continue; |
| 1054 | } |
| 1055 | if runner.repo_id.is_none() && !group.is_none_or(|g| g.allows(&run_repo)) { |
| 1056 | continue; |
| 1057 | } |
| 1058 | // A host-mode runner takes no public repository's job: it waits |
| 1059 | // for a sandboxed runner, and says so when none could take it. |
| 1060 | if runner.isolation.as_deref() == Some("host") { |
| 1061 | let visibility = self.visibility_once(looked, &job.repo_id).await; |
| 1062 | if !model::isolation_allows(runner.isolation.as_deref(), visibility.as_deref()) { |
| 1063 | if looked.noted.insert(job.id.clone()) { |
| 1064 | self.wait_for_sandbox(&job, &run_repo, &wanted).await?; |
| 1065 | } |
| 1066 | continue; |
| 1067 | } |
| 1068 | } |
| 1069 | if let Some(max) = job.max_parallel { |
| 1070 | let siblings = self |
| 1071 | .db |
| 1072 | .prepare("SELECT COUNT(*) AS n FROM jobs WHERE run_id = ? AND key = ? AND status = 'in_progress'") |
| 1073 | .bind(&[job.run_id.as_str().into(), job.key.as_str().into()])? |
| 1074 | .first::<Count>(None) |
| 1075 | .await? |
| 1076 | .map_or(0, |c| c.n); |
| 1077 | if siblings >= max { |
| 1078 | continue; |
| 1079 | } |
| 1080 | } |
| 1081 | let token = random_hex(24); |
| 1082 | let at = now(); |
| 1083 | let claimed = self |
| 1084 | .db |
| 1085 | .prepare( |
| 1086 | "UPDATE jobs SET status = 'in_progress', token_hash = ?, runner_id = ?, runner_name = ?, reason = NULL, started_at = ?, seen_at = ? |
| 1087 | WHERE id = ? AND status = 'queued' RETURNING id", |
| 1088 | ) |
| 1089 | .bind(&[ |
| 1090 | sha256_hex(&token).into(), |
| 1091 | runner.id.as_str().into(), |
| 1092 | runner.name.as_str().into(), |
| 1093 | at.as_str().into(), |
| 1094 | at.as_str().into(), |
| 1095 | job.id.as_str().into(), |
| 1096 | ])? |
| 1097 | .first::<Value>(None) |
| 1098 | .await?; |
| 1099 | if claimed.is_none() { |
| 1100 | continue; |
| 1101 | } |
| 1102 | self.db |
| 1103 | .batch(vec![ |
| 1104 | self.db |
| 1105 | .prepare("UPDATE runs SET status = 'in_progress', started_at = COALESCE(started_at, ?) WHERE id = ? AND status = 'queued'") |
| 1106 | .bind(&[at.as_str().into(), job.run_id.as_str().into()])?, |
| 1107 | self.db |
| 1108 | .prepare("UPDATE runners SET work_id = ?, work_kind = 'workflow', spent = ephemeral WHERE id = ?") |
| 1109 | .bind(&[job.id.as_str().into(), runner.id.as_str().into()])?, |
| 1110 | ]) |
| 1111 | .await?; |
| 1112 | self.audit(vec![entry( |
| 1113 | runner_actor(runner), |
| 1114 | "runner.take_job", |
| 1115 | &runner.workspace, |
| 1116 | Some(run_repo.clone()), |
| 1117 | Some(job.name.clone()), |
| 1118 | format!("The self-hosted runner {} took the job {}.", runner.name, job.name), |
| 1119 | )]) |
| 1120 | .await; |
| 1121 | // A job that deploys: its run's deployment is under way. |
| 1122 | self.job_started(&job.id).await?; |
| 1123 | let image = self.job_image(&job).await?; |
| 1124 | let visibility = self.visibility(&job.repo_id).await; |
| 1125 | return Ok(Some(Assignment { |
| 1126 | kind: "workflow".into(), |
| 1127 | id: job.id.clone(), |
| 1128 | name: job.name.clone(), |
| 1129 | repo: run_repo, |
| 1130 | timeout_minutes: job.timeout_minutes, |
| 1131 | image, |
| 1132 | token: Some(token), |
| 1133 | env: None, |
| 1134 | visibility, |
| 1135 | })); |
| 1136 | } |
| 1137 | |
| 1138 | // Agent work. |
| 1139 | let tasks = self |
| 1140 | .db |
| 1141 | .prepare("SELECT * FROM runner_tasks WHERE status = 'queued' AND workspace = ? ORDER BY created_at LIMIT 20") |
| 1142 | .bind(&[runner.workspace.as_str().into()])? |
| 1143 | .all() |
| 1144 | .await? |
| 1145 | .results::<TaskRow>()?; |
| 1146 | for task in tasks { |
| 1147 | let wanted = Wanted::of(&serde_json::from_str::<Vec<String>>(&task.labels).unwrap_or_default(), None); |
| 1148 | if !wanted.matches(&labels, None) { |
| 1149 | continue; |
| 1150 | } |
| 1151 | // Nor a public repository's agent work. |
| 1152 | if runner.isolation.as_deref() == Some("host") |
| 1153 | && let Some(repo_id) = &task.repo_id |
| 1154 | { |
| 1155 | let visibility = self.visibility_once(looked, repo_id).await; |
| 1156 | if !model::isolation_allows(runner.isolation.as_deref(), visibility.as_deref()) { |
| 1157 | continue; |
| 1158 | } |
| 1159 | } |
| 1160 | if let Some(repo_id) = &runner.repo_id |
| 1161 | && task.repo_id.as_deref() != Some(repo_id.as_str()) |
| 1162 | { |
| 1163 | continue; |
| 1164 | } |
| 1165 | if runner.repo_id.is_none() && !group.is_none_or(|g| g.allows(&task.repo)) { |
| 1166 | continue; |
| 1167 | } |
| 1168 | let Some(sealed) = task.env.clone() else { continue }; |
| 1169 | let at = now(); |
| 1170 | let claimed = self |
| 1171 | .db |
| 1172 | .prepare( |
| 1173 | "UPDATE runner_tasks SET status = 'in_progress', runner_id = ?, runner_name = ?, env = NULL, started_at = ?, seen_at = ? |
| 1174 | WHERE id = ? AND status = 'queued' RETURNING id", |
| 1175 | ) |
| 1176 | .bind(&[runner.id.as_str().into(), runner.name.as_str().into(), at.as_str().into(), at.as_str().into(), task.id.as_str().into()])? |
| 1177 | .first::<Value>(None) |
| 1178 | .await?; |
| 1179 | if claimed.is_none() { |
| 1180 | continue; |
| 1181 | } |
| 1182 | self.db |
| 1183 | .prepare("UPDATE runners SET work_id = ?, work_kind = 'agent', spent = ephemeral WHERE id = ?") |
| 1184 | .bind(&[task.id.as_str().into(), runner.id.as_str().into()])? |
| 1185 | .run() |
| 1186 | .await?; |
| 1187 | let env: Map<String, Value> = self |
| 1188 | .sealer |
| 1189 | .as_ref() |
| 1190 | .and_then(|sealer| sealer.open(&sealed, &task.id)) |
| 1191 | .and_then(|text| serde_json::from_str(&text).ok()) |
| 1192 | .unwrap_or_default(); |
| 1193 | if env.is_empty() { |
| 1194 | self.end_task(&TaskRow { status: "in_progress".into(), ..task.clone() }, 1, "The work's environment could not be opened.").await?; |
| 1195 | continue; |
| 1196 | } |
| 1197 | self.audit(vec![entry( |
| 1198 | runner_actor(runner), |
| 1199 | "runner.take_task", |
| 1200 | &runner.workspace, |
| 1201 | Some(task.repo.clone()), |
| 1202 | Some(task.kind.clone()), |
| 1203 | format!("The self-hosted runner {} took agent work: {}.", runner.name, task.title), |
| 1204 | )]) |
| 1205 | .await; |
| 1206 | let visibility = match &task.repo_id { |
| 1207 | Some(repo_id) => self.visibility(repo_id).await, |
| 1208 | None => None, |
| 1209 | }; |
| 1210 | return Ok(Some(Assignment { |
| 1211 | kind: "agent".into(), |
| 1212 | id: task.id.clone(), |
| 1213 | name: task.title.clone(), |
| 1214 | repo: task.repo.clone(), |
| 1215 | timeout_minutes: task.timeout_minutes, |
| 1216 | image: None, |
| 1217 | token: None, |
| 1218 | env: Some(env), |
| 1219 | visibility, |
| 1220 | })); |
| 1221 | } |
| 1222 | Ok(None) |
| 1223 | } |
| 1224 | |
| 1225 | /// [`Self::visibility`], asked once per repository in a poll. |
| 1226 | async fn visibility_once(&self, looked: &mut Looked, repo_id: &str) -> Option<String> { |
| 1227 | if let Some(known) = looked.visibilities.get(repo_id) { |
| 1228 | return known.clone(); |
| 1229 | } |
| 1230 | let found = self.visibility(repo_id).await; |
| 1231 | looked.visibilities.insert(repo_id.to_owned(), found.clone()); |
| 1232 | found |
| 1233 | } |
| 1234 | |
| 1235 | /// A public repository's job that a host-mode runner passed over: when |
| 1236 | /// no online runner that may take it could (its labels, group, |
| 1237 | /// repository, and isolation), its page says it waits for a sandboxed |
| 1238 | /// one. Written only when that changes what it says. |
| 1239 | async fn wait_for_sandbox(&self, job: &JobRow, run_repo: &str, wanted: &Wanted) -> Result<()> { |
| 1240 | let at = now_ms(); |
| 1241 | let groups = self.group_rows(&job.namespace.to_lowercase()).await?; |
| 1242 | let runners = self.runner_rows(&job.namespace.to_lowercase()).await?; |
| 1243 | let sandboxed = runners.iter().any(|other| takes_public_job(other, &groups, &job.repo_id, run_repo, wanted, at)); |
| 1244 | if sandboxed { |
| 1245 | return Ok(()); |
| 1246 | } |
| 1247 | self.db |
| 1248 | .prepare("UPDATE jobs SET reason = ? WHERE id = ? AND status = 'queued' AND reason IS NOT ?") |
| 1249 | .bind(&[model::WAITING_FOR_SANDBOX.into(), job.id.as_str().into(), model::WAITING_FOR_SANDBOX.into()])? |
| 1250 | .run() |
| 1251 | .await?; |
| 1252 | Ok(()) |
| 1253 | } |
| 1254 | |
| 1255 | /// A repository's visibility for a runner, `public` or `private`; none |
| 1256 | /// when it cannot be read, and the runner decides without it. |
| 1257 | async fn visibility(&self, repo_id: &str) -> Option<String> { |
| 1258 | match self.repo_by_id(repo_id).await { |
| 1259 | Ok(Some((repo, _))) => Some(if repo.is_private { "private" } else { "public" }.to_owned()), |
| 1260 | _ => None, |
| 1261 | } |
| 1262 | } |
| 1263 | |
| 1264 | /// The image a job's `container:` names, when it names one plainly. |
| 1265 | async fn job_image(&self, job: &JobRow) -> Result<Option<String>> { |
| 1266 | let Some(run) = self.run_row(&job.run_id).await? else { return Ok(None) }; |
| 1267 | let raw = match job.callee() { |
| 1268 | Some((_, spec, _)) => spec.raw, |
| 1269 | None => match g1t_actions::workflow::parse(&run.source) { |
| 1270 | Ok(workflow) => match workflow.jobs.into_iter().find(|j| j.id == job.key) { |
| 1271 | Some(spec) => spec.raw, |
| 1272 | None => return Ok(None), |
| 1273 | }, |
| 1274 | Err(_) => return Ok(None), |
| 1275 | }, |
| 1276 | }; |
| 1277 | let image = match raw.get("container") { |
| 1278 | Some(Value::String(image)) => Some(image.clone()), |
| 1279 | Some(Value::Object(container)) => container.get("image").and_then(Value::as_str).map(str::to_owned), |
| 1280 | _ => None, |
| 1281 | }; |
| 1282 | Ok(image.filter(|image| !image.contains("${{") && !image.trim().is_empty())) |
| 1283 | } |
| 1284 | |
| 1285 | /// `runner_finished`. |
| 1286 | pub async fn runner_finished(&self, a: FinishedArgs) -> Result<Outcome<bool>> { |
| 1287 | let Some((row, _)) = self.authenticated(&a.auth).await? else { |
| 1288 | return Ok(Self::unknown_runner()); |
| 1289 | }; |
| 1290 | let reason = a.reason.clone().filter(|r| !r.trim().is_empty()).map(|r| r.chars().take(2000).collect::<String>()); |
| 1291 | let mut changed = false; |
| 1292 | if let Some(job) = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[a.id.as_str().into()])?.first::<JobRow>(None).await? { |
| 1293 | if job.runner_id.as_deref() == Some(row.id.as_str()) && job.status == "in_progress" { |
| 1294 | let why = reason.unwrap_or_else(|| format!("The job's process on {} exited with code {} before it reported how the job went.", row.name, a.exit_code)); |
| 1295 | self.finish_job(&job.id, "failure", Some(&why), None).await?; |
| 1296 | changed = true; |
| 1297 | } |
| 1298 | } else if let Some(task) = self.db.prepare("SELECT * FROM runner_tasks WHERE id = ?").bind(&[a.id.as_str().into()])?.first::<TaskRow>(None).await? |
| 1299 | && task.runner_id.as_deref() == Some(row.id.as_str()) |
| 1300 | && task.status == "in_progress" |
| 1301 | { |
| 1302 | let why = reason.unwrap_or_else(|| if a.exit_code == 0 { String::new() } else { format!("The work exited with code {} on {}.", a.exit_code, row.name) }); |
| 1303 | self.end_task(&task, a.exit_code, &why).await?; |
| 1304 | changed = true; |
| 1305 | } |
| 1306 | self.db |
| 1307 | .prepare("UPDATE runners SET work_id = NULL, work_kind = NULL, last_seen_at = ? WHERE id = ? AND work_id = ?") |
| 1308 | .bind(&[now().into(), row.id.as_str().into(), a.id.as_str().into()])? |
| 1309 | .run() |
| 1310 | .await?; |
| 1311 | Ok(Outcome::Ok(changed)) |
| 1312 | } |
| 1313 | |
| 1314 | /// `runner_remove_self`. |
| 1315 | pub async fn runner_remove_self(&self, a: RemoveSelfArgs) -> Result<Outcome<bool>> { |
| 1316 | let Some((row, _)) = self.authenticated(&a.auth).await? else { |
| 1317 | return Ok(Self::unknown_runner()); |
| 1318 | }; |
| 1319 | self.forget_runner(&row, &format!("The runner {} was removed from its machine.", row.name)).await?; |
| 1320 | self.audit(vec![entry( |
| 1321 | runner_actor(&row), |
| 1322 | "remove_runner", |
| 1323 | &row.workspace, |
| 1324 | row.repo.clone(), |
| 1325 | Some(row.name.clone()), |
| 1326 | format!("The self-hosted runner {} removed itself.", row.name), |
| 1327 | )]) |
| 1328 | .await; |
| 1329 | Ok(Outcome::Ok(true)) |
| 1330 | } |
| 1331 | |
| 1332 | /// A self-hosted job finished, however it did: its runner is free, and |
| 1333 | /// its time goes on the workspace's usage as self-hosted, at $0. |
| 1334 | pub async fn released(&self, job: &JobRow) -> Result<()> { |
| 1335 | let Some(runner_id) = &job.runner_id else { return Ok(()) }; |
| 1336 | self.db |
| 1337 | .prepare("UPDATE runners SET work_id = NULL, work_kind = NULL WHERE id = ? AND work_id = ?") |
| 1338 | .bind(&[runner_id.as_str().into(), job.id.as_str().into()])? |
| 1339 | .run() |
| 1340 | .await?; |
| 1341 | let (Some(started), Some(finished)) = (job.started_at.as_deref(), job.finished_at.as_deref().map(str::to_owned).or_else(|| Some(now()))) else { |
| 1342 | return Ok(()); |
| 1343 | }; |
| 1344 | let seconds = match (g1t_contracts::time::parse_rfc3339(started), g1t_contracts::time::parse_rfc3339(&finished)) { |
| 1345 | (Some(a), Some(b)) if b > a => ((b - a) / 1000).max(1), |
| 1346 | _ => 1, |
| 1347 | }; |
| 1348 | let run = self.run_row(&job.run_id).await?; |
| 1349 | let repo = run.as_ref().map(|r| r.repo.clone()); |
| 1350 | let recorded: Result<Outcome<bool>> = g1t_kit::call( |
| 1351 | &self.billing, |
| 1352 | "record_sandbox", |
| 1353 | &RecordSandboxArgs { |
| 1354 | workspace: job.namespace.to_lowercase(), |
| 1355 | seconds: u32::try_from(seconds).unwrap_or(u32::MAX), |
| 1356 | description: format!( |
| 1357 | "{} in {} on the self-hosted runner {}", |
| 1358 | job.name, |
| 1359 | repo.clone().unwrap_or_default(), |
| 1360 | job.runner_name.clone().unwrap_or_default() |
| 1361 | ), |
| 1362 | repo, |
| 1363 | reference: format!("selfhosted/{}/{}", job.id, started), |
| 1364 | kind: Some(ComputeKind::Workflow), |
| 1365 | cpu_seconds: None, |
| 1366 | // A workflow job on a self-hosted runner: no agent ran it and nobody asked an agent for it. |
| 1367 | agent: None, |
| 1368 | asked_by: None, |
| 1369 | reservation_id: None, |
| 1370 | self_hosted: true, |
| 1371 | instance: None, |
| 1372 | }, |
| 1373 | ) |
| 1374 | .await; |
| 1375 | if let Err(error) = recorded { |
| 1376 | worker::console_error!("actions: self-hosted time not recorded for {}: {error}", job.id); |
| 1377 | } |
| 1378 | Ok(()) |
| 1379 | } |
| 1380 | |
| 1381 | /// `runner` and `RUNNER_*` for a job on a self-hosted runner. |
| 1382 | pub async fn runner_context_for(&self, runner_id: &str, variables: &mut Map<String, Value>) -> Result<Value> { |
| 1383 | let mut context = g1t_actions::events::runner_context(); |
| 1384 | let Some(runner) = self.runner_by_id(runner_id).await? else { return Ok(context) }; |
| 1385 | let os = match runner.os.as_str() { |
| 1386 | "macos" => "macOS", |
| 1387 | "windows" => "Windows", |
| 1388 | _ => "Linux", |
| 1389 | }; |
| 1390 | let arch = if runner.arch == "arm64" { "ARM64" } else { "X64" }; |
| 1391 | context["name"] = json!(runner.name); |
| 1392 | context["os"] = json!(os); |
| 1393 | context["arch"] = json!(arch); |
| 1394 | context["environment"] = json!("self-hosted"); |
| 1395 | { |
| 1396 | let vars = variables; |
| 1397 | vars.insert("RUNNER_NAME".into(), json!(runner.name)); |
| 1398 | vars.insert("RUNNER_OS".into(), json!(os)); |
| 1399 | vars.insert("RUNNER_ARCH".into(), json!(arch)); |
| 1400 | vars.insert("RUNNER_ENVIRONMENT".into(), json!("self-hosted")); |
| 1401 | } |
| 1402 | Ok(context) |
| 1403 | } |
| 1404 | |
| 1405 | // --- Agent work, for the runner service ---------------------------------- |
| 1406 | |
| 1407 | /// `runner_route`: the labels g1t's own work in `repo` runs on, when |
| 1408 | /// its workspace (or the repository) sends it to self-hosted runners. |
| 1409 | pub async fn runner_route(&self, a: RouteArgs) -> Result<Option<Vec<String>>> { |
| 1410 | let repo_id = match self.repo_by_path(&a.repo).await? { |
| 1411 | Some(repo) => Some(repo.id), |
| 1412 | None => None, |
| 1413 | }; |
| 1414 | let settings = self.effective_runner_settings(&a.workspace, repo_id.as_deref()).await?; |
| 1415 | Ok(settings.agents_on_self_hosted.then_some(settings.agent_labels)) |
| 1416 | } |
| 1417 | |
| 1418 | async fn repo_by_path(&self, path: &RepoPath) -> Result<Option<Repo>> { |
| 1419 | let Some(actor) = self.workspace_actor(&path.namespace).await? else { return Ok(None) }; |
| 1420 | self.visible_repo(path, &Some(actor)).await |
| 1421 | } |
| 1422 | |
| 1423 | /// `enqueue_task`. |
| 1424 | pub async fn enqueue_task(&self, a: EnqueueTaskArgs) -> Result<Outcome<String>> { |
| 1425 | let Some(sealer) = &self.sealer else { |
| 1426 | return Ok(fail(FailureCode::Conflict, "Agent work cannot be handed to self-hosted runners yet: g1t's key for it is not set.")); |
| 1427 | }; |
| 1428 | let id = new_id("rtk", now_ms()); |
| 1429 | let sealed = sealer.seal(&serde_json::to_string(&a.env)?, &id); |
| 1430 | let repo_id = self.repo_by_path(&a.repo).await?.map(|r| r.id); |
| 1431 | let labels = match model::runner_labels(&a.labels, model::SELF_HOSTED, model::SELF_HOSTED) { |
| 1432 | Ok(labels) => labels, |
| 1433 | Err(problem) => return Ok(fail(FailureCode::Invalid, problem)), |
| 1434 | }; |
| 1435 | let at = now(); |
| 1436 | let inserted = self |
| 1437 | .db |
| 1438 | .prepare( |
| 1439 | "INSERT OR IGNORE INTO runner_tasks (id, sandbox, workspace, repo_id, repo, kind, title, labels, env, timeout_minutes, status, created_at, run_id) |
| 1440 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?) RETURNING id", |
| 1441 | ) |
| 1442 | .bind(&[ |
| 1443 | id.as_str().into(), |
| 1444 | a.sandbox.as_str().into(), |
| 1445 | a.workspace.to_lowercase().into(), |
| 1446 | optional(repo_id.as_deref()), |
| 1447 | format!("{}/{}", a.repo.namespace, a.repo.name).into(), |
| 1448 | a.kind.as_str().into(), |
| 1449 | a.title.chars().take(300).collect::<String>().into(), |
| 1450 | serde_json::to_string(&labels)?.into(), |
| 1451 | sealed.into(), |
| 1452 | a.timeout_minutes.max(1).into(), |
| 1453 | at.into(), |
| 1454 | optional(a.run_id.as_deref().filter(|id| !id.is_empty())), |
| 1455 | ])? |
| 1456 | .first::<Value>(None) |
| 1457 | .await?; |
| 1458 | if inserted.is_none() { |
| 1459 | return Ok(fail(FailureCode::Conflict, "That sandbox already handed work to a self-hosted runner.")); |
| 1460 | } |
| 1461 | Ok(Outcome::Ok(id)) |
| 1462 | } |
| 1463 | |
| 1464 | /// `cancel_task`: the sandbox stopped it. Its runner hears on its next |
| 1465 | /// poll; the sandbox does its own ending. |
| 1466 | pub async fn cancel_task(&self, a: CancelTaskArgs) -> Result<Outcome<bool>> { |
| 1467 | let done = self |
| 1468 | .db |
| 1469 | .prepare( |
| 1470 | "UPDATE runner_tasks SET status = 'completed', exit_code = -1, reason = ?, env = NULL, finished_at = ? |
| 1471 | WHERE sandbox = ? AND status != 'completed' RETURNING id", |
| 1472 | ) |
| 1473 | .bind(&[optional(a.reason.as_deref()), now().into(), a.sandbox.as_str().into()])? |
| 1474 | .first::<Value>(None) |
| 1475 | .await?; |
| 1476 | Ok(Outcome::Ok(done.is_some())) |
| 1477 | } |
| 1478 | |
| 1479 | /// Ends a task and tells the sandbox waiting on it, once. |
| 1480 | async fn end_task(&self, task: &TaskRow, exit_code: i32, reason: &str) -> Result<()> { |
| 1481 | let ended = self |
| 1482 | .db |
| 1483 | .prepare( |
| 1484 | "UPDATE runner_tasks SET status = 'completed', exit_code = ?, reason = ?, env = NULL, finished_at = ? |
| 1485 | WHERE id = ? AND status != 'completed' RETURNING id", |
| 1486 | ) |
| 1487 | .bind(&[exit_code.into(), optional(Some(reason).filter(|r| !r.is_empty())), now().into(), task.id.as_str().into()])? |
| 1488 | .first::<Value>(None) |
| 1489 | .await?; |
| 1490 | if ended.is_none() { |
| 1491 | return Ok(()); |
| 1492 | } |
| 1493 | if let Some(runner_id) = &task.runner_id { |
| 1494 | self.db |
| 1495 | .prepare("UPDATE runners SET work_id = NULL, work_kind = NULL WHERE id = ? AND work_id = ?") |
| 1496 | .bind(&[runner_id.as_str().into(), task.id.as_str().into()])? |
| 1497 | .run() |
| 1498 | .await?; |
| 1499 | } |
| 1500 | let told: Result<Value> = g1t_kit::call( |
| 1501 | &self.runner, |
| 1502 | "task_ended", |
| 1503 | &json!({ |
| 1504 | "sandbox": task.sandbox, |
| 1505 | "exitCode": exit_code, |
| 1506 | "reason": if reason.is_empty() { Value::Null } else { json!(reason) }, |
| 1507 | "runner": task.runner_name, |
| 1508 | }), |
| 1509 | ) |
| 1510 | .await; |
| 1511 | if let Err(error) = told { |
| 1512 | worker::console_error!("actions: the sandbox waiting on task {} was not told: {error}", task.id); |
| 1513 | } |
| 1514 | Ok(()) |
| 1515 | } |
| 1516 | |
| 1517 | /// `runner_activity`: what runs for the workspace now, in g1t's |
| 1518 | /// sandboxes and handed to its own runners. Its owners only, as its |
| 1519 | /// runners are. |
| 1520 | pub async fn runner_activity(&self, a: RunnerActivityArgs) -> Result<Outcome<RunnerActivity>> { |
| 1521 | let owner = RunnersOwner { repo: None, workspace: Some(a.workspace.clone()) }; |
| 1522 | let place = check!(self.runner_place(&a.actor, &owner, false).await?); |
| 1523 | let workspace = place.workspace; |
| 1524 | let cloud_jobs = self |
| 1525 | .db |
| 1526 | .prepare( |
| 1527 | "SELECT jobs.id, jobs.name, jobs.run_id, runs.repo, jobs.started_at FROM jobs JOIN runs ON runs.id = jobs.run_id |
| 1528 | WHERE lower(jobs.namespace) = ? AND jobs.status = 'in_progress' AND jobs.labels IS NULL |
| 1529 | ORDER BY jobs.started_at DESC LIMIT ?", |
| 1530 | ) |
| 1531 | .bind(&[workspace.as_str().into(), ACTIVITY_LIMIT.into()])? |
| 1532 | .all() |
| 1533 | .await? |
| 1534 | .results::<CloudJob>()?; |
| 1535 | let cloud_jobs_queued = self.queued_jobs(&workspace, false).await?; |
| 1536 | let self_hosted_jobs_queued = self.queued_jobs(&workspace, true).await?; |
| 1537 | let handed_over = self |
| 1538 | .db |
| 1539 | .prepare( |
| 1540 | "SELECT id, run_id, kind, title, repo, status, runner_name, created_at, started_at FROM runner_tasks |
| 1541 | WHERE status IN ('queued', 'in_progress') AND workspace = ? |
| 1542 | ORDER BY created_at DESC LIMIT ?", |
| 1543 | ) |
| 1544 | .bind(&[workspace.as_str().into(), ACTIVITY_LIMIT.into()])? |
| 1545 | .all() |
| 1546 | .await? |
| 1547 | .results::<HandedOverTask>()?; |
| 1548 | Ok(Outcome::Ok(RunnerActivity { cloud_jobs, cloud_jobs_queued, handed_over, self_hosted_jobs_queued })) |
| 1549 | } |
| 1550 | |
| 1551 | /// The workspace's workflow jobs queued for its own runners |
| 1552 | /// (`labelled`), or for g1t's sandboxes. |
| 1553 | async fn queued_jobs(&self, workspace: &str, labelled: bool) -> Result<u32> { |
| 1554 | let sql = if labelled { |
| 1555 | "SELECT COUNT(*) AS n FROM jobs WHERE lower(namespace) = ? AND status = 'queued' AND labels IS NOT NULL" |
| 1556 | } else { |
| 1557 | "SELECT COUNT(*) AS n FROM jobs WHERE lower(namespace) = ? AND status = 'queued' AND labels IS NULL" |
| 1558 | }; |
| 1559 | Ok(self.db.prepare(sql).bind(&[workspace.into()])?.first::<Count>(None).await?.map_or(0, |c| c.n)) |
| 1560 | } |
| 1561 | |
| 1562 | /// `stuck_jobs`: jobs of the viewer's workspaces that have waited ten |
| 1563 | /// minutes or more for a self-hosted runner while none that could take |
| 1564 | /// them is online, for Mission control's Needs you. |
| 1565 | pub async fn stuck_jobs(&self, a: StuckJobsArgs) -> Result<Vec<StuckJob>> { |
| 1566 | let Some(viewer) = a.viewer else { return Ok(Vec::new()) }; |
| 1567 | let at = now_ms(); |
| 1568 | let mut out = Vec::new(); |
| 1569 | for membership in viewer.workspaces.iter().take(20) { |
| 1570 | let workspace = membership.slug.to_lowercase(); |
| 1571 | #[derive(Deserialize)] |
| 1572 | struct Waiting { |
| 1573 | id: String, |
| 1574 | name: String, |
| 1575 | run_id: String, |
| 1576 | labels: String, |
| 1577 | queued_at: Option<String>, |
| 1578 | repo: String, |
| 1579 | repo_id: String, |
| 1580 | } |
| 1581 | let waiting = self |
| 1582 | .db |
| 1583 | .prepare( |
| 1584 | "SELECT jobs.id, jobs.name, jobs.run_id, jobs.labels, jobs.queued_at, runs.repo, jobs.repo_id FROM jobs JOIN runs ON runs.id = jobs.run_id |
| 1585 | WHERE jobs.status = 'queued' AND jobs.labels IS NOT NULL AND lower(jobs.namespace) = ? AND jobs.queued_at < ? |
| 1586 | ORDER BY jobs.rowid LIMIT 20", |
| 1587 | ) |
| 1588 | .bind(&[workspace.as_str().into(), rfc3339(at.saturating_sub(10 * 60 * 1000)).into()])? |
| 1589 | .all() |
| 1590 | .await? |
| 1591 | .results::<Waiting>()?; |
| 1592 | if waiting.is_empty() { |
| 1593 | continue; |
| 1594 | } |
| 1595 | let groups = self.group_rows(&workspace).await?; |
| 1596 | let online: Vec<RunnerRow> = self.runner_rows(&workspace).await?.into_iter().filter(|r| r.online(at)).collect(); |
| 1597 | // A repository's visibility is asked only when a host-mode runner |
| 1598 | // is online, since only those pass over a public repository's jobs. |
| 1599 | let any_host = online.iter().any(|runner| runner.isolation.as_deref() == Some("host")); |
| 1600 | let mut looked = Looked::default(); |
| 1601 | for job in waiting { |
| 1602 | let wanted = Wanted::from_stored(&serde_json::from_str::<Vec<String>>(&job.labels).unwrap_or_default()); |
| 1603 | let visibility = if any_host { self.visibility_once(&mut looked, &job.repo_id).await } else { None }; |
| 1604 | let served = online.iter().any(|runner| { |
| 1605 | let group = runner.group_id.as_ref().and_then(|id| groups.iter().find(|g| &g.id == id)); |
| 1606 | wanted.matches(&runner.labels(), group.map(|g| g.name.as_str())) |
| 1607 | && model::isolation_allows(runner.isolation.as_deref(), visibility.as_deref()) |
| 1608 | }); |
| 1609 | if !served { |
| 1610 | out.push(StuckJob { |
| 1611 | id: job.id, |
| 1612 | name: job.name, |
| 1613 | run_id: job.run_id, |
| 1614 | repo: job.repo, |
| 1615 | labels: wanted.describe(), |
| 1616 | queued_at: job.queued_at.unwrap_or_default(), |
| 1617 | }); |
| 1618 | } |
| 1619 | } |
| 1620 | } |
| 1621 | Ok(out) |
| 1622 | } |
| 1623 | |
| 1624 | // --- Every minute ---------------------------------------------------------- |
| 1625 | |
| 1626 | /// Jobs and tasks that waited too long, tasks whose runner went quiet, |
| 1627 | /// runners offline for weeks, and registration tokens long expired. |
| 1628 | pub async fn sweep_runners(&self, at: u64) -> Result<()> { |
| 1629 | let before = |ms: u64| rfc3339(at.saturating_sub(ms)); |
| 1630 | let waited = model::MAX_WAIT_HOURS * 60 * 60 * 1000; |
| 1631 | let stale = self |
| 1632 | .db |
| 1633 | .prepare("SELECT * FROM jobs WHERE status = 'queued' AND labels IS NOT NULL AND queued_at < ? LIMIT 50") |
| 1634 | .bind(&[before(waited).into()])? |
| 1635 | .all() |
| 1636 | .await? |
| 1637 | .results::<JobRow>()?; |
| 1638 | for job in stale { |
| 1639 | let stored: Vec<String> = job.labels.as_deref().and_then(|l| serde_json::from_str(l).ok()).unwrap_or_default(); |
| 1640 | let why = format!( |
| 1641 | "No self-hosted runner with labels {} took it within {} hours.", |
| 1642 | Wanted::from_stored(&stored).describe(), |
| 1643 | model::MAX_WAIT_HOURS |
| 1644 | ); |
| 1645 | self.finish_job(&job.id, "failure", Some(&why), None).await?; |
| 1646 | } |
| 1647 | let tasks = self |
| 1648 | .db |
| 1649 | .prepare( |
| 1650 | "SELECT * FROM runner_tasks WHERE (status = 'queued' AND created_at < ?) OR (status = 'in_progress' AND seen_at < ?) LIMIT 50", |
| 1651 | ) |
| 1652 | .bind(&[before(waited).into(), before(SILENT_MS).into()])? |
| 1653 | .all() |
| 1654 | .await? |
| 1655 | .results::<TaskRow>()?; |
| 1656 | for task in tasks { |
| 1657 | let why = if task.status == "queued" { |
| 1658 | format!("No self-hosted runner with labels {} took it within {} hours.", Wanted::of(&serde_json::from_str::<Vec<String>>(&task.labels).unwrap_or_default(), None).describe(), model::MAX_WAIT_HOURS) |
| 1659 | } else { |
| 1660 | format!("The self-hosted runner {} stopped answering.", task.runner_name.clone().unwrap_or_default()) |
| 1661 | }; |
| 1662 | self.end_task(&task, 1, &why).await?; |
| 1663 | } |
| 1664 | let forgotten = self |
| 1665 | .db |
| 1666 | .prepare("SELECT * FROM runners WHERE COALESCE(last_seen_at, created_at) < ? LIMIT 50") |
| 1667 | .bind(&[before(FORGET_OFFLINE_MS).into()])? |
| 1668 | .all() |
| 1669 | .await? |
| 1670 | .results::<RunnerRow>()?; |
| 1671 | for row in forgotten { |
| 1672 | self.forget_runner(&row, "The runner was offline for 14 days.").await?; |
| 1673 | } |
| 1674 | // An ephemeral runner that took its job and then went quiet. |
| 1675 | let spent = self |
| 1676 | .db |
| 1677 | .prepare("SELECT * FROM runners WHERE ephemeral = 1 AND spent = 1 AND work_id IS NULL AND COALESCE(last_seen_at, created_at) < ? LIMIT 50") |
| 1678 | .bind(&[before(60 * 60 * 1000).into()])? |
| 1679 | .all() |
| 1680 | .await? |
| 1681 | .results::<RunnerRow>()?; |
| 1682 | for row in spent { |
| 1683 | self.forget_runner(&row, "The ephemeral runner finished its job.").await?; |
| 1684 | } |
| 1685 | self.db |
| 1686 | .prepare("DELETE FROM runner_registrations WHERE expires_at < ?") |
| 1687 | .bind(&[before(24 * 60 * 60 * 1000).into()])? |
| 1688 | .run() |
| 1689 | .await?; |
| 1690 | Ok(()) |
| 1691 | } |
| 1692 | } |
| 1693 | |
| 1694 | #[derive(Deserialize)] |
| 1695 | struct IdRow { |
| 1696 | id: String, |
| 1697 | } |
| 1698 | |
| 1699 | /// Whether `runner` is online and could take a public repository's job: |
| 1700 | /// not host-mode, and its repository, group and labels let it. |
| 1701 | fn takes_public_job(runner: &RunnerRow, groups: &[GroupRow], repo_id: &str, run_repo: &str, wanted: &Wanted, at: u64) -> bool { |
| 1702 | let group = runner.group_id.as_ref().and_then(|id| groups.iter().find(|g| &g.id == id)); |
| 1703 | runner.online(at) |
| 1704 | && model::isolation_allows(runner.isolation.as_deref(), Some("public")) |
| 1705 | && runner.repo_id.as_deref().is_none_or(|repo| repo == repo_id) |
| 1706 | && (runner.repo_id.is_some() || group.is_none_or(|g| g.allows(run_repo))) |
| 1707 | && wanted.matches(&runner.labels(), group.map(|g| g.name.as_str())) |
| 1708 | } |
| 1709 | |
| 1710 | fn group_view(group: &GroupRow, runners: &[RunnerRow]) -> RunnerGroup { |
| 1711 | RunnerGroup { |
| 1712 | id: group.id.clone(), |
| 1713 | name: group.name.clone(), |
| 1714 | default: group.is_default != 0, |
| 1715 | repositories: group.repositories(), |
| 1716 | runners: runners.iter().filter(|r| r.group_id.as_deref() == Some(group.id.as_str())).count() as u32, |
| 1717 | updated_at: group.updated_at.clone(), |
| 1718 | } |
| 1719 | } |
| 1720 | |
| 1721 | #[cfg(test)] |
| 1722 | mod tests { |
| 1723 | use super::*; |
| 1724 | |
| 1725 | fn group(repositories: &[&str]) -> GroupRow { |
| 1726 | GroupRow { |
| 1727 | id: "rng_1".into(), |
| 1728 | name: "Default".into(), |
| 1729 | is_default: 1, |
| 1730 | repositories: serde_json::to_string(&repositories).unwrap(), |
| 1731 | updated_at: "2026-10-06T00:00:00Z".into(), |
| 1732 | } |
| 1733 | } |
| 1734 | |
| 1735 | #[test] |
| 1736 | fn being_seen_is_written_once_a_minute() { |
| 1737 | let at = 2_000_000_000_000; |
| 1738 | assert!(seen_is_stale(None, at)); |
| 1739 | assert!(!seen_is_stale(Some(&rfc3339(at - 59_000)), at)); |
| 1740 | assert!(seen_is_stale(Some(&rfc3339(at - 61_000)), at)); |
| 1741 | // A runner written down at the last moment is still online until the |
| 1742 | // next write, a poll later. |
| 1743 | assert!(SEEN_RESOLUTION_MS + model::MAX_POLL_WAIT_MS < model::ONLINE_WITHIN_MS); |
| 1744 | assert!(SEEN_RESOLUTION_MS < crate::SILENT_MS); |
| 1745 | } |
| 1746 | |
| 1747 | #[test] |
| 1748 | fn a_group_lets_its_repositories_use_it() { |
| 1749 | assert!(group(&[]).allows("acme/web")); |
| 1750 | assert!(group(&["web", "api"]).allows("acme/WEB")); |
| 1751 | assert!(group(&["web"]).allows("web")); |
| 1752 | assert!(!group(&["web"]).allows("acme/docs")); |
| 1753 | } |
| 1754 | |
| 1755 | #[test] |
| 1756 | fn names_are_checked() { |
| 1757 | assert_eq!(valid_name(" build-01 ").unwrap(), "build-01"); |
| 1758 | assert!(valid_name("").is_err()); |
| 1759 | assert!(valid_name("has space").is_err()); |
| 1760 | assert!(valid_name(&"x".repeat(65)).is_err()); |
| 1761 | assert!(valid_group_name("GPU machines").is_ok()); |
| 1762 | assert!(valid_group_name("").is_err()); |
| 1763 | } |
| 1764 | |
| 1765 | #[test] |
| 1766 | fn only_a_sandboxed_runner_that_could_take_it_serves_a_public_job() { |
| 1767 | let at = 1_000_000_000; |
| 1768 | let runner = |isolation: Option<&str>| RunnerRow { |
| 1769 | id: "rnr_1".into(), |
| 1770 | workspace: "acme".into(), |
| 1771 | repo_id: None, |
| 1772 | repo: None, |
| 1773 | group_id: None, |
| 1774 | name: "a".into(), |
| 1775 | labels: r#"["self-hosted","linux","x64"]"#.into(), |
| 1776 | os: "linux".into(), |
| 1777 | arch: "x64".into(), |
| 1778 | version: String::new(), |
| 1779 | ephemeral: 0, |
| 1780 | credential_hash: String::new(), |
| 1781 | previous_hash: None, |
| 1782 | rotated_at: String::new(), |
| 1783 | work_id: None, |
| 1784 | work_kind: None, |
| 1785 | spent: 0, |
| 1786 | last_seen_at: Some(rfc3339(at)), |
| 1787 | created_at: String::new(), |
| 1788 | created_by: None, |
| 1789 | isolation: isolation.map(str::to_owned), |
| 1790 | }; |
| 1791 | let wanted = Wanted::of(&["self-hosted".to_owned(), "linux".to_owned()], None); |
| 1792 | let takes = |row: &RunnerRow| takes_public_job(row, &[], "rep_1", "acme/web", &wanted, at); |
| 1793 | assert!(!takes(&runner(Some("host")))); |
| 1794 | assert!(takes(&runner(Some("container")))); |
| 1795 | assert!(takes(&runner(Some("nested-container")))); |
| 1796 | assert!(takes(&runner(None)), "an older runner, which refuses for itself"); |
| 1797 | assert!(!takes(&RunnerRow { repo_id: Some("rep_2".into()), ..runner(None) }), "another repository's"); |
| 1798 | assert!(!takes(&RunnerRow { labels: r#"["self-hosted","macos"]"#.into(), ..runner(None) })); |
| 1799 | assert!(!takes(&RunnerRow { last_seen_at: None, ..runner(None) }), "offline"); |
| 1800 | } |
| 1801 | |
| 1802 | #[test] |
| 1803 | fn online_means_seen_in_the_last_ninety_seconds() { |
| 1804 | let row = RunnerRow { |
| 1805 | id: "rnr_1".into(), |
| 1806 | workspace: "acme".into(), |
| 1807 | repo_id: None, |
| 1808 | repo: None, |
| 1809 | group_id: None, |
| 1810 | name: "a".into(), |
| 1811 | labels: "[]".into(), |
| 1812 | os: "linux".into(), |
| 1813 | arch: "x64".into(), |
| 1814 | version: String::new(), |
| 1815 | ephemeral: 0, |
| 1816 | credential_hash: String::new(), |
| 1817 | previous_hash: None, |
| 1818 | rotated_at: String::new(), |
| 1819 | work_id: None, |
| 1820 | work_kind: None, |
| 1821 | spent: 0, |
| 1822 | last_seen_at: Some(rfc3339(1_000_000_000)), |
| 1823 | created_at: String::new(), |
| 1824 | created_by: None, |
| 1825 | isolation: None, |
| 1826 | }; |
| 1827 | assert!(row.online(1_000_000_000 + 60_000)); |
| 1828 | assert!(!row.online(1_000_000_000 + 120_000)); |
| 1829 | assert_eq!(row.workspace, "acme"); |
| 1830 | } |
| 1831 | } |