flagon-io/g1t

public

1	//! A run's life: made, its jobs waiting on the jobs they need, each job
2	//! skipped or expanded into its matrix and queued, started in a sandbox
3	//! when its workspace has room, reporting its steps and logs as it goes,
4	//! and finished; the run finishes with its last job.
5
6	use g1t_actions::events::{RunInfo, runner_context};
7	use g1t_actions::expr::{self, Scope, Status};
8	use g1t_actions::matrix;
9	use g1t_actions::workflow::{self, Workflow};
10	use g1t_contracts::access::Capability;
11	use g1t_contracts::actions::{JobCallArgs, RunActionArgs, RunApproval, StartJobArgs, WorkflowRun};
12	use g1t_contracts::identity::{CreateJobTokenArgs, CreatedAccessToken, RevokeJobTokensArgs};
13	use g1t_contracts::repos::{Repo, RepoPath};
14	use g1t_contracts::time::rfc3339;
15	use g1t_contracts::{FailureCode, Outcome, new_id};
16	use g1t_kit::now_ms;
17	use g1t_secrets::{random_hex, same, sha256_hex};
18	use serde::Deserialize;
19	use serde_json::{Map, Value, json};
20	use worker::Result;
21
22	use crate::protection::Gate;
23	use crate::sync::WorkflowRow;
24	use crate::{Actions, Count, MAX_TIMEOUT_MINUTES, RUNNING_PER_WORKSPACE, SELF_HOSTED_MAX_TIMEOUT_MINUTES, SILENT_MS, SITE, check, fail, optional, repo_path};
25	use g1t_contracts::runners::{Wanted, waiting_reason};
26
27	/// The most log one job keeps, in bytes; past it, the log says so and stops.
28	const MAX_LOG_BYTES: usize = 4 * 1024 * 1024;
29	/// The most a single log report may add.
30	const MAX_CHUNK_BYTES: usize = 256 * 1024;
31	const MAX_ANNOTATIONS: usize = 50;
32	/// A batch of a job's log, under the next `seq` of that job's, worked out
33	/// in the same statement (SQLite runs one write at a time). `?4` is the
34	/// runner's number for the batch (null from runners that send none): a
35	/// batch sent again, after an answer the runner did not hear, has the
36	/// number of one already kept, and is not kept twice (migration 0016).
37	const LOG_INSERT: &str = "INSERT INTO logs (job_id, seq, step, text, chunk)
38	SELECT ?1, COALESCE(MAX(seq), 0) + 1, ?2, ?3, ?4 FROM logs WHERE job_id = ?1
39	ON CONFLICT (job_id, chunk) DO NOTHING";
40	/// The most one step's job summary keeps, in bytes, and how many steps of a
41	/// job may have one, as on GitHub.
42	pub(crate) const MAX_SUMMARY_BYTES: usize = 1024 * 1024;
43	pub(crate) const MAX_SUMMARIES: u32 = 20;
44
45	/// Whether a step's summary of `adding` bytes is kept: a job keeps up to
46	/// 20 steps' summaries (`steps` it has, `mine` this step's bytes so far)
47	/// of up to 1 MiB each.
48	pub(crate) fn summary_fits(steps: u32, mine: Option<usize>, adding: usize) -> bool {
49	if adding == 0 {
50	return false;
51	}
52	match mine {
53	Some(used) => used + adding <= MAX_SUMMARY_BYTES,
54	None => steps < MAX_SUMMARIES && adding <= MAX_SUMMARY_BYTES,
55	}
56	}
57
58	/// The repository whose unfinished runs `event` stops, by id: one deleted,
59	/// or archived (not unarchived).
60	pub fn stops_runs(event: &g1t_contracts::events::Event) -> Option<String> {
61	let stops = match event.kind.as_str() {
62	"repo.deleted" => true,
63	"repo.archived" => event.data["archived"].as_bool() != Some(false),
64	_ => false,
65	};
66	if !stops {
67	return None;
68	}
69	event.repo_id.clone().or_else(\|\| event.data["repoId"].as_str().map(str::to_owned))
70	}
71
72	pub struct NewRun {
73	pub repo: Repo,
74	pub path: String,
75	pub source: String,
76	pub workflow: Workflow,
77	pub info: RunInfo,
78	pub action: Option<String>,
79	pub pull: Option<u32>,
80	pub title: String,
81	pub inputs: Map<String, Value>,
82	pub event_key: String,
83	pub actor_id: Option<String>,
84	pub actor: Option<String>,
85	pub trusted: bool,
86	/// Why the run waits for someone with the Write role to approve it
87	/// before anything starts: a pull request from outside (protection.rs).
88	pub approval: Option<String>,
89	}
90
91	#[derive(Clone, Deserialize)]
92	pub struct RunRow {
93	pub id: String,
94	pub workflow_id: String,
95	pub repo_id: String,
96	pub repo: String,
97	pub path: String,
98	pub name: String,
99	pub title: String,
100	pub number: u64,
101	pub attempt: u64,
102	pub event: String,
103	pub action: Option<String>,
104	pub git_ref: String,
105	pub sha: String,
106	pub pull: Option<u32>,
107	pub status: String,
108	pub conclusion: Option<String>,
109	pub error: Option<String>,
110	pub actor: Option<String>,
111	pub actor_id: Option<String>,
112	pub source: String,
113	pub info: String,
114	pub inputs: String,
115	pub trusted: u32,
116	pub concurrency_group: Option<String>,
117	pub created_at: String,
118	pub started_at: Option<String>,
119	pub finished_at: Option<String>,
120	/// JSON `RunApproval`, for a run that needed approval (migration 0006).
121	#[serde(default)]
122	pub approval: Option<String>,
123	/// Whether its concurrency group cancels what it replaces.
124	#[serde(default)]
125	pub cancel_in_progress: u32,
126	/// Who started the current attempt, once it is re-run (migration 0009).
127	#[serde(default)]
128	pub triggering_actor: Option<String>,
129	/// The current attempt runs with debug logging.
130	#[serde(default)]
131	pub debug: u32,
132	}
133
134	impl RunRow {
135	pub fn approval(&self) -> Option<RunApproval> {
136	self.approval.as_deref().and_then(\|text\| serde_json::from_str(text).ok())
137	}
138
139	pub fn info(&self) -> RunInfo {
140	let mut info: RunInfo = serde_json::from_str(&self.info).unwrap_or_default();
141	info.run_id = self.id.clone();
142	info.run_number = self.number;
143	info.run_attempt = self.attempt;
144	info.workflow_path = self.path.clone();
145	if info.workflow.is_empty() {
146	info.workflow = self.name.clone();
147	}
148	info
149	}
150
151	pub fn inputs(&self) -> Map<String, Value> {
152	serde_json::from_str(&self.inputs).unwrap_or_default()
153	}
154
155	pub fn summary(&self) -> WorkflowRun {
156	WorkflowRun {
157	id: self.id.clone(),
158	workflow_id: self.workflow_id.clone(),
159	path: self.path.clone(),
160	name: self.name.clone(),
161	title: self.title.clone(),
162	number: self.number,
163	attempt: self.attempt,
164	event: self.event.clone(),
165	git_ref: self.git_ref.clone(),
166	sha: self.sha.clone(),
167	pull: self.pull,
168	status: self.status.clone(),
169	conclusion: self.conclusion.clone(),
170	error: self.error.clone(),
171	actor: self.actor.clone(),
172	created_at: self.created_at.clone(),
173	started_at: self.started_at.clone(),
174	finished_at: self.finished_at.clone(),
175	}
176	}
177	}
178
179	#[derive(Clone, Deserialize)]
180	pub struct JobRow {
181	pub id: String,
182	pub run_id: String,
183	pub repo_id: String,
184	pub namespace: String,
185	pub key: String,
186	pub ordinal: u32,
187	pub name: String,
188	pub needs: String,
189	pub matrix: Option<String>,
190	pub status: String,
191	pub conclusion: Option<String>,
192	pub steps: String,
193	pub annotations: String,
194	pub outputs: String,
195	pub reason: Option<String>,
196	pub token_hash: Option<String>,
197	pub timeout_minutes: u32,
198	pub continue_on_error: u32,
199	pub max_parallel: Option<u32>,
200	/// Set for a job that calls a reusable workflow, and for that
201	/// workflow's jobs (see migration 0002).
202	pub call: Option<String>,
203	pub seen_at: Option<String>,
204	pub started_at: Option<String>,
205	pub finished_at: Option<String>,
206	/// For a job whose `runs-on` names self-hosted runners: what it asks
207	/// for, as a JSON array (see `g1t_contracts::runners::Wanted`), when it
208	/// started waiting, and the runner that took it (migration 0004).
209	#[serde(default)]
210	pub labels: Option<String>,
211	#[serde(default)]
212	pub queued_at: Option<String>,
213	#[serde(default)]
214	pub runner_id: Option<String>,
215	#[serde(default)]
216	pub runner_name: Option<String>,
217	/// The environment it names, read when its needs were done; a job its
218	/// rules hold is `pending` (migration 0006).
219	#[serde(default)]
220	pub environment: Option<String>,
221	/// Its own `concurrency` group, and whether that cancels what it replaces.
222	#[serde(default)]
223	pub concurrency_group: Option<String>,
224	#[serde(default)]
225	pub cancel_in_progress: u32,
226	/// When it was told to stop, while it runs its cleanup steps
227	/// (migration 0009).
228	#[serde(default)]
229	pub cancel_requested_at: Option<String>,
230	}
231
232	/// How long a cancelled job has to run its `if: always()` and `cancelled()`
233	/// steps and its post steps before it is stopped outright, as on GitHub.
234	pub const CANCEL_GRACE_MS: u64 = 5 * 60 * 1000;
235
236	/// Which jobs a re-run runs again.
237	#[derive(Clone, Copy, Debug, PartialEq)]
238	pub enum Rerun<'a> {
239	All,
240	/// Those that did not succeed.
241	Failed,
242	/// One job's key.
243	Job(&'a str),
244	}
245
246	/// The keys a re-run runs again, in `order`: those `which` names and,
247	/// transitively, every key that needs one of them. `needs` gives a key's
248	/// needs; `succeeded` whether every job of a key succeeded.
249	pub fn rerun_keys(order: &[&str], needs: impl Fn(&str) -> Vec<String>, succeeded: impl Fn(&str) -> bool, which: Rerun) -> Vec<String> {
250	let mut again: Vec<String> = Vec::new();
251	for key in order {
252	let named = match which {
253	Rerun::All => true,
254	Rerun::Failed => !succeeded(key),
255	Rerun::Job(job) => *key == job,
256	};
257	if named \|\| needs(key).iter().any(\|need\| again.contains(need)) {
258	again.push((*key).to_owned());
259	}
260	}
261	again
262	}
263
264	/// The key under `jobs:` a job belongs to: a called workflow's jobs
265	/// (`build/test`, or `build[1]/test` for a matrix's second call) run again
266	/// with the job that calls it (`build`).
267	pub fn top_key(key: &str) -> &str {
268	let first = key.split('/').next().unwrap_or(key);
269	first.split('[').next().unwrap_or(first)
270	}
271
272	/// What the keys of the jobs a caller's call makes start with: the
273	/// caller's key, and for a matrix the combination's index after it
274	/// (`build[1]`), so each combination's call has jobs of its own. A job id
275	/// has no `[`, so the two never meet.
276	fn call_prefix(key: &str, index: Option<usize>) -> String {
277	match index {
278	Some(index) => format!("{key}[{index}]"),
279	None => key.to_owned(),
280	}
281	}
282
283	/// Whether a run may call one more reusable workflow, `file`, `depth`
284	/// levels below the workflow it started from, having called `called`
285	/// already (one name per call; repeats are one workflow). As on GitHub: ten
286	/// levels of workflows at most, the top one and nine reusable ones below
287	/// it, and 50 different reusable workflows in a run.
288	fn call_allowed(depth: u64, called: &[&str], file: &str) -> std::result::Result<(), String> {
289	if depth > MAX_CALL_DEPTH {
290	return Err(format!(
291	"`{file}` would be the {}th level of workflows: a run connects at most ten, the workflow it started from and nine levels of reusable workflows below it.",
292	depth + 1
293	));
294	}
295	let mut unique: Vec<&str> = called.to_vec();
296	unique.push(file);
297	unique.sort_unstable();
298	unique.dedup();
299	if unique.len() > MAX_CALLED_WORKFLOWS {
300	return Err(format!("Calling `{file}` would make {} reusable workflows in this run: a run calls at most {MAX_CALLED_WORKFLOWS} different ones.", unique.len()));
301	}
302	Ok(())
303	}
304
305	/// What a job is shown as: its `name` (else its id), under the job whose
306	/// call made it (`caller`), read with its matrix when it has an
307	/// expression, else with the matrix's values after it.
308	fn shown_name(job: &workflow::Job, caller: Option<&str>, combination: &Map<String, Value>, scope: &Scope) -> String {
309	let base = job.name.clone().unwrap_or(job.id.clone());
310	let base = match caller {
311	Some(caller) => format!("{caller} / {base}"),
312	None => base,
313	};
314	if expr::has_expression(&base) {
315	expr::interpolate(&base, scope).unwrap_or(base)
316	} else if job.matrix.is_some() {
317	matrix::job_name(&base, combination)
318	} else {
319	base
320	}
321	}
322
323	/// Debug logging for a job, as a re-run with it turned on gives GitHub's:
324	/// `RUNNER_DEBUG=1` and `runner.debug`, and `ACTIONS_STEP_DEBUG` and
325	/// `ACTIONS_RUNNER_DEBUG` set, which the runner reads as the secrets of
326	/// those names (`::debug::` lines shown, and how each step's `if` read).
327	pub fn debug_logging(variables: &mut Map<String, Value>, runner: &mut Value) {
328	variables.insert("RUNNER_DEBUG".into(), json!("1"));
329	variables.insert("ACTIONS_STEP_DEBUG".into(), json!("true"));
330	variables.insert("ACTIONS_RUNNER_DEBUG".into(), json!("true"));
331	runner["debug"] = json!("1");
332	}
333
334	impl JobRow {
335	pub fn needs(&self) -> Vec<String> {
336	serde_json::from_str(&self.needs).unwrap_or_default()
337	}
338
339	pub fn call(&self) -> Option<Value> {
340	self.call.as_deref().and_then(\|call\| serde_json::from_str(call).ok())
341	}
342
343	/// For a job that calls a reusable workflow: what the keys of the jobs
344	/// its call makes start with (`call_prefix`).
345	pub fn call_prefix(&self) -> Option<String> {
346	let call = self.call().filter(\|call\| call["role"] == "caller")?;
347	Some(call["prefix"].as_str().unwrap_or(&self.key).to_owned())
348	}
349
350	/// A job that calls a reusable workflow and has not called it yet: it
351	/// waits for room under its matrix's `max-parallel`, or in its
352	/// concurrency group.
353	pub fn holds_call(&self) -> bool {
354	self.status == "waiting" && self.call_prefix().is_some()
355	}
356
357	/// For a job of a called workflow: that workflow, the job's own id in
358	/// it, and the job.
359	pub fn callee(&self) -> Option<(Workflow, workflow::Job, Value)> {
360	let call = self.call().filter(\|call\| call["role"] == "callee")?;
361	let called = workflow::parse(call["source"].as_str()?).ok()?;
362	let job = called.jobs.iter().find(\|job\| call["job"].as_str() == Some(job.id.as_str()))?.clone();
363	Some((called, job, call))
364	}
365	}
366
367	/// A job to decide on: its key, its definition, and what it needs, as
368	/// (name in `needs`, key of the jobs).
369	type Unit = (String, workflow::Job, Vec<(String, String)>);
370
371	/// How many levels of reusable workflows may sit below the workflow a run
372	/// started from, as on GitHub (ten levels of workflows in all).
373	const MAX_CALL_DEPTH: u64 = 9;
374	/// How many different reusable workflows a run may call, as on GitHub.
375	const MAX_CALLED_WORKFLOWS: usize = 50;
376
377	/// What the jobs of one key came to, for `needs.<key>`.
378	fn key_result(rows: &[&JobRow]) -> &'static str {
379	let failed = \|row: &&&JobRow\| row.conclusion.as_deref() == Some("failure") && row.continue_on_error == 0;
380	if rows.iter().any(\|row\| failed(&row)) {
381	"failure"
382	} else if rows.iter().any(\|row\| row.conclusion.as_deref() == Some("cancelled")) {
383	"cancelled"
384	} else if rows.iter().all(\|row\| row.conclusion.as_deref() == Some("skipped")) {
385	"skipped"
386	} else {
387	"success"
388	}
389	}
390
391	/// Whether any job before `key` failed: one it needs, or one those need,
392	/// however far back. A job after a skipped one still sees the failure
393	/// that skipped it, as GitHub's failure() does. `needs_of`: each key's
394	/// needs, as keys; `failed`: whether a key's jobs came to a failure.
395	fn ancestor_failed(needs_of: &std::collections::HashMap<&str, Vec<&str>>, key: &str, failed: impl Fn(&str) -> bool) -> bool {
396	let mut seen = std::collections::HashSet::new();
397	let mut stack: Vec<&str> = needs_of.get(key).cloned().unwrap_or_default();
398	while let Some(next) = stack.pop() {
399	if !seen.insert(next) {
400	continue;
401	}
402	if failed(next) {
403	return true;
404	}
405	stack.extend(needs_of.get(next).into_iter().flatten().copied());
406	}
407	false
408	}
409
410	/// A job's `environment:`, as deployments read it: its name, the address in
411	/// `environment.url` once its expressions are filled in from the run, and
412	/// whether the job deploys to it. A job with `deployment: false` only reads
413	/// the environment's secrets and variables, and makes no deployment.
414	#[derive(Clone, Debug, Default, PartialEq)]
415	pub(crate) struct JobEnvironment {
416	pub(crate) name: String,
417	pub(crate) url: Option<String>,
418	pub(crate) deploys: bool,
419	}
420
421	/// The environment `environment:` names, as written in `raw` (a job), with
422	/// `contexts` to fill in expressions: null when it names none, or only
423	/// through an expression this cannot read before the job runs.
424	pub(crate) fn environment_of(raw: &Value, contexts: &Map<String, Value>) -> Option<JobEnvironment> {
425	let scope = Scope { contexts, status: Status::Success, hash_files: None };
426	let plain = \|value: &Value\| -> Option<String> {
427	let text = match value {
428	Value::String(text) if text.contains("${{") => expr::interpolate_value(value, &scope).ok().map(\|v\| expr::to_text(&v))?,
429	Value::String(text) => text.clone(),
430	_ => return None,
431	};
432	let text = text.trim().to_owned();
433	(!text.is_empty()).then_some(text)
434	};
435	match raw.get("environment")? {
436	name @ Value::String(_) => Some(JobEnvironment { name: plain(name)?, url: None, deploys: workflow::records_deployment(raw) }),
437	Value::Object(env) => {
438	let name = plain(env.get("name")?)?;
439	let url = env
440	.get("url")
441	.and_then(plain)
442	.filter(\|url\| url.starts_with("https://") \|\| url.starts_with("http://"));
443	Some(JobEnvironment { name, url, deploys: workflow::records_deployment(raw) })
444	}
445	_ => None,
446	}
447	}
448
449	/// The contexts a job's `runs-on` and `environment` are read with before it
450	/// runs: the run's `github`, its inputs, and the job's matrix.
451	fn start_contexts(run: &RunRow, job: &JobRow, key: &str) -> Map<String, Value> {
452	let mut contexts = Map::new();
453	contexts.insert("github".into(), run.info().context(key, "", run.action.as_deref()));
454	contexts.insert("inputs".into(), Value::Object(run.inputs()));
455	contexts.insert("matrix".into(), job.matrix.as_deref().and_then(\|m\| serde_json::from_str(m).ok()).unwrap_or_else(\|\| json!({})));
456	contexts
457	}
458
459	/// The job as its workflow (or the workflow it calls) defines it.
460	fn job_spec(run: &RunRow, job: &JobRow) -> Option<workflow::Job> {
461	match job.callee() {
462	Some((_, spec, _)) => Some(spec),
463	None => workflow::parse(&run.source).ok().and_then(\|w\| w.jobs.into_iter().find(\|j\| j.id == job.key)),
464	}
465	}
466
467	/// The environment a job deploys to, if it deploys: by the name read when
468	/// its needs were done (which may have needed their outputs), else as
469	/// its `environment:` reads now.
470	pub(crate) fn deploys_to(run: &RunRow, job: &JobRow) -> Option<JobEnvironment> {
471	let spec = job_spec(run, job)?;
472	let read = environment_of(&spec.raw, &start_contexts(run, job, &spec.id));
473	let env = match (read, &job.environment) {
474	(Some(env), Some(name)) => Some(JobEnvironment { name: name.clone(), ..env }),
475	(Some(env), None) => Some(env),
476	(None, Some(name)) => Some(JobEnvironment { name: name.clone(), url: None, deploys: workflow::records_deployment(&spec.raw) }),
477	(None, None) => None,
478	};
479	env.filter(\|env\| env.deploys)
480	}
481
482	/// What the runner is told about a job it starts, from its workflow: the
483	/// environment it names plainly, and the machine its `runs-on` asks for
484	/// (`instance_for`; none for the standard one).
485	#[derive(Default)]
486	struct StartDetails {
487	environment: Option<String>,
488	instance: Option<String>,
489	}
490
491	fn start_details(run: &RunRow, job: &JobRow) -> StartDetails {
492	let spec = match job.callee() {
493	Some((_, spec, _)) => spec,
494	None => match workflow::parse(&run.source).ok().and_then(\|w\| w.jobs.into_iter().find(\|j\| j.id == job.key)) {
495	Some(spec) => spec,
496	None => return StartDetails::default(),
497	},
498	};
499	// As read when its needs were done, an expression's included.
500	let environment = job.environment.clone();
501	// `runs-on` as the job was queued with: its matrix and the run's
502	// inputs. A label that needs more than those is the standard machine.
503	let contexts = start_contexts(run, job, &spec.id);
504	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
505	let labels: Vec<String> = match expr::interpolate_value(&spec.runs_on, &scope).unwrap_or(Value::Null) {
506	Value::String(label) => vec![label],
507	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
508	Value::Object(given) => match given.get("labels") {
509	Some(Value::Array(labels)) => labels.iter().map(expr::to_text).collect(),
510	Some(label) => vec![expr::to_text(label)],
511	None => Vec::new(),
512	},
513	_ => Vec::new(),
514	};
515	let instance = g1t_contracts::actions::instance_for(&labels);
516	StartDetails {
517	environment,
518	instance: (instance != g1t_contracts::actions::STANDARD_INSTANCE).then(\|\| instance.label.to_owned()),
519	}
520	}
521
522	fn now() -> String {
523	rfc3339(now_ms())
524	}
525
526	/// How a finished run went for one environment, from the conclusions of
527	/// the jobs that deploy to it: failed if any failed, an error if any was
528	/// cancelled, else a success; nothing when none ran (all skipped).
529	pub(crate) fn deployment_outcome(conclusions: &[Option<String>]) -> Option<&'static str> {
530	let ran: Vec<&str> = conclusions.iter().flatten().map(String::as_str).filter(\|c\| *c != "skipped").collect();
531	if ran.is_empty() {
532	return None;
533	}
534	if ran.iter().any(\|c\| c == "failure" \|\| c == "timed_out") {
535	return Some("failure");
536	}
537	if ran.contains(&"cancelled") {
538	return Some("error");
539	}
540	Some("success")
541	}
542
543	impl Actions {
544	pub async fn run_row(&self, id: &str) -> Result<Option<RunRow>> {
545	self.db.prepare("SELECT * FROM runs WHERE id = ?").bind(&[id.into()])?.first::<RunRow>(None).await
546	}
547
548	pub async fn job_rows(&self, run_id: &str) -> Result<Vec<JobRow>> {
549	self.db
550	.prepare("SELECT * FROM jobs WHERE run_id = ? ORDER BY rowid")
551	.bind(&[run_id.into()])?
552	.all()
553	.await?
554	.results::<JobRow>()
555	}
556
557	pub async fn run_summary(&self, id: &str) -> Result<Outcome<WorkflowRun>> {
558	Ok(match self.run_row(id).await? {
559	Some(row) => Outcome::Ok(row.summary()),
560	None => fail(FailureCode::NotFound, "No such run."),
561	})
562	}
563
564	/// The contexts every expression outside a job's steps may use.
565	fn base_contexts(run: &RunRow, vars: &Map<String, Value>, job: &str) -> Map<String, Value> {
566	let mut contexts = Map::new();
567	contexts.insert("github".into(), run.info().context(job, "", run.action.as_deref()));
568	contexts.insert("inputs".into(), Value::Object(run.inputs()));
569	contexts.insert("vars".into(), Value::Object(vars.clone()));
570	contexts.insert("needs".into(), json!({}));
571	contexts.insert("runner".into(), runner_context());
572	contexts
573	}
574
575	/// Whether `namespace` is a person's own workspace: usernames and
576	/// workspaces share one namespace, so it is when someone has that
577	/// username. Kept in the isolate for a few minutes, so a burst of
578	/// runs asks identity once.
579	async fn personal_namespace(&self, namespace: &str) -> Result<bool> {
580	const KEEP_MS: u64 = 5 * 60 * 1000;
581	thread_local! {
582	static PERSONAL: std::cell::RefCell<std::collections::HashMap<String, (bool, u64)>> = Default::default();
583	}
584	let now = now_ms();
585	let key = namespace.to_ascii_lowercase();
586	if let Some(personal) = PERSONAL.with(\|kept\| kept.borrow().get(&key).filter(\|(_, at)\| now.saturating_sub(at) < KEEP_MS).map(\|(personal, _)\| personal)) {
587	return Ok(personal);
588	}
589	let found: g1t_contracts::Viewer = g1t_kit::call(
590	&self.identity,
591	"user_by_username",
592	&g1t_contracts::identity::UsernameArgs { username: key.clone() },
593	)
594	.await?;
595	let personal = found.is_some_and(\|user\| user.username.eq_ignore_ascii_case(namespace));
596	PERSONAL.with(\|kept\| {
597	let mut kept = kept.borrow_mut();
598	if kept.len() > 1000 {
599	kept.clear();
600	}
601	kept.insert(key, (personal, now));
602	});
603	Ok(personal)
604	}
605
606	/// Makes a run and its jobs, and starts what can start. `None` when the
607	/// event already started this workflow.
608	pub async fn create_run(&self, new: NewRun) -> Result<Option<String>> {
609	// Nothing starts on an archived repository. A deleted one is never
610	// found to start anything on.
611	if new.repo.archived() {
612	return Ok(None);
613	}
614	let workflow_row = self.workflow_row(&new.repo, &new.path, &new.workflow.display_name(&new.path), &new.source).await?;
615	let numbered = self
616	.db
617	.prepare("UPDATE workflows SET run_count = run_count + 1 WHERE id = ? RETURNING run_count AS n")
618	.bind(&[workflow_row.id.as_str().into()])?
619	.first::<Count>(None)
620	.await?
621	.map_or(1, \|count\| count.n);
622	let id = new_id("run", now_ms());
623	let mut info = new.info.clone();
624	// A personal workspace's repositories are a `User`'s, as on GitHub.
625	if self.personal_namespace(&new.repo.namespace).await? {
626	crate::payload::owned_by_person(&mut info.event, &new.repo.namespace);
627	}
628	info.workflow = new.workflow.display_name(&new.path);
629	info.workflow_path = new.path.clone();
630	info.run_id = id.clone();
631	info.run_number = u64::from(numbered);
632	let vars = self
633	.variables_for(&new.repo.id, &format!("{}/{}", new.repo.namespace, new.repo.name), None, new.trusted)
634	.await?;
635
636	// run-name and the concurrency group read github, inputs and vars.
637	let mut contexts = Map::new();
638	contexts.insert("github".into(), info.context("", "", new.action.as_deref()));
639	contexts.insert("inputs".into(), Value::Object(new.inputs.clone()));
640	contexts.insert("vars".into(), Value::Object(vars));
641	let scope = Scope {
642	contexts: &contexts,
643	status: Status::Success,
644	hash_files: None,
645	};
646	let title = new
647	.workflow
648	.run_name
649	.as_deref()
650	.and_then(\|run_name\| expr::interpolate(run_name, &scope).ok())
651	.filter(\|title\| !title.trim().is_empty())
652	.unwrap_or(new.title.clone());
653	let group = new.workflow.concurrency.as_ref().and_then(\|c\| expr::interpolate(&c.group, &scope).ok());
654	let cancel_in_progress = new
655	.workflow
656	.concurrency
657	.as_ref()
658	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
659	.is_some_and(\|value\| expr::truthy(&value));
660
661	let approval = new.approval.as_ref().map(\|reason\| RunApproval { state: "required".into(), reason: reason.clone(), approved_by: None });
662	let inserted = self
663	.db
664	.prepare(
665	"INSERT OR IGNORE INTO runs (id, workflow_id, repo_id, repo, path, name, title, number, event, action, git_ref, sha,
666	pull, status, actor, actor_id, source, info, inputs, trusted, concurrency_group, event_key, created_at,
667	approval, cancel_in_progress)
668	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",
669	)
670	.bind(&[
671	id.as_str().into(),
672	workflow_row.id.as_str().into(),
673	new.repo.id.as_str().into(),
674	format!("{}/{}", new.repo.namespace, new.repo.name).into(),
675	new.path.as_str().into(),
676	info.workflow.as_str().into(),
677	title.as_str().into(),
678	numbered.into(),
679	info.event_name.as_str().into(),
680	optional(new.action.as_deref()),
681	info.git_ref.as_str().into(),
682	info.sha.as_str().into(),
683	new.pull.map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
684	if approval.is_some() { "action_required" } else { "queued" }.into(),
685	optional(new.actor.as_deref()),
686	optional(new.actor_id.as_deref()),
687	new.source.as_str().into(),
688	serde_json::to_string(&info)?.into(),
689	serde_json::to_string(&new.inputs)?.into(),
690	u32::from(new.trusted).into(),
691	optional(group.as_deref()),
692	new.event_key.as_str().into(),
693	now().into(),
694	approval.as_ref().map(serde_json::to_string).transpose()?.as_deref().map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
695	u32::from(cancel_in_progress).into(),
696	])?
697	.first::<Value>(None)
698	.await?;
699	if inserted.is_none() {
700	return Ok(None);
701	}
702	if let Some(run) = self.run_row(&id).await? {
703	self.report_pending(&run).await?;
704	}
705
706	// Every job, waiting; each is expanded when the jobs it needs are done.
707	let mut statements = Vec::new();
708	for job in &new.workflow.jobs {
709	statements.push(
710	self.db
711	.prepare(
712	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, name, needs, status) VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting')",
713	)
714	.bind(&[
715	new_id("job", now_ms()).into(),
716	id.as_str().into(),
717	new.repo.id.as_str().into(),
718	new.repo.namespace.as_str().into(),
719	job.id.as_str().into(),
720	job.name.clone().filter(\|n\| !expr::has_expression(n)).unwrap_or(job.id.clone()).into(),
721	serde_json::to_string(&job.needs)?.into(),
722	])?,
723	);
724	}
725	self.db.batch(statements).await?;
726
727	// A pull request's run from outside waits to be approved; it joins
728	// its concurrency group once it is (protection.rs, approve_run).
729	if approval.is_some() {
730	return Ok(Some(id));
731	}
732	if let Some(run) = self.run_row(&id).await? {
733	self.enter_group(&run).await?;
734	}
735	Ok(Some(id))
736	}
737
738	/// A run about to start: one run at a time per concurrency group, as on
739	/// GitHub. A newer run replaces one that waits in the group, and with
740	/// `cancel-in-progress` the one that runs; otherwise it waits as
741	/// `pending` for the one that runs. Then it moves along.
742	pub(crate) async fn enter_group(&self, run: &RunRow) -> Result<()> {
743	if let Some(group) = &run.concurrency_group {
744	let cancel_in_progress = run.cancel_in_progress != 0;
745	let others = self
746	.db
747	.prepare(
748	"SELECT * FROM runs WHERE repo_id = ? AND concurrency_group = ? AND id != ? AND status NOT IN ('completed', 'action_required') ORDER BY id",
749	)
750	.bind(&[run.repo_id.as_str().into(), group.as_str().into(), run.id.as_str().into()])?
751	.all()
752	.await?
753	.results::<RunRow>()?;
754	for other in &others {
755	if cancel_in_progress \|\| other.status == "pending" {
756	// A newer run replaces a waiting one, as on GitHub.
757	self.cancel_run(other, "A newer run in the same concurrency group replaced it.", false).await?;
758	}
759	}
760	if !cancel_in_progress && others.iter().any(\|other\| other.status != "pending") {
761	self.db
762	.prepare("UPDATE runs SET status = 'pending' WHERE id = ?")
763	.bind(&[run.id.as_str().into()])?
764	.run()
765	.await?;
766	return Ok(());
767	}
768	}
769	self.advance(&run.id).await
770	}
771
772	/// A run that could not start, such as for a workflow file that does not read.
773	#[allow(clippy::too_many_arguments)]
774	pub async fn record_failed_run(
775	&self,
776	row: &WorkflowRow,
777	git_ref: &str,
778	sha: &str,
779	event_key: &str,
780	actor_id: Option<&str>,
781	actor: &str,
782	problem: &str,
783	) -> Result<()> {
784	let numbered = self
785	.db
786	.prepare("UPDATE workflows SET run_count = run_count + 1 WHERE id = ? RETURNING run_count AS n")
787	.bind(&[row.id.as_str().into()])?
788	.first::<Count>(None)
789	.await?
790	.map_or(1, \|count\| count.n);
791	let at = now();
792	self.db
793	.prepare(
794	"INSERT OR IGNORE INTO runs (id, workflow_id, repo_id, repo, path, name, title, number, event, git_ref, sha, status,
795	conclusion, error, actor, actor_id, source, info, event_key, created_at, finished_at)
796	VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'push', ?, ?, 'completed', 'failure', ?, ?, ?, ?, '{}', ?, ?, ?)",
797	)
798	.bind(&[
799	new_id("run", now_ms()).into(),
800	row.id.as_str().into(),
801	row.repo_id.as_str().into(),
802	row.repo.as_str().into(),
803	row.path.as_str().into(),
804	row.path.as_str().into(),
805	"Invalid workflow file".into(),
806	numbered.into(),
807	git_ref.into(),
808	sha.into(),
809	problem.into(),
810	actor.into(),
811	optional(actor_id),
812	row.source.as_str().into(),
813	event_key.into(),
814	at.as_str().into(),
815	at.as_str().into(),
816	])?
817	.run()
818	.await?;
819	Ok(())
820	}
821
822	/// Moves a run along: jobs whose needs are done are decided on, and
823	/// jobs that can start are started.
824	pub async fn advance(&self, run_id: &str) -> Result<()> {
825	// Each pass may finish jobs (skipped ones), which may free others.
826	for _ in 0..20 {
827	let Some(run) = self.run_row(run_id).await? else { return Ok(()) };
828	if matches!(run.status.as_str(), "completed" \| "pending" \| "action_required") {
829	return Ok(());
830	}
831	let workflow = match workflow::parse(&run.source) {
832	Ok(workflow) => workflow,
833	Err(problem) => {
834	self.finish_run(&run, Some(&problem)).await?;
835	return Ok(());
836	}
837	};
838	let jobs = self.job_rows(run_id).await?;
839	let mut changed = false;
840	// What to decide on: the workflow's jobs, and the jobs of the
841	// workflows they call, each with its needs as (name, key).
842	let mut units: Vec<Unit> = workflow
843	.jobs
844	.iter()
845	.map(\|job\| (job.id.clone(), job.clone(), job.needs.iter().map(\|n\| (n.clone(), n.clone())).collect()))
846	.collect();
847	let mut seen = std::collections::HashSet::new();
848	for row in &jobs {
849	if !seen.insert(row.key.clone()) {
850	continue;
851	}
852	if let Some((_, job, call)) = row.callee() {
853	let parent = call["parent"].as_str().unwrap_or_default().to_owned();
854	let needs = job.needs.iter().map(\|n\| (n.clone(), format!("{parent}/{n}"))).collect();
855	units.push((row.key.clone(), job, needs));
856	}
857	}
858	// Each key's needs, as keys, to look back through for failure().
859	let needs_of: std::collections::HashMap<&str, Vec<&str>> =
860	units.iter().map(\|(key, _, needs)\| (key.as_str(), needs.iter().map(\|(_, need)\| need.as_str()).collect())).collect();
861	let key_failed = \|key: &str\| key_result(&jobs.iter().filter(\|row\| row.key == key).collect::<Vec<_>>()) == "failure";
862	for (key, job, needs) in &units {
863	let rows: Vec<&JobRow> = jobs.iter().filter(\|row\| &row.key == key).collect();
864	if rows.is_empty() \|\| !rows.iter().all(\|row\| row.status == "waiting" && !row.holds_call()) {
865	continue;
866	}
867	let needed: Vec<(&String, Vec<&JobRow>)> =
868	needs.iter().map(\|(name, need)\| (name, jobs.iter().filter(\|row\| &row.key == need).collect())).collect();
869	if !needed.iter().all(\|(_, rows)\| rows.iter().all(\|row\| row.status == "completed")) {
870	continue;
871	}
872	let failed_before = ancestor_failed(&needs_of, key, key_failed);
873	self.decide(&run, job, rows[0], &needed, failed_before).await?;
874	changed = true;
875	}
876	// A job that called a workflow finishes with that workflow's jobs.
877	for row in jobs.iter().filter(\|row\| row.status == "calling") {
878	let prefix = row.call_prefix().unwrap_or(row.key.clone());
879	let children: Vec<&JobRow> = jobs
880	.iter()
881	.filter(\|child\| child.call().is_some_and(\|call\| call["role"] == "callee" && call["parent"].as_str() == Some(prefix.as_str())))
882	.collect();
883	if !children.is_empty() && children.iter().all(\|child\| child.status == "completed") {
884	self.finish_call(&run, row, &children).await?;
885	changed = true;
886	}
887	}
888	// Calls that wait start while their matrix's `max-parallel` and
889	// their concurrency group have room.
890	let mut calling: std::collections::HashMap<&str, u32> = std::collections::HashMap::new();
891	for row in jobs.iter().filter(\|row\| row.status == "calling") {
892	*calling.entry(row.key.as_str()).or_default() += 1;
893	}
894	for row in jobs.iter().filter(\|row\| row.holds_call()) {
895	let running = calling.get(row.key.as_str()).copied().unwrap_or(0);
896	if row.max_parallel.is_some_and(\|max\| running >= max) \|\| self.group_busy(row).await? {
897	continue;
898	}
899	if self.start_call(row).await? {
900	calling.insert(row.key.as_str(), running + 1);
901	changed = true;
902	}
903	}
904	if !changed {
905	break;
906	}
907	}
908	self.start_queued().await?;
909	self.finish_if_done(run_id).await
910	}
911
912	/// Decides on one job whose needs are done: skip it, fail it, or expand
913	/// it into its matrix and queue it. `failed_before`: whether any job
914	/// before it failed, however far back (`ancestor_failed`).
915	async fn decide(&self, run: &RunRow, job: &workflow::Job, row: &JobRow, needed: &[(&String, Vec<&JobRow>)], failed_before: bool) -> Result<()> {
916	let vars = self.variables_for(&run.repo_id, &run.repo, None, run.trusted != 0).await?;
917	let mut contexts = Self::base_contexts(run, &vars, &job.id);
918	// A called workflow's jobs read the inputs they were called with.
919	let call = row.call();
920	// Shown under the job whose call made it, by that job's name.
921	let caller = call.as_ref().filter(\|c\| c["role"] == "callee").and_then(\|c\| {
922	c["title"].as_str().map(str::to_owned).or_else(\|\| c["parent"].as_str().map(\|parent\| parent.replace('/', " / ")))
923	});
924	if let Some(call) = call.as_ref().filter(\|c\| c["role"] == "callee") {
925	contexts.insert("inputs".into(), call["inputs"].clone());
926	}
927	let mut needs = Map::new();
928	let mut results = Vec::new();
929	for (key, rows) in needed {
930	let result = key_result(rows);
931	let mut outputs = Map::new();
932	for row in rows {
933	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&row.outputs) {
934	outputs.extend(more);
935	}
936	}
937	results.push(result);
938	needs.insert((*key).clone(), json!({ "result": result, "outputs": outputs }));
939	}
940	// As on GitHub: a need that was skipped makes success() false but
941	// not failure(); failure() is a failure anywhere before the job.
942	let status = expr::job_status(results, failed_before, run.conclusion.as_deref() == Some("cancelled"));
943	contexts.insert("needs".into(), Value::Object(needs));
944	let scope = Scope {
945	contexts: &contexts,
946	status,
947	hash_files: None,
948	};
949	let condition = job.condition.as_deref().unwrap_or_default();
950	match expr::condition(condition, &scope) {
951	Ok(true) => {}
952	Ok(false) => return self.skip_job(row, None).await,
953	Err(problem) => return self.fail_job(row, &format!("Its `if` does not read: {problem}")).await,
954	}
955	// Its matrix, which may come from a needed job's outputs.
956	let combinations = match &job.matrix {
957	None => vec![Map::new()],
958	Some(matrix) => {
959	let value = match expr::interpolate_value(matrix, &scope) {
960	Ok(value) => value,
961	Err(problem) => return self.fail_job(row, &format!("Its matrix does not read: {problem}")).await,
962	};
963	match matrix::expand(&value) {
964	Ok(combinations) if !combinations.is_empty() => combinations,
965	Ok(_) => return self.fail_job(row, "Its matrix makes no jobs.").await,
966	Err(problem) => return self.fail_job(row, &problem).await,
967	}
968	}
969	};
970	// A job that calls a reusable workflow calls it once for each.
971	if let Some(uses) = &job.uses {
972	return self.call_workflow(run, job, row, uses, &contexts, &combinations, caller.as_deref()).await;
973	}
974	let total = combinations.len();
975	let raw = job.raw.as_object().cloned().unwrap_or_default();
976	// Whether pull requests from forks may use self-hosted runners here,
977	// asked once, and only for a run that is not trusted.
978	let mut forks_allowed: Option<bool> = None;
979	// Each concurrency group its jobs join, and whether it cancels.
980	let mut groups: Vec<(String, bool)> = Vec::new();
981	let mut statements = Vec::new();
982	for (index, combination) in combinations.iter().enumerate() {
983	let mut contexts = contexts.clone();
984	contexts.insert("matrix".into(), Value::Object(combination.clone()));
985	contexts.insert(
986	"strategy".into(),
987	json!({ "fail-fast": job.fail_fast, "job-index": index, "job-total": total, "max-parallel": job.max_parallel.unwrap_or(total as u32) }),
988	);
989	let scope = Scope {
990	contexts: &contexts,
991	status: Status::Success,
992	hash_files: None,
993	};
994	let name = shown_name(job, caller.as_deref(), combination, &scope);
995	let runs_on = expr::interpolate_value(&job.runs_on, &scope).unwrap_or(Value::Null);
996	let labels = match &runs_on {
997	Value::String(label) => vec![label.clone()],
998	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
999	Value::Object(spec) => spec.get("labels").map(\|l\| match l {
1000	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
1001	other => vec![expr::to_text(other)],
1002	}).unwrap_or_default(),
1003	_ => Vec::new(),
1004	};
1005	// `runs-on: self-hosted` (or a group): the workspace's own
1006	// machines, which may run any OS. Otherwise g1t's Linux sandboxes.
1007	let group = match &runs_on {
1008	Value::Object(spec) => spec.get("group").map(expr::to_text),
1009	_ => None,
1010	};
1011	let wanted = Wanted::of(&labels, group.as_deref());
1012	let mut reason = if wanted.self_hosted {
1013	None
1014	} else {
1015	labels
1016	.iter()
1017	.find(\|label\| {
1018	let lower = label.to_ascii_lowercase();
1019	lower.contains("windows") \|\| lower.contains("macos")
1020	})
1021	.map(\|label\| {
1022	let os = if label.to_ascii_lowercase().contains("windows") { "windows" } else { "macos" };
1023	format!("`runs-on: {label}`: g1t's own runners are Linux. A self-hosted runner can run it: `runs-on: [self-hosted, {os}]`.")
1024	})
1025	};
1026	// A pull request from a fork runs code anyone could write: never
1027	// on the workspace's machines unless it said they may.
1028	if wanted.self_hosted && run.trusted == 0 {
1029	let allowed = match forks_allowed {
1030	Some(allowed) => allowed,
1031	None => {
1032	let allowed = self.effective_runner_settings(&row.namespace, Some(&run.repo_id)).await?.fork_pull_requests;
1033	forks_allowed = Some(allowed);
1034	allowed
1035	}
1036	};
1037	if !allowed {
1038	reason = Some(
1039	"Pull requests from forks do not run on self-hosted runners here. An admin can allow it under Settings, Runners.".to_owned(),
1040	);
1041	}
1042	}
1043	let max_minutes = if wanted.self_hosted { SELF_HOSTED_MAX_TIMEOUT_MINUTES } else { MAX_TIMEOUT_MINUTES };
1044	let timeout = raw
1045	.get("timeout-minutes")
1046	.and_then(\|value\| expr::interpolate_value(value, &scope).ok())
1047	.and_then(\|value\| value.as_f64().or_else(\|\| expr::to_text(&value).parse().ok()))
1048	.map_or(g1t_actions::workflow::DEFAULT_TIMEOUT_MINUTES.min(max_minutes), \|minutes\| (minutes.ceil() as u32).clamp(1, max_minutes));
1049	let continue_on_error = raw
1050	.get("continue-on-error")
1051	.and_then(\|value\| expr::interpolate_value(value, &scope).ok())
1052	.is_some_and(\|value\| expr::truthy(&value));
1053	let (mut status, mut conclusion, mut finished) = match &reason {
1054	Some(_) => ("completed", Some("failure"), Some(now())),
1055	None => ("queued", None, None),
1056	};
1057	// A self-hosted job waits, saying for what, until a runner takes it.
1058	let (labels_json, queued_at) = if wanted.self_hosted && reason.is_none() {
1059	reason = Some(waiting_reason(&wanted));
1060	(Some(serde_json::to_string(&wanted.stored())?), Some(now()))
1061	} else {
1062	(None, None)
1063	};
1064	// The environment it names, an expression read now, and what
1065	// that environment's protection rules say: it starts, waits as
1066	// `pending` (protection.rs), or may not deploy there.
1067	let environment = environment_of(&job.raw, &contexts).map(\|env\| env.name);
1068	if status == "queued"
1069	&& let Some(name) = &environment
1070	{
1071	match self.gate(run, name).await? {
1072	Gate::Open => {}
1073	Gate::Held(why) => {
1074	status = "pending";
1075	reason = Some(why);
1076	}
1077	Gate::Refused(why) => {
1078	(status, conclusion, finished) = ("completed", Some("failure"), Some(now()));
1079	reason = Some(why);
1080	}
1081	}
1082	}
1083	// Its own concurrency group.
1084	let job_group = job
1085	.concurrency
1086	.as_ref()
1087	.and_then(\|c\| expr::interpolate(&c.group, &scope).ok())
1088	.map(\|group\| group.trim().to_owned())
1089	.filter(\|group\| !group.is_empty());
1090	let job_cancel = job
1091	.concurrency
1092	.as_ref()
1093	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
1094	.is_some_and(\|value\| expr::truthy(&value));
1095	if status != "completed"
1096	&& let Some(group) = &job_group
1097	&& !groups.iter().any(\|(known, _)\| known == group)
1098	{
1099	groups.push((group.clone(), job_cancel));
1100	}
1101	let values: Vec<worker::wasm_bindgen::JsValue> = vec![
1102	name.into(),
1103	serde_json::to_string(combination)?.into(),
1104	status.into(),
1105	optional(conclusion),
1106	optional(reason.as_deref()),
1107	timeout.into(),
1108	u32::from(continue_on_error).into(),
1109	job.max_parallel.map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1110	optional(finished.as_deref()),
1111	optional(labels_json.as_deref()),
1112	optional(queued_at.as_deref()),
1113	optional(environment.as_deref()),
1114	optional(job_group.as_deref()),
1115	u32::from(job_cancel).into(),
1116	];
1117	if index == 0 {
1118	let mut bound = values;
1119	bound.push(row.id.as_str().into());
1120	statements.push(
1121	self.db
1122	.prepare(
1123	"UPDATE jobs SET name = ?, matrix = ?, status = ?, conclusion = ?, reason = ?, timeout_minutes = ?,
1124	continue_on_error = ?, max_parallel = ?, finished_at = ?, labels = ?, queued_at = ?, environment = ?,
1125	concurrency_group = ?, cancel_in_progress = ? WHERE id = ?",
1126	)
1127	.bind(&bound)?,
1128	);
1129	} else {
1130	let mut bound: Vec<worker::wasm_bindgen::JsValue> = vec![
1131	new_id("job", now_ms()).into(),
1132	row.run_id.as_str().into(),
1133	row.repo_id.as_str().into(),
1134	row.namespace.as_str().into(),
1135	row.key.as_str().into(),
1136	(index as u32).into(),
1137	row.needs.as_str().into(),
1138	optional(row.call.as_deref()),
1139	];
1140	bound.extend(values);
1141	statements.push(
1142	self.db
1143	.prepare(
1144	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, call, name, matrix, status, conclusion, reason,
1145	timeout_minutes, continue_on_error, max_parallel, finished_at, labels, queued_at, environment,
1146	concurrency_group, cancel_in_progress)
1147	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1148	)
1149	.bind(&bound)?,
1150	);
1151	}
1152	}
1153	self.db.batch(statements).await?;
1154	self.replace_in_groups(run, &row.key, &groups).await
1155	}
1156
1157	/// A job queued in its own concurrency group: with `cancel-in-progress`
1158	/// it cancels the group's other jobs; otherwise it replaces any that
1159	/// are queued and not started, and waits for the one running
1160	/// (`start_queued` starts one of a group at a time). As on GitHub.
1161	async fn replace_in_groups(&self, run: &RunRow, key: &str, groups: &[(String, bool)]) -> Result<()> {
1162	if groups.is_empty() {
1163	return Ok(());
1164	}
1165	#[derive(Deserialize)]
1166	struct Id {
1167	id: String,
1168	}
1169	let mine: Vec<String> = self
1170	.db
1171	.prepare("SELECT id FROM jobs WHERE run_id = ? AND key = ?")
1172	.bind(&[run.id.as_str().into(), key.into()])?
1173	.all()
1174	.await?
1175	.results::<Id>()?
1176	.into_iter()
1177	.map(\|row\| row.id)
1178	.collect();
1179	let mut moved: Vec<String> = Vec::new();
1180	for (group, cancel) in groups {
1181	let others = self
1182	.db
1183	// A call that waits in the group, or calls, is in it too.
1184	.prepare("SELECT * FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('waiting', 'queued', 'pending', 'in_progress', 'calling')")
1185	.bind(&[run.repo_id.as_str().into(), group.as_str().into()])?
1186	.all()
1187	.await?
1188	.results::<JobRow>()?;
1189	for other in others.iter().filter(\|other\| !mine.contains(&other.id)) {
1190	if *cancel \|\| matches!(other.status.as_str(), "queued" \| "waiting") {
1191	self.stop_job(other, "A newer job in the same concurrency group replaced it.").await?;
1192	if !moved.contains(&other.run_id) {
1193	moved.push(other.run_id.clone());
1194	}
1195	}
1196	}
1197	}
1198	for other_run in moved.iter().filter(\|id\| **id != run.id) {
1199	Box::pin(self.advance(other_run)).await?;
1200	}
1201	Ok(())
1202	}
1203
1204	/// A job that calls a reusable workflow, in the repository or another
1205	/// (reach.rs): once for each combination of its matrix, as on GitHub,
1206	/// each with the inputs and secrets it passes read with that
1207	/// combination's `matrix`. Each call waits (`holds_call`) until its
1208	/// matrix's `max-parallel` and its concurrency group have room, when
1209	/// `start_call` puts the workflow's jobs in the run under it.
1210	#[allow(clippy::too_many_arguments)]
1211	async fn call_workflow(
1212	&self,
1213	run: &RunRow,
1214	job: &workflow::Job,
1215	row: &JobRow,
1216	uses: &str,
1217	contexts: &Map<String, Value>,
1218	combinations: &[Map<String, Value>],
1219	caller: Option<&str>,
1220	) -> Result<()> {
1221	let depth = row.call().and_then(\|c\| c["depth"].as_u64()).unwrap_or(0) + 1;
1222	if depth > MAX_CALL_DEPTH {
1223	return self.fail_job(row, &call_allowed(depth, &[], uses).unwrap_err()).await;
1224	}
1225	let (file, source, origin) = match self.called_workflow(run, row, uses).await? {
1226	Ok(found) => found,
1227	Err(why) => return self.fail_job(row, &why).await,
1228	};
1229	let jobs = self.job_rows(&run.id).await?;
1230	// Each call so far, by the workflow it called.
1231	let called_paths: Vec<String> = jobs
1232	.iter()
1233	.filter(\|other\| other.call_prefix().is_some())
1234	.filter_map(JobRow::call)
1235	.filter_map(\|call\| call["path"].as_str().map(str::to_owned))
1236	.collect();
1237	let called_already: Vec<&str> = called_paths.iter().map(String::as_str).collect();
1238	if let Err(why) = call_allowed(depth, &called_already, &file) {
1239	return self.fail_job(row, &why).await;
1240	}
1241	let called = match workflow::parse(&source) {
1242	Ok(called) => called,
1243	Err(problem) => return self.fail_job(row, &format!("`{file}` does not read: {problem}")).await,
1244	};
1245	let Some(trigger) = called.trigger("workflow_call") else {
1246	return self.fail_job(row, &format!("`{file}` cannot be called: it has no `on: workflow_call`.")).await;
1247	};
1248	// Secrets: none but the job's token unless `secrets:` passes them,
1249	// by name or with `inherit`; read when each job starts (job_spec).
1250	let outer = row.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c.get("secrets").cloned());
1251	let total = combinations.len();
1252	let mut groups: Vec<(String, bool)> = Vec::new();
1253	let mut statements = Vec::new();
1254	for (index, combination) in combinations.iter().enumerate() {
1255	let mut contexts = contexts.clone();
1256	contexts.insert("matrix".into(), Value::Object(combination.clone()));
1257	contexts.insert(
1258	"strategy".into(),
1259	json!({ "fail-fast": job.fail_fast, "job-index": index, "job-total": total, "max-parallel": job.max_parallel.unwrap_or(total as u32) }),
1260	);
1261	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
1262	let name = shown_name(job, caller, combination, &scope);
1263	// Inputs: what `with:` passes, as the called workflow types them.
1264	let given = match job.raw.get("with") {
1265	Some(with) => match expr::interpolate_value(with, &scope) {
1266	Ok(Value::Object(given)) => Ok(given),
1267	Ok(_) => Ok(Map::new()),
1268	Err(problem) => Err(format!("Its `with` does not read: {problem}")),
1269	},
1270	None => Ok(Map::new()),
1271	};
1272	let inputs = given.and_then(\|given\| crate::reach::call_inputs(&file, &trigger.inputs, &given));
1273	let secrets = crate::reach::secrets_plan(&job.raw, &contexts, outer.clone());
1274	let problem = match &inputs {
1275	Err(why) => Some(why.clone()),
1276	Ok(_) => crate::reach::missing_secrets(&called.raw, &secrets)
1277	.first()
1278	.map(\|name\| format!("`{file}` needs the secret `{name}`: pass it under `secrets:`, or use `secrets: inherit`."))
1279	.or_else(\|\| {
1280	crate::reach::undeclared_secrets(&called.raw, &secrets)
1281	.first()
1282	.map(\|name\| format!("Invalid secret: `{name}` is not defined in `{file}`."))
1283	}),
1284	};
1285	// Its own concurrency group, which it holds while its workflow runs.
1286	let group = job
1287	.concurrency
1288	.as_ref()
1289	.and_then(\|c\| expr::interpolate(&c.group, &scope).ok())
1290	.map(\|group\| group.trim().to_owned())
1291	.filter(\|group\| !group.is_empty());
1292	let cancel = job
1293	.concurrency
1294	.as_ref()
1295	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
1296	.is_some_and(\|value\| expr::truthy(&value));
1297	if problem.is_none()
1298	&& let Some(group) = &group
1299	&& !groups.iter().any(\|(known, _)\| known == group)
1300	{
1301	groups.push((group.clone(), cancel));
1302	}
1303	let call = json!({
1304	"role": "caller", "path": file, "source": source, "origin": origin, "depth": depth,
1305	"prefix": call_prefix(&row.key, job.matrix.is_some().then_some(index)), "title": name,
1306	"inputs": inputs.unwrap_or_default(), "secrets": secrets, "fail_fast": job.fail_fast,
1307	});
1308	let (status, conclusion, finished) = match &problem {
1309	Some(_) => ("completed", Some("failure"), Some(now())),
1310	None => ("waiting", None, None),
1311	};
1312	let values: Vec<worker::wasm_bindgen::JsValue> = vec![
1313	name.as_str().into(),
1314	serde_json::to_string(combination)?.into(),
1315	status.into(),
1316	optional(conclusion),
1317	optional(problem.as_deref()),
1318	job.max_parallel.map(\|max\| max.max(1)).map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1319	optional(finished.as_deref()),
1320	optional(group.as_deref()),
1321	u32::from(cancel).into(),
1322	serde_json::to_string(&call)?.into(),
1323	];
1324	if index == 0 {
1325	let mut bound = values;
1326	bound.push(row.id.as_str().into());
1327	statements.push(
1328	self.db
1329	.prepare(
1330	"UPDATE jobs SET name = ?, matrix = ?, status = ?, conclusion = ?, reason = ?, max_parallel = ?, finished_at = ?,
1331	concurrency_group = ?, cancel_in_progress = ?, call = ? WHERE id = ?",
1332	)
1333	.bind(&bound)?,
1334	);
1335	} else {
1336	let mut bound: Vec<worker::wasm_bindgen::JsValue> = vec![
1337	new_id("job", now_ms()).into(),
1338	row.run_id.as_str().into(),
1339	row.repo_id.as_str().into(),
1340	row.namespace.as_str().into(),
1341	row.key.as_str().into(),
1342	(index as u32).into(),
1343	row.needs.as_str().into(),
1344	];
1345	bound.extend(values);
1346	statements.push(
1347	self.db
1348	.prepare(
1349	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, name, matrix, status, conclusion, reason,
1350	max_parallel, finished_at, concurrency_group, cancel_in_progress, call)
1351	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1352	)
1353	.bind(&bound)?,
1354	);
1355	}
1356	}
1357	self.db.batch(statements).await?;
1358	self.replace_in_groups(run, &row.key, &groups).await
1359	}
1360
1361	/// Whether another job of `row`'s concurrency group runs, or calls a
1362	/// workflow, so that `row` waits for it.
1363	async fn group_busy(&self, row: &JobRow) -> Result<bool> {
1364	let Some(group) = &row.concurrency_group else { return Ok(false) };
1365	let busy = self
1366	.db
1367	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('in_progress', 'calling') AND id != ?")
1368	.bind(&[row.repo_id.as_str().into(), group.as_str().into(), row.id.as_str().into()])?
1369	.first::<Count>(None)
1370	.await?
1371	.map_or(0, \|count\| count.n);
1372	Ok(busy > 0)
1373	}
1374
1375	/// A call that waited (`holds_call`) is made: the called workflow's
1376	/// jobs join the run under it, named under its name and keyed under its
1377	/// prefix. False when it no longer waits.
1378	async fn start_call(&self, row: &JobRow) -> Result<bool> {
1379	let Some(call) = row.call() else { return Ok(false) };
1380	let Some(prefix) = row.call_prefix() else { return Ok(false) };
1381	let Some(called) = call["source"].as_str().and_then(\|source\| workflow::parse(source).ok()) else { return Ok(false) };
1382	let file = call["path"].as_str().unwrap_or_default();
1383	let started = self
1384	.db
1385	.prepare("UPDATE jobs SET status = 'calling', reason = ?, started_at = ? WHERE id = ? AND status = 'waiting' RETURNING id")
1386	.bind(&[format!("Calls `{file}`.").into(), now().into(), row.id.as_str().into()])?
1387	.first::<Value>(None)
1388	.await?;
1389	if started.is_none() {
1390	return Ok(false);
1391	}
1392	let mut statements = Vec::new();
1393	for called_job in &called.jobs {
1394	let needs: Vec<String> = called_job.needs.iter().map(\|n\| format!("{prefix}/{n}")).collect();
1395	let callee = json!({
1396	"role": "callee", "parent": prefix, "title": row.name, "job": called_job.id, "path": file,
1397	"source": call["source"], "inputs": call["inputs"], "depth": call["depth"], "origin": call["origin"], "secrets": call["secrets"],
1398	});
1399	statements.push(
1400	self.db
1401	.prepare("INSERT INTO jobs (id, run_id, repo_id, namespace, key, name, needs, status, call) VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting', ?)")
1402	.bind(&[
1403	new_id("job", now_ms()).into(),
1404	row.run_id.as_str().into(),
1405	row.repo_id.as_str().into(),
1406	row.namespace.as_str().into(),
1407	format!("{prefix}/{}", called_job.id).into(),
1408	format!("{} / {}", row.name, called_job.name.clone().unwrap_or(called_job.id.clone())).into(),
1409	serde_json::to_string(&needs)?.into(),
1410	serde_json::to_string(&callee)?.into(),
1411	])?,
1412	);
1413	}
1414	if !statements.is_empty() {
1415	self.db.batch(statements).await?;
1416	}
1417	Ok(true)
1418	}
1419
1420	/// The runs with a call waiting in `group`, but `run_id`, moved along
1421	/// now that a job of the group has finished.
1422	async fn wake_group(&self, repo_id: &str, group: &str, run_id: &str) -> Result<()> {
1423	#[derive(Deserialize)]
1424	struct RunId {
1425	run_id: String,
1426	}
1427	let waiting = self
1428	.db
1429	.prepare("SELECT DISTINCT run_id FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status = 'waiting' AND run_id != ?")
1430	.bind(&[repo_id.into(), group.into(), run_id.into()])?
1431	.all()
1432	.await?
1433	.results::<RunId>()?;
1434	for other in waiting {
1435	Box::pin(self.advance(&other.run_id)).await?;
1436	}
1437	Ok(())
1438	}
1439
1440	/// A job that called a workflow, finished with its jobs: their result,
1441	/// and the outputs the workflow declares. A failed call of a fail-fast
1442	/// matrix stops the matrix's other calls.
1443	async fn finish_call(&self, run: &RunRow, row: &JobRow, children: &[&JobRow]) -> Result<()> {
1444	let call = row.call().unwrap_or_default();
1445	let called = call["source"].as_str().and_then(\|s\| workflow::parse(s).ok());
1446	let mut jobs_context = Map::new();
1447	let mut by_key: std::collections::BTreeMap<String, Vec<&JobRow>> = std::collections::BTreeMap::new();
1448	for child in children {
1449	by_key.entry(child.key.clone()).or_default().push(child);
1450	}
1451	for (key, rows) in &by_key {
1452	let mut outputs = Map::new();
1453	for child in rows {
1454	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&child.outputs) {
1455	outputs.extend(more);
1456	}
1457	}
1458	let id = key.rsplit('/').next().unwrap_or(key);
1459	jobs_context.insert(id.to_owned(), json!({ "result": key_result(rows), "outputs": outputs }));
1460	}
1461	let inputs = children.first().and_then(\|c\| c.call()).map(\|c\| c["inputs"].clone()).unwrap_or(json!({}));
1462	let mut contexts = Map::new();
1463	contexts.insert("jobs".into(), Value::Object(jobs_context));
1464	contexts.insert("inputs".into(), inputs);
1465	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
1466	let mut outputs = Map::new();
1467	if let Some(Value::Object(declared)) = called.as_ref().map(\|w\| {
1468	let on = w.raw.get("on").or_else(\|\| w.raw.get("true")).cloned().unwrap_or(Value::Null);
1469	on.get("workflow_call").and_then(\|c\| c.get("outputs")).cloned().unwrap_or(Value::Null)
1470	}) {
1471	for (name, spec) in declared {
1472	if let Some(value) = spec.get("value") {
1473	let value = expr::interpolate_value(value, &scope).unwrap_or(Value::Null);
1474	outputs.insert(name, Value::String(expr::to_text(&value)));
1475	}
1476	}
1477	}
1478	let conclusion = key_result(children);
1479	let finished = self
1480	.db
1481	.prepare("UPDATE jobs SET status = 'completed', conclusion = ?, outputs = ?, finished_at = ? WHERE id = ? AND status = 'calling' RETURNING id")
1482	.bind(&[conclusion.into(), serde_json::to_string(&outputs)?.into(), now().into(), row.id.as_str().into()])?
1483	.first::<Value>(None)
1484	.await?;
1485	if finished.is_none() {
1486	return Ok(());
1487	}
1488	if conclusion == "failure" && row.matrix.as_deref().is_some_and(\|m\| m != "{}") && call["fail_fast"] != json!(false) {
1489	let siblings = self
1490	.db
1491	.prepare("SELECT * FROM jobs WHERE run_id = ? AND key = ? AND status != 'completed'")
1492	.bind(&[row.run_id.as_str().into(), row.key.as_str().into()])?
1493	.all()
1494	.await?
1495	.results::<JobRow>()?;
1496	for sibling in siblings {
1497	self.stop_job(&sibling, "Another job of its matrix failed, and the matrix is fail-fast.").await?;
1498	}
1499	}
1500	if let Some(group) = &row.concurrency_group {
1501	self.wake_group(&run.repo_id, group, &run.id).await?;
1502	}
1503	Ok(())
1504	}
1505
1506	/// A file's text at a commit, if it is there.
1507	pub(crate) async fn read_file(&self, path: &RepoPath, ws: &g1t_contracts::User, sha: &str, file: &str) -> Result<Option<String>> {
1508	let blob: Outcome<g1t_contracts::repos::BlobView> = g1t_kit::call(
1509	&self.repos,
1510	"blob",
1511	&g1t_contracts::repos::BlobArgs {
1512	path: path.clone(),
1513	viewer: Some(ws.clone()),
1514	git_ref: sha.to_owned(),
1515	file_path: file.to_owned(),
1516	},
1517	)
1518	.await?;
1519	Ok(match blob {
1520	Outcome::Ok(view) => view.text,
1521	Outcome::Fail(_) => None,
1522	})
1523	}
1524
1525	async fn skip_job(&self, row: &JobRow, reason: Option<&str>) -> Result<()> {
1526	self.db
1527	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'skipped', reason = ?, finished_at = ? WHERE id = ?")
1528	.bind(&[optional(reason), now().into(), row.id.as_str().into()])?
1529	.run()
1530	.await?;
1531	Ok(())
1532	}
1533
1534	async fn fail_job(&self, row: &JobRow, reason: &str) -> Result<()> {
1535	self.db
1536	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'failure', reason = ?, finished_at = ? WHERE id = ? AND status != 'completed'")
1537	.bind(&[reason.into(), now().into(), row.id.as_str().into()])?
1538	.run()
1539	.await?;
1540	Ok(())
1541	}
1542
1543	/// Starts queued jobs, oldest first, while their workspace has room.
1544	pub async fn start_queued(&self) -> Result<()> {
1545	let queued = self
1546	.db
1547	// Self-hosted jobs are taken by their runners (runners.rs).
1548	.prepare("SELECT * FROM jobs WHERE status = 'queued' AND labels IS NULL ORDER BY rowid LIMIT 50")
1549	.all()
1550	.await?
1551	.results::<JobRow>()?;
1552	let mut running: std::collections::HashMap<String, u32> = std::collections::HashMap::new();
1553	for job in queued {
1554	let in_workspace = match running.get(&job.namespace) {
1555	Some(n) => *n,
1556	None => {
1557	let n = self
1558	.db
1559	// Only g1t's own sandboxes count against the workspace's room.
1560	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE status = 'in_progress' AND namespace = ? AND runner_id IS NULL")
1561	.bind(&[job.namespace.as_str().into()])?
1562	.first::<Count>(None)
1563	.await?
1564	.map_or(0, \|count\| count.n);
1565	running.insert(job.namespace.clone(), n);
1566	n
1567	}
1568	};
1569	if in_workspace >= RUNNING_PER_WORKSPACE {
1570	continue;
1571	}
1572	if let Some(max) = job.max_parallel {
1573	let siblings = self
1574	.db
1575	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE run_id = ? AND key = ? AND status = 'in_progress'")
1576	.bind(&[job.run_id.as_str().into(), job.key.as_str().into()])?
1577	.first::<Count>(None)
1578	.await?
1579	.map_or(0, \|count\| count.n);
1580	if siblings >= max {
1581	continue;
1582	}
1583	}
1584	// One job of a concurrency group runs at a time.
1585	if let Some(group) = &job.concurrency_group {
1586	let running = self
1587	.db
1588	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('in_progress', 'calling') AND id != ?")
1589	.bind(&[job.repo_id.as_str().into(), group.as_str().into(), job.id.as_str().into()])?
1590	.first::<Count>(None)
1591	.await?
1592	.map_or(0, \|count\| count.n);
1593	if running > 0 {
1594	continue;
1595	}
1596	}
1597	let token = random_hex(24);
1598	let at = now();
1599	let claimed = self
1600	.db
1601	.prepare(
1602	"UPDATE jobs SET status = 'in_progress', token_hash = ?, started_at = ?, seen_at = ? WHERE id = ? AND status = 'queued' RETURNING id",
1603	)
1604	.bind(&[sha256_hex(&token).into(), at.as_str().into(), at.as_str().into(), job.id.as_str().into()])?
1605	.first::<Value>(None)
1606	.await?;
1607	if claimed.is_none() {
1608	continue;
1609	}
1610	running.insert(job.namespace.clone(), in_workspace + 1);
1611	self.db
1612	.prepare("UPDATE runs SET status = 'in_progress', started_at = COALESCE(started_at, ?) WHERE id = ? AND status = 'queued'")
1613	.bind(&[at.as_str().into(), job.run_id.as_str().into()])?
1614	.run()
1615	.await?;
1616	let run = self.run_row(&job.run_id).await?;
1617	let repo: RepoPath = run.as_ref().map(\|run\| repo_path(&run.repo)).unwrap_or(RepoPath {
1618	namespace: job.namespace.clone(),
1619	name: String::new(),
1620	});
1621	// Its environment and the machine it asked for, for the runner.
1622	let details = run.as_ref().map(\|run\| start_details(run, &job)).unwrap_or_default();
1623	let started: Outcome<Value> = g1t_kit::call(
1624	&self.runner,
1625	"start_actions_job",
1626	&StartJobArgs {
1627	job: job.id.clone(),
1628	token,
1629	repo,
1630	timeout_minutes: job.timeout_minutes,
1631	workflow: run.as_ref().map(\|run\| run.path.clone()),
1632	environment: details.environment,
1633	trusted: run.as_ref().is_some_and(\|run\| run.trusted != 0),
1634	instance: details.instance,
1635	},
1636	)
1637	.await
1638	.unwrap_or_else(\|error\| fail(FailureCode::Conflict, format!("The runner could not be reached: {error}")));
1639	if let Outcome::Fail(refused) = started {
1640	// Billing would not pay for a scheduled run's job: the
1641	// schedule waits rather than making runs only to refuse them.
1642	if refused.code == FailureCode::PaymentRequired
1643	&& let Some(run) = run.as_ref().filter(\|run\| run.event == "schedule")
1644	{
1645	self.db
1646	.prepare("UPDATE workflows SET schedule_refused_until = ? WHERE id = ?")
1647	.bind(&[rfc3339(now_ms() + crate::SCHEDULE_REFUSED_MS).into(), run.workflow_id.as_str().into()])?
1648	.run()
1649	.await?;
1650	}
1651	Box::pin(self.finish_job(&job.id, "failure", Some(&refused.message), None)).await?;
1652	} else {
1653	self.job_started(&job.id).await?;
1654	}
1655	}
1656	Ok(())
1657	}
1658
1659	/// Finishes a job and moves its run along.
1660	pub async fn finish_job(&self, job_id: &str, conclusion: &str, reason: Option<&str>, outputs: Option<&Map<String, Value>>) -> Result<()> {
1661	let finished = self
1662	.db
1663	.prepare(
1664	"UPDATE jobs SET status = 'completed', conclusion = ?, reason = COALESCE(?, reason), outputs = COALESCE(?, outputs),
1665	finished_at = ?, token_hash = NULL WHERE id = ? AND status != 'completed' RETURNING *",
1666	)
1667	.bind(&[
1668	conclusion.into(),
1669	optional(reason),
1670	outputs.map(\|o\| serde_json::to_string(o).unwrap_or_default()).as_deref().map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1671	now().into(),
1672	job_id.into(),
1673	])?
1674	.first::<JobRow>(None)
1675	.await?;
1676	let Some(job) = finished else { return Ok(()) };
1677	// Its G1T_TOKEN stops working with it.
1678	self.revoke_job_tokens(&job.id).await;
1679	// A job that deploys failed: so did its run's deployment, now.
1680	if conclusion == "failure" && job.continue_on_error == 0 && job.started_at.is_some()
1681	&& let Some(run) = self.run_row(&job.run_id).await?
1682	&& let Some(env) = deploys_to(&run, &job)
1683	{
1684	self.report_deployment(&run, &env, "failure", false).await;
1685	}
1686	// A self-hosted runner's job: the runner is free again, and its time
1687	// is recorded, at nothing.
1688	if job.runner_id.is_some() {
1689	self.released(&job).await?;
1690	}
1691	// Steps still marked as going are not going any more.
1692	let mut steps: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
1693	let mut touched = false;
1694	for step in steps.iter_mut() {
1695	if step["status"] != "completed" {
1696	let was_running = step["status"] == "in_progress";
1697	step["status"] = json!("completed");
1698	step["conclusion"] = json!(if was_running { conclusion } else { "skipped" });
1699	touched = true;
1700	}
1701	}
1702	if touched {
1703	self.db
1704	.prepare("UPDATE jobs SET steps = ? WHERE id = ?")
1705	.bind(&[serde_json::to_string(&steps)?.into(), job.id.as_str().into()])?
1706	.run()
1707	.await?;
1708	}
1709	// fail-fast: one failed combination stops the rest of its matrix.
1710	if conclusion == "failure" && job.continue_on_error == 0 && job.matrix.as_deref().is_some_and(\|m\| m != "{}") {
1711	let run = self.run_row(&job.run_id).await?;
1712	// As its workflow, or the workflow it is a job of, says.
1713	let fail_fast = run.as_ref().and_then(\|run\| job_spec(run, &job)).is_none_or(\|j\| j.fail_fast);
1714	if fail_fast {
1715	let siblings = self
1716	.db
1717	.prepare("SELECT * FROM jobs WHERE run_id = ? AND key = ? AND status != 'completed'")
1718	.bind(&[job.run_id.as_str().into(), job.key.as_str().into()])?
1719	.all()
1720	.await?
1721	.results::<JobRow>()?;
1722	for sibling in siblings {
1723	self.stop_job(&sibling, "Another job of its matrix failed, and the matrix is fail-fast.").await?;
1724	}
1725	}
1726	}
1727	// A call waiting in its concurrency group, in another run, may start.
1728	if let Some(group) = &job.concurrency_group {
1729	self.wake_group(&job.repo_id, group, &job.run_id).await?;
1730	}
1731	Box::pin(self.advance(&job.run_id)).await
1732	}
1733
1734	/// Cancels a job. One that is running is told to stop (the answer to
1735	/// its next report), and runs its `if: always()` and `cancelled()`
1736	/// steps and its post steps before it ends `cancelled`; one that has
1737	/// not started is cancelled at once.
1738	async fn stop_job(&self, job: &JobRow, reason: &str) -> Result<()> {
1739	// A job calling a workflow stops with that workflow's jobs.
1740	if job.status == "calling"
1741	&& let Some(prefix) = job.call_prefix()
1742	{
1743	let under = format!("{prefix}/");
1744	for child in self.job_rows(&job.run_id).await?.iter().filter(\|child\| child.key.starts_with(&under) && child.status != "completed") {
1745	Box::pin(self.stop_job(child, reason)).await?;
1746	}
1747	}
1748	if job.status == "in_progress" && job.started_at.is_some() {
1749	self.db
1750	.prepare("UPDATE jobs SET cancel_requested_at = COALESCE(cancel_requested_at, ?), reason = ? WHERE id = ? AND status = 'in_progress'")
1751	.bind(&[now().into(), reason.into(), job.id.as_str().into()])?
1752	.run()
1753	.await?;
1754	return Ok(());
1755	}
1756	self.hard_stop(job, reason).await
1757	}
1758
1759	/// Cancels a job outright, stopping its sandbox if it has one.
1760	async fn hard_stop(&self, job: &JobRow, reason: &str) -> Result<()> {
1761	// A self-hosted runner hears it was cancelled on its next poll.
1762	if job.status == "in_progress" && job.runner_id.is_none() {
1763	let _: Result<Value> = g1t_kit::call(&self.runner, "stop_actions_job", &json!({ "job": job.id })).await;
1764	}
1765	let stopped = self
1766	.db
1767	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'cancelled', reason = ?, finished_at = ?, token_hash = NULL WHERE id = ? AND status != 'completed' RETURNING *")
1768	.bind(&[reason.into(), now().into(), job.id.as_str().into()])?
1769	.first::<JobRow>(None)
1770	.await?;
1771	if stopped.as_ref().is_some_and(\|row\| row.started_at.is_some()) {
1772	self.revoke_job_tokens(&job.id).await;
1773	}
1774	if let Some(stopped) = stopped.filter(\|row\| row.runner_id.is_some()) {
1775	self.released(&stopped).await?;
1776	}
1777	Ok(())
1778	}
1779
1780	/// Ends a job's tokens at once. A failure is logged: the token expires
1781	/// on its own soon after the job's time limit.
1782	async fn revoke_job_tokens(&self, job_id: &str) {
1783	let revoked: Result<bool> =
1784	g1t_kit::call(&self.identity, "revoke_job_tokens", &RevokeJobTokensArgs { job_id: job_id.to_owned() }).await;
1785	if let Err(error) = revoked {
1786	worker::console_error!("actions: the tokens of job {job_id} were not revoked: {error}");
1787	}
1788	}
1789
1790	/// Finishes the run when every job has.
1791	async fn finish_if_done(&self, run_id: &str) -> Result<()> {
1792	let Some(run) = self.run_row(run_id).await? else { return Ok(()) };
1793	if matches!(run.status.as_str(), "completed" \| "pending" \| "action_required") {
1794	return Ok(());
1795	}
1796	let jobs = self.job_rows(run_id).await?;
1797	if !jobs.iter().all(\|job\| job.status == "completed") {
1798	return Ok(());
1799	}
1800	self.finish_run(&run, None).await
1801	}
1802
1803	async fn finish_run(&self, run: &RunRow, error: Option<&str>) -> Result<()> {
1804	let jobs = self.job_rows(&run.id).await?;
1805	let rows: Vec<&JobRow> = jobs.iter().collect();
1806	let conclusion = if error.is_some() {
1807	"failure"
1808	} else if run.conclusion.as_deref() == Some("cancelled") {
1809	"cancelled"
1810	} else if rows.is_empty() {
1811	"skipped"
1812	} else {
1813	key_result(&rows)
1814	};
1815	let done = self
1816	.db
1817	.prepare("UPDATE runs SET status = 'completed', conclusion = ?, error = COALESCE(?, error), finished_at = ? WHERE id = ? AND status != 'completed' RETURNING id")
1818	.bind(&[conclusion.into(), optional(error), now().into(), run.id.as_str().into()])?
1819	.first::<Value>(None)
1820	.await?;
1821	if done.is_none() {
1822	return Ok(());
1823	}
1824	self.report_status(run, conclusion).await?;
1825	self.settle_deployments(run, &jobs, conclusion == "cancelled").await;
1826	let published: Result<()> = g1t_kit::call(
1827	&self.events,
1828	"publish",
1829	&g1t_contracts::events::Publish {
1830	events: vec![g1t_contracts::events::NewEvent {
1831	kind: "workflow.completed",
1832	source: "actions",
1833	repo_id: Some(run.repo_id.clone()),
1834	actor: run.actor_id.clone(),
1835	data: g1t_contracts::events::WorkflowEvent {
1836	run_id: run.id.clone(),
1837	repo_id: run.repo_id.clone(),
1838	workflow: run.name.clone(),
1839	path: run.path.clone(),
1840	number: run.number,
1841	event: run.event.clone(),
1842	conclusion: conclusion.to_owned(),
1843	git_ref: run.git_ref.clone(),
1844	sha: run.sha.clone(),
1845	pull: run.pull,
1846	},
1847	}],
1848	},
1849	)
1850	.await;
1851	if let Err(error) = published {
1852	worker::console_error!("actions: could not publish workflow.completed: {error}");
1853	}
1854	// The next run waiting in its concurrency group.
1855	if let Some(group) = &run.concurrency_group {
1856	let next = self
1857	.db
1858	.prepare("SELECT * FROM runs WHERE repo_id = ? AND concurrency_group = ? AND status = 'pending' ORDER BY id LIMIT 1")
1859	.bind(&[run.repo_id.as_str().into(), group.as_str().into()])?
1860	.first::<RunRow>(None)
1861	.await?;
1862	if let Some(next) = next {
1863	self.db.prepare("UPDATE runs SET status = 'queued' WHERE id = ?").bind(&[next.id.as_str().into()])?.run().await?;
1864	Box::pin(self.advance(&next.id)).await?;
1865	}
1866	}
1867	Ok(())
1868	}
1869
1870	/// Tells the deployments service how a run's deployment to `env` stands:
1871	/// made when its first job naming the environment starts, failed when one
1872	/// of them fails, and settled (`last`) when the run finishes. Never fails
1873	/// the run: a deployment that could not be recorded is logged.
1874	async fn report_deployment(&self, run: &RunRow, env: &JobEnvironment, state: &str, last: bool) {
1875	let path = repo_path(&run.repo);
1876	let reported: Result<Value> = g1t_kit::call(
1877	&self.deployments,
1878	"actions_deployment",
1879	&json!({
1880	"repoId": run.repo_id,
1881	"repo": { "namespace": path.namespace, "name": path.name },
1882	"runId": run.id,
1883	"attempt": run.attempt,
1884	"runUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1885	"environment": env.name,
1886	"url": env.url,
1887	"ref": run.git_ref,
1888	"sha": run.sha,
1889	"state": state,
1890	"final": last,
1891	"creator": run.actor,
1892	"workflow": run.name,
1893	}),
1894	)
1895	.await;
1896	if let Err(error) = reported {
1897	worker::console_error!("actions: deployment to {} not recorded for run {}: {error}", env.name, run.id);
1898	}
1899	}
1900
1901	/// A job that deploys has started: its run's deployment to the
1902	/// environment is under way.
1903	pub(crate) async fn job_started(&self, job_id: &str) -> Result<()> {
1904	let Some(job) = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[job_id.into()])?.first::<JobRow>(None).await? else {
1905	return Ok(());
1906	};
1907	let Some(run) = self.run_row(&job.run_id).await? else { return Ok(()) };
1908	if let Some(env) = deploys_to(&run, &job) {
1909	self.report_deployment(&run, &env, "in_progress", false).await;
1910	}
1911	Ok(())
1912	}
1913
1914	/// The run is over: each environment its jobs deployed to takes the
1915	/// outcome of those jobs (`deployment_outcome`).
1916	async fn settle_deployments(&self, run: &RunRow, jobs: &[JobRow], cancelled: bool) {
1917	let mut seen: Vec<(JobEnvironment, Vec<Option<String>>)> = Vec::new();
1918	for job in jobs {
1919	let Some(env) = deploys_to(run, job) else { continue };
1920	let conclusion = if cancelled && job.conclusion.as_deref() != Some("skipped") && job.started_at.is_some() {
1921	Some("cancelled".to_owned())
1922	} else if job.started_at.is_none() {
1923	Some("skipped".to_owned())
1924	} else {
1925	job.conclusion.clone()
1926	};
1927	match seen.iter_mut().find(\|(known, _)\| known.name.eq_ignore_ascii_case(&env.name)) {
1928	Some((known, conclusions)) => {
1929	if known.url.is_none() {
1930	known.url = env.url.clone();
1931	}
1932	conclusions.push(conclusion);
1933	}
1934	None => seen.push((env, vec![conclusion])),
1935	}
1936	}
1937	for (env, conclusions) in seen {
1938	if let Some(state) = deployment_outcome(&conclusions) {
1939	self.report_deployment(run, &env, state, true).await;
1940	}
1941	}
1942	}
1943
1944	/// Tells the pull request (or commit) how the run went, as a status.
1945	async fn report_status(&self, run: &RunRow, conclusion: &str) -> Result<()> {
1946	let state = match conclusion {
1947	"success" \| "skipped" => "success",
1948	"cancelled" => "error",
1949	_ => "failure",
1950	};
1951	let _: Result<Value> = g1t_kit::call(
1952	&self.work,
1953	"set_commit_status",
1954	&json!({
1955	"repoId": run.repo_id,
1956	"sha": run.sha,
1957	"context": format!("{} / {}", run.name, run.event),
1958	"state": state,
1959	"description": format!("{} {}", run.name, match conclusion {
1960	"success" => "passed",
1961	"skipped" => "was skipped",
1962	"cancelled" => "was cancelled",
1963	_ => "failed",
1964	}),
1965	"targetUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1966	"source": "actions",
1967	}),
1968	)
1969	.await;
1970	Ok(())
1971	}
1972
1973	/// Tells the pull request a run has started on its head.
1974	pub async fn report_pending(&self, run: &RunRow) -> Result<()> {
1975	let _: Result<Value> = g1t_kit::call(
1976	&self.work,
1977	"set_commit_status",
1978	&json!({
1979	"repoId": run.repo_id,
1980	"sha": run.sha,
1981	"context": format!("{} / {}", run.name, run.event),
1982	"state": "pending",
1983	"description": if run.status == "action_required" {
1984	format!("{} is waiting for approval", run.name)
1985	} else {
1986	format!("{} is running", run.name)
1987	},
1988	"targetUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1989	"source": "actions",
1990	}),
1991	)
1992	.await;
1993	Ok(())
1994	}
1995
1996	/// Cancels a run: its waiting and queued jobs at once, and its running
1997	/// ones gracefully (`stop_job`), or outright when `force`.
1998	pub async fn cancel_run(&self, run: &RunRow, reason: &str, force: bool) -> Result<()> {
1999	self.db
2000	.prepare("UPDATE runs SET conclusion = 'cancelled' WHERE id = ? AND status != 'completed'")
2001	.bind(&[run.id.as_str().into()])?
2002	.run()
2003	.await?;
2004	for job in self.job_rows(&run.id).await?.iter().filter(\|job\| job.status != "completed") {
2005	if force {
2006	self.hard_stop(job, reason).await?;
2007	} else {
2008	self.stop_job(job, reason).await?;
2009	}
2010	}
2011	if run.status == "pending" \|\| run.status == "action_required" {
2012	self.db.prepare("UPDATE runs SET status = 'queued' WHERE id = ?").bind(&[run.id.as_str().into()])?.run().await?;
2013	}
2014	self.advance(&run.id).await
2015	}
2016
2017	/// Cancels every run of the repository that has not finished, for
2018	/// `repo.deleted` and `repo.archived`.
2019	pub async fn stop_runs(&self, repo_id: &str) -> Result<()> {
2020	let runs = self
2021	.db
2022	.prepare("SELECT * FROM runs WHERE repo_id = ? AND status != 'completed'")
2023	.bind(&[repo_id.into()])?
2024	.all()
2025	.await?
2026	.results::<RunRow>()?;
2027	for run in runs {
2028	self.cancel_run(&run, "The repository was archived or deleted.", true).await?;
2029	}
2030	Ok(())
2031	}
2032
2033	pub async fn cancel(&self, a: RunActionArgs) -> Result<Outcome<WorkflowRun>> {
2034	if let Outcome::Fail(refused) = self.may(&a.actor, &a.repo, Capability::Run).await? {
2035	return Ok(Outcome::Fail(refused));
2036	}
2037	let run = check!(self.run_in(&a.repo, &a.id).await?);
2038	if run.status == "completed" {
2039	return Ok(fail(FailureCode::Conflict, "The run has already finished."));
2040	}
2041	// Cancelling a run that is already cancelling stops its jobs
2042	// outright, without waiting for their cleanup steps.
2043	let force = a.force \|\| run.conclusion.as_deref() == Some("cancelled");
2044	let reason = if force {
2045	format!("{} stopped the run without waiting for its cleanup steps.", a.actor.username)
2046	} else {
2047	format!("{} cancelled the run.", a.actor.username)
2048	};
2049	self.cancel_run(&run, &reason, force).await?;
2050	self.run_summary(&run.id).await
2051	}
2052
2053	/// Runs again, as a new attempt: every job, with `failed_only` those
2054	/// that did not succeed, or with `job` that one; each with the jobs that
2055	/// need them. The attempt that ends is kept, its jobs and their logs and
2056	/// summaries with it (`job_attempts`, `run_attempts`).
2057	pub async fn rerun(&self, a: RunActionArgs) -> Result<Outcome<WorkflowRun>> {
2058	if let Outcome::Fail(refused) = self.may(&a.actor, &a.repo, Capability::Run).await? {
2059	return Ok(Outcome::Fail(refused));
2060	}
2061	// A job alone (`POST …/jobs/{job}/rerun`) names its run.
2062	let run_id = match (&a.job, a.id.is_empty()) {
2063	(Some(job), true) => {
2064	#[derive(Deserialize)]
2065	struct Of {
2066	run_id: String,
2067	}
2068	let of = self.db.prepare("SELECT run_id FROM jobs WHERE id = ?").bind(&[job.as_str().into()])?.first::<Of>(None).await?;
2069	match of {
2070	Some(of) => of.run_id,
2071	None => return Ok(fail(FailureCode::NotFound, "No such job.")),
2072	}
2073	}
2074	_ => a.id.clone(),
2075	};
2076	let run = check!(self.run_in(&a.repo, &run_id).await?);
2077	if run.status != "completed" {
2078	return Ok(fail(FailureCode::Conflict, "The run is still going: cancel it first."));
2079	}
2080	// Nothing starts again on an archived repository.
2081	match self.visible_repo(&a.repo, &Some(a.actor.clone())).await? {
2082	Some(repo) if repo.archived() => {
2083	return Ok(fail(FailureCode::Forbidden, g1t_contracts::repos::archived_message(&repo.namespace, &repo.name)));
2084	}
2085	Some(_) => {}
2086	None => return Ok(fail(FailureCode::NotFound, "There is no such repository.")),
2087	}
2088	if run.error.is_some() {
2089	return Ok(fail(FailureCode::Conflict, "This run never started: fix the workflow file and push again."));
2090	}
2091	let jobs = self.job_rows(&run.id).await?;
2092	let workflow = workflow::parse(&run.source).ok();
2093	let target = match &a.job {
2094	Some(id) => match jobs.iter().find(\|job\| &job.id == id) {
2095	Some(job) => Some(top_key(&job.key).to_owned()),
2096	None => return Ok(fail(FailureCode::NotFound, "That job is not in the run's latest attempt.")),
2097	},
2098	None => None,
2099	};
2100	let which = match (&target, a.failed_only) {
2101	(Some(key), _) => Rerun::Job(key),
2102	(None, true) => Rerun::Failed,
2103	(None, false) => Rerun::All,
2104	};
2105	let order = workflow.as_ref().map(\|w\| w.job_order()).unwrap_or_default();
2106	let again = rerun_keys(
2107	&order,
2108	\|key\| jobs.iter().find(\|j\| j.key == key).map(JobRow::needs).unwrap_or_default(),
2109	\|key\| jobs.iter().filter(\|j\| j.key == key).all(\|j\| j.conclusion.as_deref() == Some("success")),
2110	which,
2111	);
2112	if again.is_empty() {
2113	return Ok(fail(FailureCode::Conflict, "Every job succeeded: there is nothing to run again."));
2114	}
2115	// The jobs that run again, a called workflow's with the job calling it.
2116	let rerun_ids: Vec<&str> = jobs.iter().filter(\|job\| again.iter().any(\|key\| key == top_key(&job.key))).map(\|job\| job.id.as_str()).collect();
2117	let ids = serde_json::to_string(&rerun_ids)?;
2118	let ended = run.attempt.to_string();
2119	let ended = ended.as_str();
2120	let mut statements = vec![
2121	// The attempt that ends, as it ended.
2122	self.db
2123	.prepare(
2124	"INSERT OR REPLACE INTO run_attempts (run_id, attempt, repo_id, conclusion, actor, debug, started_at, finished_at)
2125	SELECT id, attempt, repo_id, conclusion, COALESCE(triggering_actor, actor), debug, started_at, finished_at FROM runs WHERE id = ?",
2126	)
2127	.bind(&[run.id.as_str().into()])?,
2128	// Earlier attempts that showed a job's logs from where it ran
2129	// then now find them where they move to.
2130	self.db
2131	.prepare("UPDATE job_attempts SET log_id = log_id \|\| '.' \|\| ?1 WHERE run_id = ?2 AND log_id IN (SELECT value FROM json_each(?3))")
2132	.bind(&[ended.into(), run.id.as_str().into(), ids.as_str().into()])?,
2133	// Each of its jobs: one that runs again keeps its logs under
2134	// `{id}.{attempt}`; one left alone is still the live job's.
2135	self.db
2136	.prepare(
2137	"INSERT OR REPLACE INTO job_attempts (id, run_id, repo_id, attempt, job_id, log_id, key, ordinal, name, needs, status, conclusion,
2138	steps, annotations, reason, environment, labels, runner_name, started_at, finished_at)
2139	SELECT id \|\| '.' \|\| ?1, run_id, repo_id, ?1, id,
2140	CASE WHEN id IN (SELECT value FROM json_each(?3)) THEN id \|\| '.' \|\| ?1 ELSE id END,
2141	key, ordinal, name, needs, status, conclusion, steps, annotations, reason, environment, labels, runner_name, started_at, finished_at
2142	FROM jobs WHERE run_id = ?2 ORDER BY rowid",
2143	)
2144	.bind(&[ended.into(), run.id.as_str().into(), ids.as_str().into()])?,
2145	self.db
2146	.prepare("UPDATE logs SET job_id = job_id \|\| '.' \|\| ?1 WHERE job_id IN (SELECT value FROM json_each(?2))")
2147	.bind(&[ended.into(), ids.as_str().into()])?,
2148	self.db
2149	.prepare("UPDATE job_summaries SET job_id = job_id \|\| '.' \|\| ?1 WHERE job_id IN (SELECT value FROM json_each(?2))")
2150	.bind(&[ended.into(), ids.as_str().into()])?,
2151	];
2152	for key in &again {
2153	statements.push(self.db.prepare("DELETE FROM jobs WHERE run_id = ? AND key = ? AND ordinal > 0").bind(&[run.id.as_str().into(), key.as_str().into()])?);
2154	// The jobs of a workflow it called (each matrix combination's
2155	// under `key[i]/`) are made again when it calls it again.
2156	statements.push(
2157	self.db
2158	.prepare("DELETE FROM jobs WHERE run_id = ? AND (key LIKE ? OR key LIKE ?)")
2159	.bind(&[run.id.as_str().into(), format!("{key}/%").into(), format!("{key}[%").into()])?,
2160	);
2161	statements.push(
2162	self.db
2163	.prepare(
2164	"UPDATE jobs SET status = 'waiting', conclusion = NULL, steps = '[]', annotations = '[]', outputs = '{}', reason = NULL,
2165	matrix = NULL, call = NULL, token_hash = NULL, seen_at = NULL, started_at = NULL, finished_at = NULL,
2166	labels = NULL, queued_at = NULL, runner_id = NULL, runner_name = NULL, environment = NULL,
2167	concurrency_group = NULL, cancel_in_progress = 0, cancel_requested_at = NULL WHERE run_id = ? AND key = ?",
2168	)
2169	.bind(&[run.id.as_str().into(), key.as_str().into()])?,
2170	);
2171	}
2172	statements.push(
2173	self.db
2174	.prepare(
2175	"UPDATE runs SET status = 'queued', conclusion = NULL, attempt = attempt + 1, started_at = NULL, finished_at = NULL,
2176	triggering_actor = ?, debug = ? WHERE id = ?",
2177	)
2178	.bind(&[a.actor.username.as_str().into(), u32::from(a.debug).into(), run.id.as_str().into()])?,
2179	);
2180	self.db.batch(statements).await?;
2181	if let Some(run) = self.run_row(&run.id).await? {
2182	self.report_pending(&run).await?;
2183	}
2184	self.advance(&run.id).await?;
2185	self.run_summary(&run.id).await
2186	}
2187
2188	pub async fn run_in(&self, repo: &RepoPath, id: &str) -> Result<Outcome<RunRow>> {
2189	let row = self
2190	.db
2191	.prepare("SELECT * FROM runs WHERE id = ? AND lower(repo) = lower(?)")
2192	.bind(&[id.into(), format!("{}/{}", repo.namespace, repo.name).into()])?
2193	.first::<RunRow>(None)
2194	.await?;
2195	Ok(row.map_or_else(\|\| fail(FailureCode::NotFound, "No such run."), Outcome::Ok))
2196	}
2197
2198	// --- The sandbox's side -----------------------------------------------------
2199
2200	pub(crate) async fn job_for_token(&self, a: &JobCallArgs) -> Result<Outcome<JobRow>> {
2201	let job = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[a.job.as_str().into()])?.first::<JobRow>(None).await?;
2202	Ok(match job {
2203	Some(job) if job.status == "in_progress" && job.token_hash.as_deref().is_some_and(\|hash\| same(hash, &sha256_hex(&a.token))) => {
2204	Outcome::Ok(job)
2205	}
2206	_ => fail(FailureCode::Unauthenticated, "That job is not running, or the token is not its."),
2207	})
2208	}
2209
2210	/// `job_auth`: which run and repository a running job's token is for,
2211	/// so the API can keep its artifacts and cache.
2212	pub async fn job_auth(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2213	let job = check!(self.job_for_token(&a).await?);
2214	Ok(Outcome::Ok(json!({ "run": job.run_id, "repoId": job.repo_id })))
2215	}
2216
2217	/// `job_spec`: everything the sandbox needs to run the job.
2218	pub async fn job_spec(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2219	let job = check!(self.job_for_token(&a).await?);
2220	let Some(run) = self.run_row(&job.run_id).await? else {
2221	return Ok(fail(FailureCode::NotFound, "No such run."));
2222	};
2223	let Ok(caller) = workflow::parse(&run.source) else {
2224	return Ok(fail(FailureCode::Invalid, "The workflow no longer reads."));
2225	};
2226	// A called workflow's job runs as that workflow defines it.
2227	let callee = job.callee();
2228	let (workflow, spec, call_inputs) = match callee {
2229	Some((called, spec, call)) => (called, spec, Some(call["inputs"].clone())),
2230	None => match caller.jobs.iter().find(\|j\| j.id == job.key) {
2231	Some(spec) => (caller.clone(), spec.clone(), None),
2232	None => return Ok(fail(FailureCode::NotFound, "The job is not in the workflow.")),
2233	},
2234	};
2235	let spec = &spec;
2236	let repo = repo_path(&run.repo);
2237	let trusted = run.trusted != 0;
2238	// The job's `environment:`, by name, as read when its needs were done
2239	// (an expression included), once the environment's protection rules
2240	// let it start: entries with a value for it give that value instead
2241	// of their default, as GitHub's environment secrets do.
2242	let environment: Option<String> = job.environment.clone();
2243	// What its token may do: its `permissions:` (a called workflow's
2244	// jobs no more than the job that calls it), else the repository's
2245	// default; read-only for a pull request from outside.
2246	// The repository's default, its workspace's taken in, and whether
2247	// its jobs may open and approve pull requests.
2248	let (default, pull_requests) = self.token_policy(&run.repo_id).await?;
2249	let mut permissions = spec.permissions(&workflow, default);
2250	if let Some(parent) = &job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c["parent"].as_str().map(str::to_owned)) {
2251	let top = top_key(parent);
2252	if let Some(caller_job) = caller.jobs.iter().find(\|j\| j.id == top) {
2253	permissions = permissions.capped_by(&caller_job.permissions(&caller, default));
2254	}
2255	}
2256	if !trusted {
2257	permissions = permissions.read_only();
2258	}
2259	// G1T_TOKEN, and GITHUB_TOKEN as its alias: a token of the
2260	// workspace's that reaches this repository only, with the scopes
2261	// its permissions give, until the job ends.
2262	let token = match self.workspace_actor(&repo.namespace).await? {
2263	Some(workspace) => {
2264	let created: CreatedAccessToken = g1t_kit::call(
2265	&self.identity,
2266	"create_job_token",
2267	&CreateJobTokenArgs {
2268	workspace,
2269	repo: repo.clone(),
2270	run_id: run.id.clone(),
2271	job_id: job.id.clone(),
2272	name: format!("G1T_TOKEN for {} run {}", run.repo, run.number),
2273	ttl_seconds: u64::from(job.timeout_minutes) * 60 + 600,
2274	scopes: permissions.scopes().into_iter().map(str::to_owned).collect(),
2275	pull_requests: pull_requests && trusted,
2276	},
2277	)
2278	.await?;
2279	created.token
2280	}
2281	None => String::new(),
2282	};
2283	// A run that is not trusted (a pull request from outside the
2284	// workspace) gets no secrets and an empty token.
2285	let passed = job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c.get("secrets").cloned());
2286	let mut secrets = match (trusted, passed) {
2287	(false, _) => Map::new(),
2288	(true, None) => self.secrets_for(&run.repo_id, &run.repo, environment.as_deref(), true).await?,
2289	// A called workflow's job: what its callers passed it (reach.rs),
2290	// and its own environment's secrets over them.
2291	(true, Some(plan)) => {
2292	let base = self.secrets_for(&run.repo_id, &run.repo, None, true).await?;
2293	let vars = self.variables_for(&run.repo_id, &run.repo, None, true).await?;
2294	let github = run.info().context(&job.key, "", run.action.as_deref());
2295	let mut passed = crate::reach::resolve_secrets(&plan, &base, &github, &vars);
2296	if let Some(name) = environment.as_deref() {
2297	let own = self.secrets_for(&run.repo_id, &run.repo, Some(name), true).await?;
2298	for (key, value) in own {
2299	if base.get(&key) != Some(&value) {
2300	passed.insert(key, value);
2301	}
2302	}
2303	}
2304	passed
2305	}
2306	};
2307	secrets.insert("G1T_TOKEN".into(), Value::String(token.clone()));
2308	secrets.insert("GITHUB_TOKEN".into(), Value::String(token.clone()));
2309	// Each secret as it is, a line at a time, base64 and JSON-escaped.
2310	let mut masks: Vec<String> = g1t_actions::mask::all_variants(secrets.values().filter_map(\|v\| v.as_str()));
2311	let vars = self.variables_for(&run.repo_id, &run.repo, environment.as_deref(), trusted).await?;
2312	// The toolkit's runtime token (runtime.rs), for as long as the job
2313	// may run; the API puts it and the toolkit's addresses in the
2314	// job's variables.
2315	let runtime_token = crate::runtime::runtime_token(
2316	&job.id,
2317	&job.run_id,
2318	job.token_hash.as_deref().unwrap_or_default(),
2319	now_ms() / 1000,
2320	u64::from(job.timeout_minutes) * 60 + 600,
2321	);
2322	masks.push(runtime_token.clone());
2323	let retention_days = self.retention_setting(&run.repo_id).await?;
2324
2325	let jobs = self.job_rows(&run.id).await?;
2326	let mut needs = Map::new();
2327	// In a called workflow, its jobs' keys sit under the job that called it.
2328	let parent = job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c["parent"].as_str().map(str::to_owned));
2329	for need in &spec.needs {
2330	let key = match &parent {
2331	Some(parent) => format!("{parent}/{need}"),
2332	None => need.clone(),
2333	};
2334	let rows: Vec<&JobRow> = jobs.iter().filter(\|row\| row.key == key).collect();
2335	let mut outputs = Map::new();
2336	for row in &rows {
2337	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&row.outputs) {
2338	outputs.extend(more);
2339	}
2340	}
2341	needs.insert(need.clone(), json!({ "result": key_result(&rows), "outputs": outputs }));
2342	}
2343	let siblings = jobs.iter().filter(\|row\| row.key == job.key).count();
2344	let matrix: Value = job.matrix.as_deref().and_then(\|m\| serde_json::from_str(m).ok()).unwrap_or(json!({}));
2345	let info = run.info();
2346	let mut github = info.context(&job.key, &token, run.action.as_deref());
2347	github["token"] = json!(token);
2348	github["retention_days"] = json!(retention_days);
2349	// On a self-hosted runner, `runner` and `RUNNER_*` describe that
2350	// machine rather than g1t's sandbox.
2351	let mut variables = info.variables(&job.key);
2352	variables.insert("GITHUB_RETENTION_DAYS".into(), json!(retention_days.to_string()));
2353	let mut runner = match &job.runner_id {
2354	Some(id) => self.runner_context_for(id, &mut variables).await?,
2355	None => runner_context(),
2356	};
2357	// A re-run with debug logging: what GitHub sets for one.
2358	if run.debug != 0 {
2359	debug_logging(&mut variables, &mut runner);
2360	}
2361
2362	// Where to check out: a pull request's fork, or the repository.
2363	let clone_url = match run.pull {
2364	Some(number) if run.event.starts_with("pull_request") && run.event != "pull_request_target" => {
2365	let located: Outcome<g1t_contracts::work::PullDetail> = g1t_kit::call(
2366	&self.work,
2367	"get_pull",
2368	&g1t_contracts::work::ViewArgs {
2369	repo: repo.clone(),
2370	number,
2371	viewer: self.workspace_actor(&repo.namespace).await?,
2372	after_seq: 0,
2373	},
2374	)
2375	.await?;
2376	match located {
2377	Outcome::Ok(detail) => match detail.pull.fork {
2378	Some(fork) => format!("{SITE}/{}/{}.git", fork.namespace, fork.name),
2379	None => format!("{SITE}/{}.git", run.repo),
2380	},
2381	Outcome::Fail(_) => format!("{SITE}/{}.git", run.repo),
2382	}
2383	}
2384	_ => format!("{SITE}/{}.git", run.repo),
2385	};
2386
2387	Ok(Outcome::Ok(json!({
2388	"job": job.id,
2389	"run": run.id,
2390	"key": job.key,
2391	"name": job.name,
2392	"spec": spec.raw,
2393	"workflow": {
2394	"env": workflow.env,
2395	"defaults": workflow.raw.get("defaults").cloned().unwrap_or(Value::Null),
2396	},
2397	"github": github,
2398	"variables": variables,
2399	"event": info.event,
2400	"contexts": {
2401	"vars": vars,
2402	"secrets": secrets,
2403	"inputs": call_inputs.unwrap_or_else(\|\| Value::Object(run.inputs())),
2404	"matrix": matrix,
2405	"needs": needs,
2406	"strategy": {
2407	"fail-fast": spec.fail_fast,
2408	"job-index": job.ordinal,
2409	"job-total": siblings,
2410	"max-parallel": spec.max_parallel.unwrap_or(siblings as u32),
2411	},
2412	"runner": runner,
2413	},
2414	"checkout": {
2415	"repository": run.repo,
2416	"url": clone_url,
2417	"sha": run.sha,
2418	"ref": run.git_ref,
2419	"token": token,
2420	},
2421	"timeoutMinutes": job.timeout_minutes,
2422	"masks": masks,
2423	// As the job's log lists them at its start.
2424	"permissions": permissions.listed().into_iter().map(\|(name, access)\| (name.to_owned(), json!(access.as_str()))).collect::<Map<String, Value>>(),
2425	// Whether the job may ask for an OIDC token decides whether it
2426	// is told where to.
2427	"runtime": {
2428	"token": runtime_token,
2429	"idToken": self.oidc_allowed(&run, &job),
2430	},
2431	})))
2432	}
2433
2434	/// `job_report`: the sandbox telling how the job is going.
2435	pub async fn job_report(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2436	let job = check!(self.job_for_token(&a).await?);
2437	let report = &a.report;
2438	let at = now();
2439	match report["kind"].as_str().unwrap_or_default() {
2440	"steps" => {
2441	// The list can grow as the job goes (post steps), so steps
2442	// already reported keep where they stand.
2443	let known: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
2444	let steps: Vec<Value> = report["steps"]
2445	.as_array()
2446	.map(\|names\| {
2447	names
2448	.iter()
2449	.enumerate()
2450	.map(\|(i, name)\| match known.get(i) {
2451	Some(step) if step["status"] != "queued" => step.clone(),
2452	_ => json!({ "number": i + 1, "name": expr::to_text(name), "status": "queued", "conclusion": null, "startedAt": null, "finishedAt": null }),
2453	})
2454	.collect()
2455	})
2456	.unwrap_or_default();
2457	self.db
2458	.prepare("UPDATE jobs SET steps = ?, seen_at = ? WHERE id = ?")
2459	.bind(&[serde_json::to_string(&steps)?.into(), at.as_str().into(), job.id.as_str().into()])?
2460	.run()
2461	.await?;
2462	}
2463	"step" => {
2464	let number = report["number"].as_u64().unwrap_or(0) as usize;
2465	let mut steps: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
2466	if let Some(step) = number.checked_sub(1).and_then(\|i\| steps.get_mut(i)) {
2467	let status = report["status"].as_str().unwrap_or("in_progress");
2468	step["status"] = json!(status);
2469	if status == "in_progress" {
2470	step["startedAt"] = json!(at);
2471	}
2472	if status == "completed" {
2473	step["finishedAt"] = json!(at);
2474	step["conclusion"] = report["conclusion"].clone();
2475	}
2476	if let Some(name) = report["name"].as_str() {
2477	step["name"] = json!(name);
2478	}
2479	}
2480	self.db
2481	.prepare("UPDATE jobs SET steps = ?, seen_at = ? WHERE id = ?")
2482	.bind(&[serde_json::to_string(&steps)?.into(), at.as_str().into(), job.id.as_str().into()])?
2483	.run()
2484	.await?;
2485	}
2486	"log" => {
2487	let mut text = report["text"].as_str().unwrap_or_default().to_owned();
2488	if text.len() > MAX_CHUNK_BYTES {
2489	let mut cut = MAX_CHUNK_BYTES;
2490	while !text.is_char_boundary(cut) {
2491	cut -= 1;
2492	}
2493	text.truncate(cut);
2494	}
2495	#[derive(Deserialize)]
2496	struct Size {
2497	n: Option<f64>,
2498	}
2499	let size = self
2500	.db
2501	.prepare("SELECT SUM(LENGTH(text)) AS n FROM logs WHERE job_id = ?")
2502	.bind(&[job.id.as_str().into()])?
2503	.first::<Size>(None)
2504	.await?;
2505	let used = size.and_then(\|s\| s.n).unwrap_or(0.0) as usize;
2506	if used < MAX_LOG_BYTES {
2507	if used + text.len() >= MAX_LOG_BYTES {
2508	text.push_str("\n… The log reached its limit of 4 MB; the rest is not kept.\n");
2509	}
2510	// The next `seq` is taken in the insert itself, so two
2511	// batches at once (a report sent again while the first
2512	// was still being written) never reach for the same one
2513	// and lose a chunk to the primary key.
2514	self.db
2515	.prepare(LOG_INSERT)
2516	.bind(&[
2517	job.id.as_str().into(),
2518	(report["step"].as_u64().unwrap_or(0) as u32).into(),
2519	text.into(),
2520	report["chunk"].as_u64().map_or(worker::wasm_bindgen::JsValue::NULL, \|n\| (n as f64).into()),
2521	])?
2522	.run()
2523	.await?;
2524	}
2525	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2526	}
2527	"summary" => {
2528	// `$GITHUB_STEP_SUMMARY`, masked by the runner: added to the
2529	// step's summary, up to 1 MiB a step and 20 steps a job.
2530	let step = report["step"].as_u64().unwrap_or(0) as u32;
2531	let markdown = report["markdown"].as_str().unwrap_or_default();
2532	#[derive(Deserialize)]
2533	struct Held {
2534	steps: u32,
2535	mine: Option<f64>,
2536	}
2537	let held = self
2538	.db
2539	.prepare("SELECT COUNT(*) AS steps, MAX(CASE WHEN step = ? THEN LENGTH(markdown) END) AS mine FROM job_summaries WHERE job_id = ?")
2540	.bind(&[step.into(), job.id.as_str().into()])?
2541	.first::<Held>(None)
2542	.await?;
2543	let (steps, mine) = held.map_or((0, None), \|held\| (held.steps, held.mine.map(\|n\| n as usize)));
2544	if summary_fits(steps, mine, markdown.len()) {
2545	self.db
2546	.prepare(
2547	"INSERT INTO job_summaries (job_id, step, markdown) VALUES (?, ?, ?)
2548	ON CONFLICT (job_id, step) DO UPDATE SET markdown = job_summaries.markdown \|\| excluded.markdown",
2549	)
2550	.bind(&[job.id.as_str().into(), step.into(), markdown.into()])?
2551	.run()
2552	.await?;
2553	}
2554	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2555	}
2556	"annotation" => {
2557	let mut annotations: Vec<Value> = serde_json::from_str(&job.annotations).unwrap_or_default();
2558	if annotations.len() < MAX_ANNOTATIONS {
2559	annotations.push(json!({
2560	"level": report["level"].as_str().unwrap_or("notice"),
2561	"message": report["message"].as_str().unwrap_or_default().chars().take(4000).collect::<String>(),
2562	"title": report["title"],
2563	"file": report["file"],
2564	"line": report["line"],
2565	}));
2566	self.db
2567	.prepare("UPDATE jobs SET annotations = ?, seen_at = ? WHERE id = ?")
2568	.bind(&[serde_json::to_string(&annotations)?.into(), at.as_str().into(), job.id.as_str().into()])?
2569	.run()
2570	.await?;
2571	}
2572	}
2573	// Nothing to tell; the answer says whether to stop.
2574	"ping" => {
2575	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2576	}
2577	"done" => {
2578	// A job told to stop ends cancelled, however its cleanup went.
2579	let conclusion = if job.cancel_requested_at.is_some() {
2580	"cancelled"
2581	} else {
2582	report["conclusion"]
2583	.as_str()
2584	.filter(\|c\| matches!(*c, "success" \| "failure" \| "cancelled"))
2585	.unwrap_or("failure")
2586	};
2587	let outputs = report["outputs"].as_object().cloned();
2588	Box::pin(self.finish_job(&job.id, conclusion, report["reason"].as_str(), outputs.as_ref())).await?;
2589	}
2590	other => return Ok(fail(FailureCode::Invalid, format!("There is no report called `{other}`."))),
2591	}
2592	// `cancelled`: the run was cancelled, and the runner should stop
2593	// the step it is on and run only its cleanup steps.
2594	Ok(Outcome::Ok(json!({ "ok": true, "cancelled": job.cancel_requested_at.is_some() })))
2595	}
2596
2597	// --- Every minute ---------------------------------------------------------------
2598
2599	pub async fn on_minute(&self, now_ms: u64) -> Result<()> {
2600	let minute = now_ms / 60_000 * 60_000;
2601	// Staff (or billing's usage watcher) paused scheduled runs across
2602	// g1t: this minute's schedules are skipped, not queued for later.
2603	// Kept 30 seconds in the isolate (g1t_kit::pause).
2604	if g1t_kit::pause::paused(&self.billing, g1t_contracts::billing::PauseLevel::Schedules).await {
2605	worker::console_log!("actions: schedules are paused across g1t; skipped this minute's");
2606	} else if let Err(error) = self.run_schedules(minute).await {
2607	worker::console_error!("actions: schedules failed: {error}");
2608	}
2609	// Jobs whose sandbox went quiet or ran past their time.
2610	let running = self.db.prepare("SELECT * FROM jobs WHERE status = 'in_progress'").all().await?.results::<JobRow>()?;
2611	for job in running {
2612	// Times in g1t's format compare as text.
2613	let before = \|ms: u64\| rfc3339(now_ms.saturating_sub(ms));
2614	let silent = job.seen_at.as_deref().is_some_and(\|seen\| seen < before(SILENT_MS).as_str());
2615	let limit = (u64::from(job.timeout_minutes) * 60 + 120) * 1000;
2616	let over = job.started_at.as_deref().is_some_and(\|started\| started < before(limit).as_str());
2617	let cancel_overdue = job.cancel_requested_at.as_deref().is_some_and(\|at\| at < before(CANCEL_GRACE_MS).as_str());
2618	if cancel_overdue {
2619	// Cancelled, and still going after its grace period.
2620	self.hard_stop(&job, "It was cancelled, and did not finish its cleanup steps within 5 minutes.").await?;
2621	self.advance(&job.run_id).await?;
2622	} else if over {
2623	let reason = format!("It ran longer than its time limit of {} minutes.", job.timeout_minutes);
2624	// A self-hosted runner is told to stop on its next poll.
2625	if job.runner_id.is_none() {
2626	let _: Result<Value> = g1t_kit::call(&self.runner, "stop_actions_job", &json!({ "job": job.id })).await;
2627	}
2628	self.finish_job(&job.id, "failure", Some(&reason), None).await?;
2629	} else if silent {
2630	let reason = match &job.runner_name {
2631	Some(name) => format!("The self-hosted runner {name} stopped answering."),
2632	None => "The runner stopped answering.".to_owned(),
2633	};
2634	self.finish_job(&job.id, "failure", Some(&reason), None).await?;
2635	}
2636	}
2637	// Calls of reusable workflows waiting in a concurrency group, in
2638	// case the job they waited for was cancelled in another run.
2639	#[derive(Deserialize)]
2640	struct RunId {
2641	run_id: String,
2642	}
2643	let held = self
2644	.db
2645	.prepare("SELECT DISTINCT run_id FROM jobs WHERE status = 'waiting' AND concurrency_group IS NOT NULL LIMIT 20")
2646	.all()
2647	.await?
2648	.results::<RunId>()?;
2649	for run in held {
2650	self.advance(&run.run_id).await?;
2651	}
2652	if let Err(error) = self.sweep_runners(now_ms).await {
2653	worker::console_error!("actions: the runners' sweep failed: {error}");
2654	}
2655	// Jobs held at an environment whose wait timer has run out.
2656	if let Err(error) = self.release_gates().await {
2657	worker::console_error!("actions: environments' gates failed: {error}");
2658	}
2659	// Once an hour: the cache's expired entries, and its storage.
2660	if (now_ms / 60_000) % 60 == 7
2661	&& let Err(error) = self.sweep_cache(now_ms).await
2662	{
2663	worker::console_error!("actions: the cache's sweep failed: {error}");
2664	}
2665	// And the archives of GitHub's actions no job uses any more.
2666	if (now_ms / 60_000) % 60 == 47
2667	&& let Err(error) = self.sweep_action_archives(now_ms).await
2668	{
2669	worker::console_error!("actions: the action archives' sweep failed: {error}");
2670	}
2671	// And artifacts past their time, and the toolkit's abandoned parts.
2672	if (now_ms / 60_000) % 60 == 37 {
2673	if let Err(error) = self.sweep_artifacts(now_ms).await {
2674	worker::console_error!("actions: the artifacts' sweep failed: {error}");
2675	}
2676	if let Err(error) = self.sweep_blob_parts(now_ms).await {
2677	worker::console_error!("actions: the blob parts' sweep failed: {error}");
2678	}
2679	}
2680	self.start_queued().await
2681	}
2682	}
2683
2684
2685	#[cfg(test)]
2686	mod log_chunks {
2687	use rusqlite::{Connection, params};
2688
2689	use super::LOG_INSERT;
2690
2691	/// The logs table as the migrations leave it.
2692	fn database() -> Connection {
2693	let db = Connection::open_in_memory().unwrap();
2694	for name in ["0001_init.sql", "0016_log_chunks.sql"] {
2695	let path = format!("{}/migrations/{name}", env!("CARGO_MANIFEST_DIR"));
2696	db.execute_batch(&std::fs::read_to_string(path).unwrap()).unwrap();
2697	}
2698	db
2699	}
2700
2701	fn log(db: &Connection, job: &str) -> Vec<(i64, i64, String)> {
2702	let mut rows = db.prepare("SELECT seq, step, text FROM logs WHERE job_id = ? ORDER BY seq").unwrap();
2703	rows.query_map([job], \|row\| Ok((row.get(0)?, row.get(1)?, row.get(2)?))).unwrap().map(Result::unwrap).collect()
2704	}
2705
2706	#[test]
2707	fn a_batch_sent_again_is_kept_once() {
2708	let db = database();
2709	let insert = \|job: &str, step: i64, text: &str, chunk: Option<i64>\| db.execute(LOG_INSERT, params![job, step, text, chunk]).unwrap();
2710	assert_eq!(insert("job_1", 0, "one\n", Some(1)), 1);
2711	// The runner did not hear the answer and sent the batch again.
2712	assert_eq!(insert("job_1", 0, "one\n", Some(1)), 0);
2713	assert_eq!(insert("job_1", 1, "two\n", Some(2)), 1);
2714	// Another job numbers its batches from 1 too.
2715	assert_eq!(insert("job_2", 0, "other\n", Some(1)), 1);
2716	// A runner that numbers nothing has every batch kept, as before.
2717	assert_eq!(insert("job_3", 0, "a\n", None), 1);
2718	assert_eq!(insert("job_3", 0, "a\n", None), 1);
2719	assert_eq!(log(&db, "job_1"), [(1, 0, "one\n".into()), (2, 1, "two\n".into())]);
2720	assert_eq!(log(&db, "job_2"), [(1, 0, "other\n".into())]);
2721	assert_eq!(log(&db, "job_3").len(), 2);
2722	// A job run again moves its log aside (`{id}.{attempt}`), and the
2723	// new attempt's runner numbers from 1 again.
2724	db.execute("UPDATE logs SET job_id = job_id \|\| '.1' WHERE job_id = 'job_1'", []).unwrap();
2725	assert_eq!(insert("job_1", 0, "again\n", Some(1)), 1);
2726	assert_eq!(log(&db, "job_1"), [(1, 0, "again\n".into())]);
2727	assert_eq!(log(&db, "job_1.1").len(), 2);
2728	}
2729	}
2730
2731	#[cfg(test)]
2732	mod stopping {
2733	use super::stops_runs;
2734	use g1t_contracts::events::Event;
2735	use serde_json::{Value, json};
2736
2737	fn event(kind: &str, data: Value) -> Event {
2738	Event {
2739	id: "evt_1".into(),
2740	kind: kind.into(),
2741	source: "repos".into(),
2742	time: "2026-10-05T00:00:00Z".into(),
2743	repo_id: Some("rep_1".into()),
2744	actor: None,
2745	data,
2746	}
2747	}
2748
2749	#[test]
2750	fn deleting_or_archiving_stops_runs() {
2751	assert_eq!(stops_runs(&event("repo.deleted", json!({ "repoId": "rep_1" }))).as_deref(), Some("rep_1"));
2752	assert_eq!(stops_runs(&event("repo.archived", json!({ "archived": true }))).as_deref(), Some("rep_1"));
2753	assert_eq!(stops_runs(&event("repo.unarchived", json!({ "archived": false }))), None);
2754	assert_eq!(stops_runs(&event("repo.restored", json!({}))), None);
2755	assert_eq!(stops_runs(&event("git.push", json!({}))), None);
2756	}
2757	}
2758
2759	#[cfg(test)]
2760	mod status_of_needs {
2761	use std::collections::HashMap;
2762
2763	use super::ancestor_failed;
2764
2765	/// check -> plan -> (migrate) -> core -> edge, as deploy.yml has them,
2766	/// and a job that needs only the last.
2767	fn graph() -> HashMap<&'static str, Vec<&'static str>> {
2768	HashMap::from([
2769	("check", vec![]),
2770	("plan", vec!["check"]),
2771	("migrate", vec!["plan"]),
2772	("core", vec!["plan", "migrate"]),
2773	("edge", vec!["plan", "migrate", "core"]),
2774	("notify", vec!["edge"]),
2775	])
2776	}
2777
2778	#[test]
2779	fn a_failure_is_seen_however_far_back() {
2780	let needs = graph();
2781	let failed = \|which: &'static str\| move \|key: &str\| key == which;
2782	// check failed; plan, and everything after, was skipped for it.
2783	assert!(ancestor_failed(&needs, "notify", failed("check")));
2784	assert!(ancestor_failed(&needs, "core", failed("check")));
2785	assert!(ancestor_failed(&needs, "edge", failed("core")));
2786	// Nothing before a job failed: a skipped migrate is not a failure.
2787	assert!(!ancestor_failed(&needs, "edge", \|_\| false));
2788	assert!(!ancestor_failed(&needs, "core", failed("edge")));
2789	assert!(!ancestor_failed(&needs, "check", failed("check")));
2790	}
2791
2792	#[test]
2793	fn cycles_and_unknown_keys_end() {
2794	let needs = HashMap::from([("a", vec!["b"]), ("b", vec!["a"])]);
2795	assert!(!ancestor_failed(&needs, "a", \|_\| false));
2796	assert!(!ancestor_failed(&needs, "missing", \|_\| true));
2797	}
2798	}
2799
2800	#[cfg(test)]
2801	mod deployments {
2802	use serde_json::{Map, Value, json};
2803
2804	use super::{JobEnvironment, deployment_outcome, environment_of};
2805
2806	#[test]
2807	fn a_jobs_environment_is_read_for_deployments() {
2808	let contexts: Map<String, Value> = serde_json::from_value(json!({
2809	"github": { "ref_name": "main", "repository": "acme/web" },
2810	"inputs": { "target": "staging" },
2811	"matrix": {},
2812	}))
2813	.unwrap();
2814	let read = \|raw: Value\| environment_of(&raw, &contexts);
2815	assert_eq!(read(json!({})), None);
2816	assert_eq!(
2817	read(json!({ "environment": "production" })),
2818	Some(JobEnvironment { name: "production".into(), url: None, deploys: true })
2819	);
2820	assert_eq!(
2821	read(json!({ "environment": { "name": "production", "url": "https://g1t.sh" } })),
2822	Some(JobEnvironment { name: "production".into(), url: Some("https://g1t.sh".into()), deploys: true })
2823	);
2824	// Expressions are filled in from the run.
2825	assert_eq!(
2826	read(json!({ "environment": { "name": "${{ inputs.target }}", "url": "https://${{ github.ref_name }}.example.com" } })),
2827	Some(JobEnvironment { name: "staging".into(), url: Some("https://main.example.com".into()), deploys: true })
2828	);
2829	// Secrets only: no deployment.
2830	assert!(!read(json!({ "environment": { "name": "production", "deployment": false } })).unwrap().deploys);
2831	// The same, said in the job's g1t settings.
2832	assert!(!read(json!({ "environment": "production", "g1t": { "deployment": false } })).unwrap().deploys);
2833	assert!(!read(json!({ "environment": { "name": "production" }, "g1t": { "deployment": false } })).unwrap().deploys);
2834	// Only http(s) addresses.
2835	assert_eq!(read(json!({ "environment": { "name": "production", "url": "javascript:alert(1)" } })).unwrap().url, None);
2836	}
2837
2838	#[test]
2839	fn a_runs_outcome_for_an_environment() {
2840	let of = \|list: &[&str]\| deployment_outcome(&list.iter().map(\|c\| Some((*c).to_owned())).collect::<Vec<_>>());
2841	assert_eq!(of(&["success", "skipped"]), Some("success"));
2842	assert_eq!(of(&["success", "failure"]), Some("failure"));
2843	assert_eq!(of(&["success", "cancelled"]), Some("error"));
2844	assert_eq!(of(&["skipped"]), None);
2845	assert_eq!(deployment_outcome(&[None]), None);
2846	}
2847	}
2848
2849	#[cfg(test)]
2850	mod reruns {
2851	use serde_json::{Map, Value, json};
2852
2853	use super::{MAX_SUMMARIES, MAX_SUMMARY_BYTES, Rerun, Scope, Status, call_allowed, call_prefix, debug_logging, rerun_keys, shown_name, summary_fits, top_key};
2854
2855	/// build ← test ← deploy, and lint on its own.
2856	fn keys(which: Rerun, failed: &[&str]) -> Vec<String> {
2857	let order = ["build", "lint", "test", "deploy"];
2858	let needs = \|key: &str\| -> Vec<String> {
2859	match key {
2860	"test" => vec!["build".into()],
2861	"deploy" => vec!["test".into()],
2862	_ => Vec::new(),
2863	}
2864	};
2865	rerun_keys(&order, needs, \|key\| !failed.contains(&key), which)
2866	}
2867
2868	#[test]
2869	fn a_rerun_takes_the_jobs_it_names_and_those_that_need_them() {
2870	assert_eq!(keys(Rerun::All, &[]), ["build", "lint", "test", "deploy"]);
2871	assert_eq!(keys(Rerun::Failed, &["test"]), ["test", "deploy"]);
2872	assert_eq!(keys(Rerun::Failed, &["lint"]), ["lint"]);
2873	assert!(keys(Rerun::Failed, &[]).is_empty());
2874	// One job, whatever it came to, and what depends on it.
2875	assert_eq!(keys(Rerun::Job("build"), &[]), ["build", "test", "deploy"]);
2876	assert_eq!(keys(Rerun::Job("lint"), &["test"]), ["lint"]);
2877	assert_eq!(keys(Rerun::Job("deploy"), &[]), ["deploy"]);
2878	assert!(keys(Rerun::Job("missing"), &[]).is_empty());
2879	}
2880
2881	#[test]
2882	fn a_called_workflows_jobs_run_again_with_the_job_that_calls_it() {
2883	assert_eq!(top_key("build/test"), "build");
2884	assert_eq!(top_key("build/inner/test"), "build");
2885	assert_eq!(top_key("lint"), "lint");
2886	// A matrix's calls each have their own jobs, under `key[i]/`.
2887	assert_eq!(top_key("build[1]/test"), "build");
2888	assert_eq!(top_key("build/inner[0]/test"), "build");
2889	assert_eq!(call_prefix("build", Some(1)), "build[1]");
2890	assert_eq!(call_prefix("build/inner", None), "build/inner");
2891	}
2892
2893	#[test]
2894	fn a_run_connects_ten_levels_of_workflows_and_fifty_reusable_ones() {
2895	assert!(call_allowed(1, &[], "a.yml").is_ok());
2896	assert!(call_allowed(9, &[], "a.yml").is_ok());
2897	let deep = call_allowed(10, &[], "a.yml").unwrap_err();
2898	assert!(deep.contains("at most ten") && deep.contains("nine levels"), "{deep}");
2899	let names: Vec<String> = (0..50).map(\|n\| format!("w{n}.yml")).collect();
2900	let fifty: Vec<&str> = names.iter().map(String::as_str).collect();
2901	// A workflow called again, as by each combination of a matrix, is one.
2902	assert!(call_allowed(1, &fifty, "w3.yml").is_ok());
2903	assert!(call_allowed(1, &fifty, "another.yml").unwrap_err().contains("at most 50 different"));
2904	}
2905
2906	#[test]
2907	fn a_matrixs_calls_are_named_by_the_caller_and_its_values() {
2908	let workflow = super::workflow::parse(
2909	"on: push\njobs:\n build:\n name: Build\n strategy:\n matrix: { os: [linux], node: [20] }\n uses: ./.g1t/workflows/b.yml\n plain:\n uses: ./.g1t/workflows/b.yml\n test:\n name: Test ${{ matrix.os }}\n strategy:\n matrix: { os: [linux] }\n runs-on: ubuntu-latest\n steps: [{ run: 'true' }]",
2910	)
2911	.unwrap();
2912	let job = \|id: &str\| workflow.jobs.iter().find(\|job\| job.id == id).unwrap();
2913	let combination: Map<String, Value> = serde_json::from_value(json!({ "os": "linux", "node": 20 })).unwrap();
2914	let mut contexts = Map::new();
2915	contexts.insert("matrix".into(), Value::Object(combination.clone()));
2916	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
2917	assert_eq!(shown_name(job("build"), None, &combination, &scope), "Build (linux, 20)");
2918	assert_eq!(shown_name(job("plain"), None, &Map::new(), &scope), "plain");
2919	// A called workflow's job, under the call that made it.
2920	assert_eq!(shown_name(job("plain"), Some("Build (linux, 20)"), &Map::new(), &scope), "Build (linux, 20) / plain");
2921	assert_eq!(shown_name(job("test"), Some("Build"), &combination, &scope), "Build / Test linux");
2922	}
2923
2924	#[test]
2925	fn a_job_keeps_twenty_steps_summaries_of_a_mebibyte_each() {
2926	assert!(summary_fits(0, None, 10));
2927	assert!(!summary_fits(0, None, 0), "nothing to keep");
2928	assert!(!summary_fits(MAX_SUMMARIES, None, 10), "a twenty-first step's summary is dropped");
2929	assert!(summary_fits(MAX_SUMMARIES, Some(10), 10), "a step that has one may add to it");
2930	assert!(summary_fits(1, Some(MAX_SUMMARY_BYTES - 10), 10));
2931	assert!(!summary_fits(1, Some(MAX_SUMMARY_BYTES - 10), 11));
2932	assert!(!summary_fits(0, None, MAX_SUMMARY_BYTES + 1));
2933	}
2934
2935	#[test]
2936	fn a_debug_rerun_sets_what_github_sets() {
2937	let mut variables = Map::new();
2938	let mut runner = json!({ "name": "g1t", "debug": "" });
2939	debug_logging(&mut variables, &mut runner);
2940	assert_eq!(variables["RUNNER_DEBUG"], "1");
2941	assert_eq!(variables["ACTIONS_STEP_DEBUG"], "true");
2942	assert_eq!(variables["ACTIONS_RUNNER_DEBUG"], "true");
2943	assert_eq!(runner["debug"], Value::String("1".into()));
2944	assert_eq!(runner["name"], "g1t");
2945	}
2946	}