flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh

git agents cloudflare rust forge mcp

2,911 linesCode BlameRaw

1	//! A run's life: made, its jobs waiting on the jobs they need, each job
2	//! skipped or expanded into its matrix and queued, started in a sandbox
3	//! when its workspace has room, reporting its steps and logs as it goes,
4	//! and finished; the run finishes with its last job.
5
6	use g1t_actions::events::{RunInfo, runner_context};
7	use g1t_actions::expr::{self, Scope, Status};
8	use g1t_actions::matrix;
9	use g1t_actions::workflow::{self, Workflow};
10	use g1t_contracts::access::Capability;
11	use g1t_contracts::actions::{JobCallArgs, RunActionArgs, RunApproval, StartJobArgs, WorkflowRun};
12	use g1t_contracts::identity::{CreateJobTokenArgs, CreatedAccessToken, RevokeJobTokensArgs};
13	use g1t_contracts::repos::{Repo, RepoPath};
14	use g1t_contracts::time::rfc3339;
15	use g1t_contracts::{FailureCode, Outcome, new_id};
16	use g1t_kit::now_ms;
17	use g1t_secrets::{random_hex, same, sha256_hex};
18	use serde::Deserialize;
19	use serde_json::{Map, Value, json};
20	use worker::Result;
21
22	use crate::protection::Gate;
23	use crate::sync::WorkflowRow;
24	use crate::{Actions, Count, MAX_TIMEOUT_MINUTES, RUNNING_PER_WORKSPACE, SELF_HOSTED_MAX_TIMEOUT_MINUTES, SILENT_MS, SITE, check, fail, optional, repo_path};
25	use g1t_contracts::runners::{Wanted, waiting_reason};
26
27	/// The most log one job keeps, in bytes; past it, the log says so and stops.
28	const MAX_LOG_BYTES: usize = 4 * 1024 * 1024;
29	/// The most a single log report may add.
30	const MAX_CHUNK_BYTES: usize = 256 * 1024;
31	const MAX_ANNOTATIONS: usize = 50;
32	/// A batch of a job's log, under the next `seq` of that job's, worked out
33	/// in the same statement (SQLite runs one write at a time). `?4` is the
34	/// runner's number for the batch (null from runners that send none): a
35	/// batch sent again, after an answer the runner did not hear, has the
36	/// number of one already kept, and is not kept twice (migration 0016).
37	const LOG_INSERT: &str = "INSERT INTO logs (job_id, seq, step, text, chunk)
38	SELECT ?1, COALESCE(MAX(seq), 0) + 1, ?2, ?3, ?4 FROM logs WHERE job_id = ?1
39	ON CONFLICT (job_id, chunk) DO NOTHING";
40	/// The most one step's job summary keeps, in bytes, and how many steps of a
41	/// job may have one, as on GitHub.
42	pub(crate) const MAX_SUMMARY_BYTES: usize = 1024 * 1024;
43	pub(crate) const MAX_SUMMARIES: u32 = 20;
44
45	/// Whether a step's summary of `adding` bytes is kept: a job keeps up to
46	/// 20 steps' summaries (`steps` it has, `mine` this step's bytes so far)
47	/// of up to 1 MiB each.
48	pub(crate) fn summary_fits(steps: u32, mine: Option<usize>, adding: usize) -> bool {
49	if adding == 0 {
50	return false;
51	}
52	match mine {
53	Some(used) => used + adding <= MAX_SUMMARY_BYTES,
54	None => steps < MAX_SUMMARIES && adding <= MAX_SUMMARY_BYTES,
55	}
56	}
57
58	/// The repository whose unfinished runs `event` stops, by id: one deleted,
59	/// or archived (not unarchived).
60	pub fn stops_runs(event: &g1t_contracts::events::Event) -> Option<String> {
61	let stops = match event.kind.as_str() {
62	"repo.deleted" => true,
63	"repo.archived" => event.data["archived"].as_bool() != Some(false),
64	_ => false,
65	};
66	if !stops {
67	return None;
68	}
69	event.repo_id.clone().or_else(\|\| event.data["repoId"].as_str().map(str::to_owned))
70	}
71
72	pub struct NewRun {
73	pub repo: Repo,
74	pub path: String,
75	pub source: String,
76	pub workflow: Workflow,
77	pub info: RunInfo,
78	pub action: Option<String>,
79	pub pull: Option<u32>,
80	pub title: String,
81	pub inputs: Map<String, Value>,
82	pub event_key: String,
83	pub actor_id: Option<String>,
84	pub actor: Option<String>,
85	pub trusted: bool,
86	/// Why the run waits for someone with the Write role to approve it
87	/// before anything starts: a pull request from outside (protection.rs).
88	pub approval: Option<String>,
89	}
90
91	#[derive(Clone, Deserialize)]
92	pub struct RunRow {
93	pub id: String,
94	pub workflow_id: String,
95	pub repo_id: String,
96	pub repo: String,
97	pub path: String,
98	pub name: String,
99	pub title: String,
100	pub number: u64,
101	pub attempt: u64,
102	pub event: String,
103	pub action: Option<String>,
104	pub git_ref: String,
105	pub sha: String,
106	pub pull: Option<u32>,
107	pub status: String,
108	pub conclusion: Option<String>,
109	pub error: Option<String>,
110	pub actor: Option<String>,
111	pub actor_id: Option<String>,
112	pub source: String,
113	pub info: String,
114	pub inputs: String,
115	pub trusted: u32,
116	pub concurrency_group: Option<String>,
117	pub created_at: String,
118	pub started_at: Option<String>,
119	pub finished_at: Option<String>,
120	/// JSON `RunApproval`, for a run that needed approval (migration 0006).
121	#[serde(default)]
122	pub approval: Option<String>,
123	/// Whether its concurrency group cancels what it replaces.
124	#[serde(default)]
125	pub cancel_in_progress: u32,
126	/// Who started the current attempt, once it is re-run (migration 0009).
127	#[serde(default)]
128	pub triggering_actor: Option<String>,
129	/// The current attempt runs with debug logging.
130	#[serde(default)]
131	pub debug: u32,
132	}
133
134	impl RunRow {
135	pub fn approval(&self) -> Option<RunApproval> {
136	self.approval.as_deref().and_then(\|text\| serde_json::from_str(text).ok())
137	}
138
139	pub fn info(&self) -> RunInfo {
140	let mut info: RunInfo = serde_json::from_str(&self.info).unwrap_or_default();
141	info.run_id = self.id.clone();
142	info.run_number = self.number;
143	info.run_attempt = self.attempt;
144	info.workflow_path = self.path.clone();
145	if info.workflow.is_empty() {
146	info.workflow = self.name.clone();
147	}
148	info
149	}
150
151	pub fn inputs(&self) -> Map<String, Value> {
152	serde_json::from_str(&self.inputs).unwrap_or_default()
153	}
154
155	pub fn summary(&self) -> WorkflowRun {
156	WorkflowRun {
157	id: self.id.clone(),
158	workflow_id: self.workflow_id.clone(),
159	path: self.path.clone(),
160	name: self.name.clone(),
161	title: self.title.clone(),
162	number: self.number,
163	attempt: self.attempt,
164	event: self.event.clone(),
165	git_ref: self.git_ref.clone(),
166	sha: self.sha.clone(),
167	pull: self.pull,
168	status: self.status.clone(),
169	conclusion: self.conclusion.clone(),
170	error: self.error.clone(),
171	actor: self.actor.clone(),
172	created_at: self.created_at.clone(),
173	started_at: self.started_at.clone(),
174	finished_at: self.finished_at.clone(),
175	}
176	}
177	}
178
179	#[derive(Clone, Deserialize)]
180	pub struct JobRow {
181	pub id: String,
182	pub run_id: String,
183	pub repo_id: String,
184	pub namespace: String,
185	pub key: String,
186	pub ordinal: u32,
187	pub name: String,
188	pub needs: String,
189	pub matrix: Option<String>,
190	pub status: String,
191	pub conclusion: Option<String>,
192	pub steps: String,
193	pub annotations: String,
194	pub outputs: String,
195	pub reason: Option<String>,
196	pub token_hash: Option<String>,
197	pub timeout_minutes: u32,
198	pub continue_on_error: u32,
199	pub max_parallel: Option<u32>,
200	/// Set for a job that calls a reusable workflow, and for that
201	/// workflow's jobs (see migration 0002).
202	pub call: Option<String>,
203	pub seen_at: Option<String>,
204	pub started_at: Option<String>,
205	pub finished_at: Option<String>,
206	/// For a job whose `runs-on` names self-hosted runners: what it asks
207	/// for, as a JSON array (see `g1t_contracts::runners::Wanted`), when it
208	/// started waiting, and the runner that took it (migration 0004).
209	#[serde(default)]
210	pub labels: Option<String>,
211	#[serde(default)]
212	pub queued_at: Option<String>,
213	#[serde(default)]
214	pub runner_id: Option<String>,
215	#[serde(default)]
216	pub runner_name: Option<String>,
217	/// The environment it names, read when its needs were done; a job its
218	/// rules hold is `pending` (migration 0006).
219	#[serde(default)]
220	pub environment: Option<String>,
221	/// Its own `concurrency` group, and whether that cancels what it replaces.
222	#[serde(default)]
223	pub concurrency_group: Option<String>,
224	#[serde(default)]
225	pub cancel_in_progress: u32,
226	/// When it was told to stop, while it runs its cleanup steps
227	/// (migration 0009).
228	#[serde(default)]
229	pub cancel_requested_at: Option<String>,
230	}
231
232	/// How long a cancelled job has to run its `if: always()` and `cancelled()`
233	/// steps and its post steps before it is stopped outright, as on GitHub.
234	pub const CANCEL_GRACE_MS: u64 = 5 * 60 * 1000;
235
236	/// Which jobs a re-run runs again.
237	#[derive(Clone, Copy, Debug, PartialEq)]
238	pub enum Rerun<'a> {
239	All,
240	/// Those that did not succeed.
241	Failed,
242	/// One job's key.
243	Job(&'a str),
244	}
245
246	/// The keys a re-run runs again, in `order`: those `which` names and,
247	/// transitively, every key that needs one of them. `needs` gives a key's
248	/// needs; `succeeded` whether every job of a key succeeded.
249	pub fn rerun_keys(order: &[&str], needs: impl Fn(&str) -> Vec<String>, succeeded: impl Fn(&str) -> bool, which: Rerun) -> Vec<String> {
250	let mut again: Vec<String> = Vec::new();
251	for key in order {
252	let named = match which {
253	Rerun::All => true,
254	Rerun::Failed => !succeeded(key),
255	Rerun::Job(job) => *key == job,
256	};
257	if named \|\| needs(key).iter().any(\|need\| again.contains(need)) {
258	again.push((*key).to_owned());
259	}
260	}
261	again
262	}
263
264	/// The key under `jobs:` a job belongs to: a called workflow's jobs
265	/// (`build/test`, or `build[1]/test` for a matrix's second call) run again
266	/// with the job that calls it (`build`).
267	pub fn top_key(key: &str) -> &str {
268	let first = key.split('/').next().unwrap_or(key);
269	first.split('[').next().unwrap_or(first)
270	}
271
272	/// What the keys of the jobs a caller's call makes start with: the
273	/// caller's key, and for a matrix the combination's index after it
274	/// (`build[1]`), so each combination's call has jobs of its own. A job id
275	/// has no `[`, so the two never meet.
276	fn call_prefix(key: &str, index: Option<usize>) -> String {
277	match index {
278	Some(index) => format!("{key}[{index}]"),
279	None => key.to_owned(),
280	}
281	}
282
283	/// Whether a run may call one more reusable workflow, `file`, `depth`
284	/// levels below the workflow it started from, having called `called`
285	/// already (one name per call; repeats are one workflow). As on GitHub: ten
286	/// levels of workflows at most, the top one and nine reusable ones below
287	/// it, and 50 different reusable workflows in a run.
288	fn call_allowed(depth: u64, called: &[&str], file: &str) -> std::result::Result<(), String> {
289	if depth > MAX_CALL_DEPTH {
290	return Err(format!(
291	"`{file}` would be the {}th level of workflows: a run connects at most ten, the workflow it started from and nine levels of reusable workflows below it.",
292	depth + 1
293	));
294	}
295	let mut unique: Vec<&str> = called.to_vec();
296	unique.push(file);
297	unique.sort_unstable();
298	unique.dedup();
299	if unique.len() > MAX_CALLED_WORKFLOWS {
300	return Err(format!("Calling `{file}` would make {} reusable workflows in this run: a run calls at most {MAX_CALLED_WORKFLOWS} different ones.", unique.len()));
301	}
302	Ok(())
303	}
304
305	/// What a job is shown as: its `name` (else its id), under the job whose
306	/// call made it (`caller`), read with its matrix when it has an
307	/// expression, else with the matrix's values after it.
308	fn shown_name(job: &workflow::Job, caller: Option<&str>, combination: &Map<String, Value>, scope: &Scope) -> String {
309	let base = job.name.clone().unwrap_or(job.id.clone());
310	let base = match caller {
311	Some(caller) => format!("{caller} / {base}"),
312	None => base,
313	};
314	if expr::has_expression(&base) {
315	expr::interpolate(&base, scope).unwrap_or(base)
316	} else if job.matrix.is_some() {
317	matrix::job_name(&base, combination)
318	} else {
319	base
320	}
321	}
322
323	/// Debug logging for a job, as a re-run with it turned on gives GitHub's:
324	/// `RUNNER_DEBUG=1` and `runner.debug`, and `ACTIONS_STEP_DEBUG` and
325	/// `ACTIONS_RUNNER_DEBUG` set, which the runner reads as the secrets of
326	/// those names (`::debug::` lines shown, and how each step's `if` read).
327	pub fn debug_logging(variables: &mut Map<String, Value>, runner: &mut Value) {
328	variables.insert("RUNNER_DEBUG".into(), json!("1"));
329	variables.insert("ACTIONS_STEP_DEBUG".into(), json!("true"));
330	variables.insert("ACTIONS_RUNNER_DEBUG".into(), json!("true"));
331	runner["debug"] = json!("1");
332	}
333
334	impl JobRow {
335	pub fn needs(&self) -> Vec<String> {
336	serde_json::from_str(&self.needs).unwrap_or_default()
337	}
338
339	pub fn call(&self) -> Option<Value> {
340	self.call.as_deref().and_then(\|call\| serde_json::from_str(call).ok())
341	}
342
343	/// For a job that calls a reusable workflow: what the keys of the jobs
344	/// its call makes start with (`call_prefix`).
345	pub fn call_prefix(&self) -> Option<String> {
346	let call = self.call().filter(\|call\| call["role"] == "caller")?;
347	Some(call["prefix"].as_str().unwrap_or(&self.key).to_owned())
348	}
349
350	/// A job that calls a reusable workflow and has not called it yet: it
351	/// waits for room under its matrix's `max-parallel`, or in its
352	/// concurrency group.
353	pub fn holds_call(&self) -> bool {
354	self.status == "waiting" && self.call_prefix().is_some()
355	}
356
357	/// For a job of a called workflow: that workflow, the job's own id in
358	/// it, and the job.
359	pub fn callee(&self) -> Option<(Workflow, workflow::Job, Value)> {
360	let call = self.call().filter(\|call\| call["role"] == "callee")?;
361	let called = workflow::parse(call["source"].as_str()?).ok()?;
362	let job = called.jobs.iter().find(\|job\| call["job"].as_str() == Some(job.id.as_str()))?.clone();
363	Some((called, job, call))
364	}
365	}
366
367	/// A job to decide on: its key, its definition, and what it needs, as
368	/// (name in `needs`, key of the jobs).
369	type Unit = (String, workflow::Job, Vec<(String, String)>);
370
371	/// How many levels of reusable workflows may sit below the workflow a run
372	/// started from, as on GitHub (ten levels of workflows in all).
373	const MAX_CALL_DEPTH: u64 = 9;
374	/// How many different reusable workflows a run may call, as on GitHub.
375	const MAX_CALLED_WORKFLOWS: usize = 50;
376
377	/// What the jobs of one key came to, for `needs.<key>`.
378	fn key_result(rows: &[&JobRow]) -> &'static str {
379	let failed = \|row: &&&JobRow\| row.conclusion.as_deref() == Some("failure") && row.continue_on_error == 0;
380	if rows.iter().any(\|row\| failed(&row)) {
381	"failure"
382	} else if rows.iter().any(\|row\| row.conclusion.as_deref() == Some("cancelled")) {
383	"cancelled"
384	} else if rows.iter().all(\|row\| row.conclusion.as_deref() == Some("skipped")) {
385	"skipped"
386	} else {
387	"success"
388	}
389	}
390
391	/// Whether any job before `key` failed: one it needs, or one those need,
392	/// however far back. A job after a skipped one still sees the failure
393	/// that skipped it, as GitHub's failure() does. `needs_of`: each key's
394	/// needs, as keys; `failed`: whether a key's jobs came to a failure.
395	fn ancestor_failed(needs_of: &std::collections::HashMap<&str, Vec<&str>>, key: &str, failed: impl Fn(&str) -> bool) -> bool {
396	let mut seen = std::collections::HashSet::new();
397	let mut stack: Vec<&str> = needs_of.get(key).cloned().unwrap_or_default();
398	while let Some(next) = stack.pop() {
399	if !seen.insert(next) {
400	continue;
401	}
402	if failed(next) {
403	return true;
404	}
405	stack.extend(needs_of.get(next).into_iter().flatten().copied());
406	}
407	false
408	}
409
410	/// A job's `environment:`, as deployments read it: its name, the address in
411	/// `environment.url` once its expressions are filled in from the run, and
412	/// whether the job deploys to it. A job with `deployment: false` only reads
413	/// the environment's secrets and variables, and makes no deployment.
414	#[derive(Clone, Debug, Default, PartialEq)]
415	pub(crate) struct JobEnvironment {
416	pub(crate) name: String,
417	pub(crate) url: Option<String>,
418	pub(crate) deploys: bool,
419	}
420
421	/// The environment `environment:` names, as written in `raw` (a job), with
422	/// `contexts` to fill in expressions: null when it names none, or only
423	/// through an expression this cannot read before the job runs.
424	pub(crate) fn environment_of(raw: &Value, contexts: &Map<String, Value>) -> Option<JobEnvironment> {
425	let scope = Scope { contexts, status: Status::Success, hash_files: None };
426	let plain = \|value: &Value\| -> Option<String> {
427	let text = match value {
428	Value::String(text) if text.contains("${{") => expr::interpolate_value(value, &scope).ok().map(\|v\| expr::to_text(&v))?,
429	Value::String(text) => text.clone(),
430	_ => return None,
431	};
432	let text = text.trim().to_owned();
433	(!text.is_empty()).then_some(text)
434	};
435	match raw.get("environment")? {
436	name @ Value::String(_) => Some(JobEnvironment { name: plain(name)?, url: None, deploys: workflow::records_deployment(raw) }),
437	Value::Object(env) => {
438	let name = plain(env.get("name")?)?;
439	let url = env
440	.get("url")
441	.and_then(plain)
442	.filter(\|url\| url.starts_with("https://") \|\| url.starts_with("http://"));
443	Some(JobEnvironment { name, url, deploys: workflow::records_deployment(raw) })
444	}
445	_ => None,
446	}
447	}
448
449	/// The contexts a job's `runs-on` and `environment` are read with before it
450	/// runs: the run's `github`, its inputs, and the job's matrix.
451	fn start_contexts(run: &RunRow, job: &JobRow, key: &str) -> Map<String, Value> {
452	let mut contexts = Map::new();
453	contexts.insert("github".into(), run.info().context(key, "", run.action.as_deref()));
454	contexts.insert("inputs".into(), Value::Object(run.inputs()));
455	contexts.insert("matrix".into(), job.matrix.as_deref().and_then(\|m\| serde_json::from_str(m).ok()).unwrap_or_else(\|\| json!({})));
456	contexts
457	}
458
459	/// The job as its workflow (or the workflow it calls) defines it.
460	fn job_spec(run: &RunRow, job: &JobRow) -> Option<workflow::Job> {
461	match job.callee() {
462	Some((_, spec, _)) => Some(spec),
463	None => workflow::parse(&run.source).ok().and_then(\|w\| w.jobs.into_iter().find(\|j\| j.id == job.key)),
464	}
465	}
466
467	/// The environment a job deploys to, if it deploys: by the name read when
468	/// its needs were done (which may have needed their outputs), else as
469	/// its `environment:` reads now.
470	pub(crate) fn deploys_to(run: &RunRow, job: &JobRow) -> Option<JobEnvironment> {
471	let spec = job_spec(run, job)?;
472	let read = environment_of(&spec.raw, &start_contexts(run, job, &spec.id));
473	let env = match (read, &job.environment) {
474	(Some(env), Some(name)) => Some(JobEnvironment { name: name.clone(), ..env }),
475	(Some(env), None) => Some(env),
476	(None, Some(name)) => Some(JobEnvironment { name: name.clone(), url: None, deploys: workflow::records_deployment(&spec.raw) }),
477	(None, None) => None,
478	};
479	env.filter(\|env\| env.deploys)
480	}
481
482	/// What the runner is told about a job it starts, from its workflow: the
483	/// environment it names plainly, and the machine its `runs-on` asks for
484	/// (`instance_for`; none for the standard one).
485	#[derive(Default)]
486	struct StartDetails {
487	environment: Option<String>,
488	instance: Option<String>,
489	}
490
491	fn start_details(run: &RunRow, job: &JobRow) -> StartDetails {
492	let spec = match job.callee() {
493	Some((_, spec, _)) => spec,
494	None => match workflow::parse(&run.source).ok().and_then(\|w\| w.jobs.into_iter().find(\|j\| j.id == job.key)) {
495	Some(spec) => spec,
496	None => return StartDetails::default(),
497	},
498	};
499	// As read when its needs were done, an expression's included.
500	let environment = job.environment.clone();
501	// `runs-on` as the job was queued with: its matrix and the run's
502	// inputs. A label that needs more than those is the standard machine.
503	let contexts = start_contexts(run, job, &spec.id);
504	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
505	let labels: Vec<String> = match expr::interpolate_value(&spec.runs_on, &scope).unwrap_or(Value::Null) {
506	Value::String(label) => vec![label],
507	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
508	Value::Object(given) => match given.get("labels") {
509	Some(Value::Array(labels)) => labels.iter().map(expr::to_text).collect(),
510	Some(label) => vec![expr::to_text(label)],
511	None => Vec::new(),
512	},
513	_ => Vec::new(),
514	};
515	let instance = g1t_contracts::actions::instance_for(&labels);
516	StartDetails {
517	environment,
518	instance: (instance != g1t_contracts::actions::STANDARD_INSTANCE).then(\|\| instance.label.to_owned()),
519	}
520	}
521
522	fn now() -> String {
523	rfc3339(now_ms())
524	}
525
526	/// How a finished run went for one environment, from the conclusions of
527	/// the jobs that deploy to it: failed if any failed, an error if any was
528	/// cancelled, else a success; nothing when none ran (all skipped).
529	pub(crate) fn deployment_outcome(conclusions: &[Option<String>]) -> Option<&'static str> {
530	let ran: Vec<&str> = conclusions.iter().flatten().map(String::as_str).filter(\|c\| *c != "skipped").collect();
531	if ran.is_empty() {
532	return None;
533	}
534	if ran.iter().any(\|c\| c == "failure" \|\| c == "timed_out") {
535	return Some("failure");
536	}
537	if ran.contains(&"cancelled") {
538	return Some("error");
539	}
540	Some("success")
541	}
542
543	impl Actions {
544	pub async fn run_row(&self, id: &str) -> Result<Option<RunRow>> {
545	self.db.prepare("SELECT * FROM runs WHERE id = ?").bind(&[id.into()])?.first::<RunRow>(None).await
546	}
547
548	pub async fn job_rows(&self, run_id: &str) -> Result<Vec<JobRow>> {
549	self.db
550	.prepare("SELECT * FROM jobs WHERE run_id = ? ORDER BY rowid")
551	.bind(&[run_id.into()])?
552	.all()
553	.await?
554	.results::<JobRow>()
555	}
556
557	pub async fn run_summary(&self, id: &str) -> Result<Outcome<WorkflowRun>> {
558	Ok(match self.run_row(id).await? {
559	Some(row) => Outcome::Ok(row.summary()),
560	None => fail(FailureCode::NotFound, "No such run."),
561	})
562	}
563
564	/// The contexts every expression outside a job's steps may use.
565	fn base_contexts(run: &RunRow, vars: &Map<String, Value>, job: &str) -> Map<String, Value> {
566	let mut contexts = Map::new();
567	contexts.insert("github".into(), run.info().context(job, "", run.action.as_deref()));
568	contexts.insert("inputs".into(), Value::Object(run.inputs()));
569	contexts.insert("vars".into(), Value::Object(vars.clone()));
570	contexts.insert("needs".into(), json!({}));
571	contexts.insert("runner".into(), runner_context());
572	contexts
573	}
574
575	/// Makes a run and its jobs, and starts what can start. `None` when the
576	/// event already started this workflow.
577	pub async fn create_run(&self, new: NewRun) -> Result<Option<String>> {
578	// Nothing starts on an archived repository. A deleted one is never
579	// found to start anything on.
580	if new.repo.archived() {
581	return Ok(None);
582	}
583	let workflow_row = self.workflow_row(&new.repo, &new.path, &new.workflow.display_name(&new.path), &new.source).await?;
584	let numbered = self
585	.db
586	.prepare("UPDATE workflows SET run_count = run_count + 1 WHERE id = ? RETURNING run_count AS n")
587	.bind(&[workflow_row.id.as_str().into()])?
588	.first::<Count>(None)
589	.await?
590	.map_or(1, \|count\| count.n);
591	let id = new_id("run", now_ms());
592	let mut info = new.info.clone();
593	info.workflow = new.workflow.display_name(&new.path);
594	info.workflow_path = new.path.clone();
595	info.run_id = id.clone();
596	info.run_number = u64::from(numbered);
597	let vars = self
598	.variables_for(&new.repo.id, &format!("{}/{}", new.repo.namespace, new.repo.name), None, new.trusted)
599	.await?;
600
601	// run-name and the concurrency group read github, inputs and vars.
602	let mut contexts = Map::new();
603	contexts.insert("github".into(), info.context("", "", new.action.as_deref()));
604	contexts.insert("inputs".into(), Value::Object(new.inputs.clone()));
605	contexts.insert("vars".into(), Value::Object(vars));
606	let scope = Scope {
607	contexts: &contexts,
608	status: Status::Success,
609	hash_files: None,
610	};
611	let title = new
612	.workflow
613	.run_name
614	.as_deref()
615	.and_then(\|run_name\| expr::interpolate(run_name, &scope).ok())
616	.filter(\|title\| !title.trim().is_empty())
617	.unwrap_or(new.title.clone());
618	let group = new.workflow.concurrency.as_ref().and_then(\|c\| expr::interpolate(&c.group, &scope).ok());
619	let cancel_in_progress = new
620	.workflow
621	.concurrency
622	.as_ref()
623	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
624	.is_some_and(\|value\| expr::truthy(&value));
625
626	let approval = new.approval.as_ref().map(\|reason\| RunApproval { state: "required".into(), reason: reason.clone(), approved_by: None });
627	let inserted = self
628	.db
629	.prepare(
630	"INSERT OR IGNORE INTO runs (id, workflow_id, repo_id, repo, path, name, title, number, event, action, git_ref, sha,
631	pull, status, actor, actor_id, source, info, inputs, trusted, concurrency_group, event_key, created_at,
632	approval, cancel_in_progress)
633	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",
634	)
635	.bind(&[
636	id.as_str().into(),
637	workflow_row.id.as_str().into(),
638	new.repo.id.as_str().into(),
639	format!("{}/{}", new.repo.namespace, new.repo.name).into(),
640	new.path.as_str().into(),
641	info.workflow.as_str().into(),
642	title.as_str().into(),
643	numbered.into(),
644	info.event_name.as_str().into(),
645	optional(new.action.as_deref()),
646	info.git_ref.as_str().into(),
647	info.sha.as_str().into(),
648	new.pull.map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
649	if approval.is_some() { "action_required" } else { "queued" }.into(),
650	optional(new.actor.as_deref()),
651	optional(new.actor_id.as_deref()),
652	new.source.as_str().into(),
653	serde_json::to_string(&info)?.into(),
654	serde_json::to_string(&new.inputs)?.into(),
655	u32::from(new.trusted).into(),
656	optional(group.as_deref()),
657	new.event_key.as_str().into(),
658	now().into(),
659	approval.as_ref().map(serde_json::to_string).transpose()?.as_deref().map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
660	u32::from(cancel_in_progress).into(),
661	])?
662	.first::<Value>(None)
663	.await?;
664	if inserted.is_none() {
665	return Ok(None);
666	}
667	if let Some(run) = self.run_row(&id).await? {
668	self.report_pending(&run).await?;
669	}
670
671	// Every job, waiting; each is expanded when the jobs it needs are done.
672	let mut statements = Vec::new();
673	for job in &new.workflow.jobs {
674	statements.push(
675	self.db
676	.prepare(
677	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, name, needs, status) VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting')",
678	)
679	.bind(&[
680	new_id("job", now_ms()).into(),
681	id.as_str().into(),
682	new.repo.id.as_str().into(),
683	new.repo.namespace.as_str().into(),
684	job.id.as_str().into(),
685	job.name.clone().filter(\|n\| !expr::has_expression(n)).unwrap_or(job.id.clone()).into(),
686	serde_json::to_string(&job.needs)?.into(),
687	])?,
688	);
689	}
690	self.db.batch(statements).await?;
691
692	// A pull request's run from outside waits to be approved; it joins
693	// its concurrency group once it is (protection.rs, approve_run).
694	if approval.is_some() {
695	return Ok(Some(id));
696	}
697	if let Some(run) = self.run_row(&id).await? {
698	self.enter_group(&run).await?;
699	}
700	Ok(Some(id))
701	}
702
703	/// A run about to start: one run at a time per concurrency group, as on
704	/// GitHub. A newer run replaces one that waits in the group, and with
705	/// `cancel-in-progress` the one that runs; otherwise it waits as
706	/// `pending` for the one that runs. Then it moves along.
707	pub(crate) async fn enter_group(&self, run: &RunRow) -> Result<()> {
708	if let Some(group) = &run.concurrency_group {
709	let cancel_in_progress = run.cancel_in_progress != 0;
710	let others = self
711	.db
712	.prepare(
713	"SELECT * FROM runs WHERE repo_id = ? AND concurrency_group = ? AND id != ? AND status NOT IN ('completed', 'action_required') ORDER BY id",
714	)
715	.bind(&[run.repo_id.as_str().into(), group.as_str().into(), run.id.as_str().into()])?
716	.all()
717	.await?
718	.results::<RunRow>()?;
719	for other in &others {
720	if cancel_in_progress \|\| other.status == "pending" {
721	// A newer run replaces a waiting one, as on GitHub.
722	self.cancel_run(other, "A newer run in the same concurrency group replaced it.", false).await?;
723	}
724	}
725	if !cancel_in_progress && others.iter().any(\|other\| other.status != "pending") {
726	self.db
727	.prepare("UPDATE runs SET status = 'pending' WHERE id = ?")
728	.bind(&[run.id.as_str().into()])?
729	.run()
730	.await?;
731	return Ok(());
732	}
733	}
734	self.advance(&run.id).await
735	}
736
737	/// A run that could not start, such as for a workflow file that does not read.
738	#[allow(clippy::too_many_arguments)]
739	pub async fn record_failed_run(
740	&self,
741	row: &WorkflowRow,
742	git_ref: &str,
743	sha: &str,
744	event_key: &str,
745	actor_id: Option<&str>,
746	actor: &str,
747	problem: &str,
748	) -> Result<()> {
749	let numbered = self
750	.db
751	.prepare("UPDATE workflows SET run_count = run_count + 1 WHERE id = ? RETURNING run_count AS n")
752	.bind(&[row.id.as_str().into()])?
753	.first::<Count>(None)
754	.await?
755	.map_or(1, \|count\| count.n);
756	let at = now();
757	self.db
758	.prepare(
759	"INSERT OR IGNORE INTO runs (id, workflow_id, repo_id, repo, path, name, title, number, event, git_ref, sha, status,
760	conclusion, error, actor, actor_id, source, info, event_key, created_at, finished_at)
761	VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'push', ?, ?, 'completed', 'failure', ?, ?, ?, ?, '{}', ?, ?, ?)",
762	)
763	.bind(&[
764	new_id("run", now_ms()).into(),
765	row.id.as_str().into(),
766	row.repo_id.as_str().into(),
767	row.repo.as_str().into(),
768	row.path.as_str().into(),
769	row.path.as_str().into(),
770	"Invalid workflow file".into(),
771	numbered.into(),
772	git_ref.into(),
773	sha.into(),
774	problem.into(),
775	actor.into(),
776	optional(actor_id),
777	row.source.as_str().into(),
778	event_key.into(),
779	at.as_str().into(),
780	at.as_str().into(),
781	])?
782	.run()
783	.await?;
784	Ok(())
785	}
786
787	/// Moves a run along: jobs whose needs are done are decided on, and
788	/// jobs that can start are started.
789	pub async fn advance(&self, run_id: &str) -> Result<()> {
790	// Each pass may finish jobs (skipped ones), which may free others.
791	for _ in 0..20 {
792	let Some(run) = self.run_row(run_id).await? else { return Ok(()) };
793	if matches!(run.status.as_str(), "completed" \| "pending" \| "action_required") {
794	return Ok(());
795	}
796	let workflow = match workflow::parse(&run.source) {
797	Ok(workflow) => workflow,
798	Err(problem) => {
799	self.finish_run(&run, Some(&problem)).await?;
800	return Ok(());
801	}
802	};
803	let jobs = self.job_rows(run_id).await?;
804	let mut changed = false;
805	// What to decide on: the workflow's jobs, and the jobs of the
806	// workflows they call, each with its needs as (name, key).
807	let mut units: Vec<Unit> = workflow
808	.jobs
809	.iter()
810	.map(\|job\| (job.id.clone(), job.clone(), job.needs.iter().map(\|n\| (n.clone(), n.clone())).collect()))
811	.collect();
812	let mut seen = std::collections::HashSet::new();
813	for row in &jobs {
814	if !seen.insert(row.key.clone()) {
815	continue;
816	}
817	if let Some((_, job, call)) = row.callee() {
818	let parent = call["parent"].as_str().unwrap_or_default().to_owned();
819	let needs = job.needs.iter().map(\|n\| (n.clone(), format!("{parent}/{n}"))).collect();
820	units.push((row.key.clone(), job, needs));
821	}
822	}
823	// Each key's needs, as keys, to look back through for failure().
824	let needs_of: std::collections::HashMap<&str, Vec<&str>> =
825	units.iter().map(\|(key, _, needs)\| (key.as_str(), needs.iter().map(\|(_, need)\| need.as_str()).collect())).collect();
826	let key_failed = \|key: &str\| key_result(&jobs.iter().filter(\|row\| row.key == key).collect::<Vec<_>>()) == "failure";
827	for (key, job, needs) in &units {
828	let rows: Vec<&JobRow> = jobs.iter().filter(\|row\| &row.key == key).collect();
829	if rows.is_empty() \|\| !rows.iter().all(\|row\| row.status == "waiting" && !row.holds_call()) {
830	continue;
831	}
832	let needed: Vec<(&String, Vec<&JobRow>)> =
833	needs.iter().map(\|(name, need)\| (name, jobs.iter().filter(\|row\| &row.key == need).collect())).collect();
834	if !needed.iter().all(\|(_, rows)\| rows.iter().all(\|row\| row.status == "completed")) {
835	continue;
836	}
837	let failed_before = ancestor_failed(&needs_of, key, key_failed);
838	self.decide(&run, job, rows[0], &needed, failed_before).await?;
839	changed = true;
840	}
841	// A job that called a workflow finishes with that workflow's jobs.
842	for row in jobs.iter().filter(\|row\| row.status == "calling") {
843	let prefix = row.call_prefix().unwrap_or(row.key.clone());
844	let children: Vec<&JobRow> = jobs
845	.iter()
846	.filter(\|child\| child.call().is_some_and(\|call\| call["role"] == "callee" && call["parent"].as_str() == Some(prefix.as_str())))
847	.collect();
848	if !children.is_empty() && children.iter().all(\|child\| child.status == "completed") {
849	self.finish_call(&run, row, &children).await?;
850	changed = true;
851	}
852	}
853	// Calls that wait start while their matrix's `max-parallel` and
854	// their concurrency group have room.
855	let mut calling: std::collections::HashMap<&str, u32> = std::collections::HashMap::new();
856	for row in jobs.iter().filter(\|row\| row.status == "calling") {
857	*calling.entry(row.key.as_str()).or_default() += 1;
858	}
859	for row in jobs.iter().filter(\|row\| row.holds_call()) {
860	let running = calling.get(row.key.as_str()).copied().unwrap_or(0);
861	if row.max_parallel.is_some_and(\|max\| running >= max) \|\| self.group_busy(row).await? {
862	continue;
863	}
864	if self.start_call(row).await? {
865	calling.insert(row.key.as_str(), running + 1);
866	changed = true;
867	}
868	}
869	if !changed {
870	break;
871	}
872	}
873	self.start_queued().await?;
874	self.finish_if_done(run_id).await
875	}
876
877	/// Decides on one job whose needs are done: skip it, fail it, or expand
878	/// it into its matrix and queue it. `failed_before`: whether any job
879	/// before it failed, however far back (`ancestor_failed`).
880	async fn decide(&self, run: &RunRow, job: &workflow::Job, row: &JobRow, needed: &[(&String, Vec<&JobRow>)], failed_before: bool) -> Result<()> {
881	let vars = self.variables_for(&run.repo_id, &run.repo, None, run.trusted != 0).await?;
882	let mut contexts = Self::base_contexts(run, &vars, &job.id);
883	// A called workflow's jobs read the inputs they were called with.
884	let call = row.call();
885	// Shown under the job whose call made it, by that job's name.
886	let caller = call.as_ref().filter(\|c\| c["role"] == "callee").and_then(\|c\| {
887	c["title"].as_str().map(str::to_owned).or_else(\|\| c["parent"].as_str().map(\|parent\| parent.replace('/', " / ")))
888	});
889	if let Some(call) = call.as_ref().filter(\|c\| c["role"] == "callee") {
890	contexts.insert("inputs".into(), call["inputs"].clone());
891	}
892	let mut needs = Map::new();
893	let mut results = Vec::new();
894	for (key, rows) in needed {
895	let result = key_result(rows);
896	let mut outputs = Map::new();
897	for row in rows {
898	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&row.outputs) {
899	outputs.extend(more);
900	}
901	}
902	results.push(result);
903	needs.insert((*key).clone(), json!({ "result": result, "outputs": outputs }));
904	}
905	// As on GitHub: a need that was skipped makes success() false but
906	// not failure(); failure() is a failure anywhere before the job.
907	let status = expr::job_status(results, failed_before, run.conclusion.as_deref() == Some("cancelled"));
908	contexts.insert("needs".into(), Value::Object(needs));
909	let scope = Scope {
910	contexts: &contexts,
911	status,
912	hash_files: None,
913	};
914	let condition = job.condition.as_deref().unwrap_or_default();
915	match expr::condition(condition, &scope) {
916	Ok(true) => {}
917	Ok(false) => return self.skip_job(row, None).await,
918	Err(problem) => return self.fail_job(row, &format!("Its `if` does not read: {problem}")).await,
919	}
920	// Its matrix, which may come from a needed job's outputs.
921	let combinations = match &job.matrix {
922	None => vec![Map::new()],
923	Some(matrix) => {
924	let value = match expr::interpolate_value(matrix, &scope) {
925	Ok(value) => value,
926	Err(problem) => return self.fail_job(row, &format!("Its matrix does not read: {problem}")).await,
927	};
928	match matrix::expand(&value) {
929	Ok(combinations) if !combinations.is_empty() => combinations,
930	Ok(_) => return self.fail_job(row, "Its matrix makes no jobs.").await,
931	Err(problem) => return self.fail_job(row, &problem).await,
932	}
933	}
934	};
935	// A job that calls a reusable workflow calls it once for each.
936	if let Some(uses) = &job.uses {
937	return self.call_workflow(run, job, row, uses, &contexts, &combinations, caller.as_deref()).await;
938	}
939	let total = combinations.len();
940	let raw = job.raw.as_object().cloned().unwrap_or_default();
941	// Whether pull requests from forks may use self-hosted runners here,
942	// asked once, and only for a run that is not trusted.
943	let mut forks_allowed: Option<bool> = None;
944	// Each concurrency group its jobs join, and whether it cancels.
945	let mut groups: Vec<(String, bool)> = Vec::new();
946	let mut statements = Vec::new();
947	for (index, combination) in combinations.iter().enumerate() {
948	let mut contexts = contexts.clone();
949	contexts.insert("matrix".into(), Value::Object(combination.clone()));
950	contexts.insert(
951	"strategy".into(),
952	json!({ "fail-fast": job.fail_fast, "job-index": index, "job-total": total, "max-parallel": job.max_parallel.unwrap_or(total as u32) }),
953	);
954	let scope = Scope {
955	contexts: &contexts,
956	status: Status::Success,
957	hash_files: None,
958	};
959	let name = shown_name(job, caller.as_deref(), combination, &scope);
960	let runs_on = expr::interpolate_value(&job.runs_on, &scope).unwrap_or(Value::Null);
961	let labels = match &runs_on {
962	Value::String(label) => vec![label.clone()],
963	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
964	Value::Object(spec) => spec.get("labels").map(\|l\| match l {
965	Value::Array(labels) => labels.iter().map(expr::to_text).collect(),
966	other => vec![expr::to_text(other)],
967	}).unwrap_or_default(),
968	_ => Vec::new(),
969	};
970	// `runs-on: self-hosted` (or a group): the workspace's own
971	// machines, which may run any OS. Otherwise g1t's Linux sandboxes.
972	let group = match &runs_on {
973	Value::Object(spec) => spec.get("group").map(expr::to_text),
974	_ => None,
975	};
976	let wanted = Wanted::of(&labels, group.as_deref());
977	let mut reason = if wanted.self_hosted {
978	None
979	} else {
980	labels
981	.iter()
982	.find(\|label\| {
983	let lower = label.to_ascii_lowercase();
984	lower.contains("windows") \|\| lower.contains("macos")
985	})
986	.map(\|label\| {
987	let os = if label.to_ascii_lowercase().contains("windows") { "windows" } else { "macos" };
988	format!("`runs-on: {label}`: g1t's own runners are Linux. A self-hosted runner can run it: `runs-on: [self-hosted, {os}]`.")
989	})
990	};
991	// A pull request from a fork runs code anyone could write: never
992	// on the workspace's machines unless it said they may.
993	if wanted.self_hosted && run.trusted == 0 {
994	let allowed = match forks_allowed {
995	Some(allowed) => allowed,
996	None => {
997	let allowed = self.effective_runner_settings(&row.namespace, Some(&run.repo_id)).await?.fork_pull_requests;
998	forks_allowed = Some(allowed);
999	allowed
1000	}
1001	};
1002	if !allowed {
1003	reason = Some(
1004	"Pull requests from forks do not run on self-hosted runners here. An admin can allow it under Settings, Runners.".to_owned(),
1005	);
1006	}
1007	}
1008	let max_minutes = if wanted.self_hosted { SELF_HOSTED_MAX_TIMEOUT_MINUTES } else { MAX_TIMEOUT_MINUTES };
1009	let timeout = raw
1010	.get("timeout-minutes")
1011	.and_then(\|value\| expr::interpolate_value(value, &scope).ok())
1012	.and_then(\|value\| value.as_f64().or_else(\|\| expr::to_text(&value).parse().ok()))
1013	.map_or(g1t_actions::workflow::DEFAULT_TIMEOUT_MINUTES.min(max_minutes), \|minutes\| (minutes.ceil() as u32).clamp(1, max_minutes));
1014	let continue_on_error = raw
1015	.get("continue-on-error")
1016	.and_then(\|value\| expr::interpolate_value(value, &scope).ok())
1017	.is_some_and(\|value\| expr::truthy(&value));
1018	let (mut status, mut conclusion, mut finished) = match &reason {
1019	Some(_) => ("completed", Some("failure"), Some(now())),
1020	None => ("queued", None, None),
1021	};
1022	// A self-hosted job waits, saying for what, until a runner takes it.
1023	let (labels_json, queued_at) = if wanted.self_hosted && reason.is_none() {
1024	reason = Some(waiting_reason(&wanted));
1025	(Some(serde_json::to_string(&wanted.stored())?), Some(now()))
1026	} else {
1027	(None, None)
1028	};
1029	// The environment it names, an expression read now, and what
1030	// that environment's protection rules say: it starts, waits as
1031	// `pending` (protection.rs), or may not deploy there.
1032	let environment = environment_of(&job.raw, &contexts).map(\|env\| env.name);
1033	if status == "queued"
1034	&& let Some(name) = &environment
1035	{
1036	match self.gate(run, name).await? {
1037	Gate::Open => {}
1038	Gate::Held(why) => {
1039	status = "pending";
1040	reason = Some(why);
1041	}
1042	Gate::Refused(why) => {
1043	(status, conclusion, finished) = ("completed", Some("failure"), Some(now()));
1044	reason = Some(why);
1045	}
1046	}
1047	}
1048	// Its own concurrency group.
1049	let job_group = job
1050	.concurrency
1051	.as_ref()
1052	.and_then(\|c\| expr::interpolate(&c.group, &scope).ok())
1053	.map(\|group\| group.trim().to_owned())
1054	.filter(\|group\| !group.is_empty());
1055	let job_cancel = job
1056	.concurrency
1057	.as_ref()
1058	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
1059	.is_some_and(\|value\| expr::truthy(&value));
1060	if status != "completed"
1061	&& let Some(group) = &job_group
1062	&& !groups.iter().any(\|(known, _)\| known == group)
1063	{
1064	groups.push((group.clone(), job_cancel));
1065	}
1066	let values: Vec<worker::wasm_bindgen::JsValue> = vec![
1067	name.into(),
1068	serde_json::to_string(combination)?.into(),
1069	status.into(),
1070	optional(conclusion),
1071	optional(reason.as_deref()),
1072	timeout.into(),
1073	u32::from(continue_on_error).into(),
1074	job.max_parallel.map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1075	optional(finished.as_deref()),
1076	optional(labels_json.as_deref()),
1077	optional(queued_at.as_deref()),
1078	optional(environment.as_deref()),
1079	optional(job_group.as_deref()),
1080	u32::from(job_cancel).into(),
1081	];
1082	if index == 0 {
1083	let mut bound = values;
1084	bound.push(row.id.as_str().into());
1085	statements.push(
1086	self.db
1087	.prepare(
1088	"UPDATE jobs SET name = ?, matrix = ?, status = ?, conclusion = ?, reason = ?, timeout_minutes = ?,
1089	continue_on_error = ?, max_parallel = ?, finished_at = ?, labels = ?, queued_at = ?, environment = ?,
1090	concurrency_group = ?, cancel_in_progress = ? WHERE id = ?",
1091	)
1092	.bind(&bound)?,
1093	);
1094	} else {
1095	let mut bound: Vec<worker::wasm_bindgen::JsValue> = vec![
1096	new_id("job", now_ms()).into(),
1097	row.run_id.as_str().into(),
1098	row.repo_id.as_str().into(),
1099	row.namespace.as_str().into(),
1100	row.key.as_str().into(),
1101	(index as u32).into(),
1102	row.needs.as_str().into(),
1103	optional(row.call.as_deref()),
1104	];
1105	bound.extend(values);
1106	statements.push(
1107	self.db
1108	.prepare(
1109	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, call, name, matrix, status, conclusion, reason,
1110	timeout_minutes, continue_on_error, max_parallel, finished_at, labels, queued_at, environment,
1111	concurrency_group, cancel_in_progress)
1112	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1113	)
1114	.bind(&bound)?,
1115	);
1116	}
1117	}
1118	self.db.batch(statements).await?;
1119	self.replace_in_groups(run, &row.key, &groups).await
1120	}
1121
1122	/// A job queued in its own concurrency group: with `cancel-in-progress`
1123	/// it cancels the group's other jobs; otherwise it replaces any that
1124	/// are queued and not started, and waits for the one running
1125	/// (`start_queued` starts one of a group at a time). As on GitHub.
1126	async fn replace_in_groups(&self, run: &RunRow, key: &str, groups: &[(String, bool)]) -> Result<()> {
1127	if groups.is_empty() {
1128	return Ok(());
1129	}
1130	#[derive(Deserialize)]
1131	struct Id {
1132	id: String,
1133	}
1134	let mine: Vec<String> = self
1135	.db
1136	.prepare("SELECT id FROM jobs WHERE run_id = ? AND key = ?")
1137	.bind(&[run.id.as_str().into(), key.into()])?
1138	.all()
1139	.await?
1140	.results::<Id>()?
1141	.into_iter()
1142	.map(\|row\| row.id)
1143	.collect();
1144	let mut moved: Vec<String> = Vec::new();
1145	for (group, cancel) in groups {
1146	let others = self
1147	.db
1148	// A call that waits in the group, or calls, is in it too.
1149	.prepare("SELECT * FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('waiting', 'queued', 'pending', 'in_progress', 'calling')")
1150	.bind(&[run.repo_id.as_str().into(), group.as_str().into()])?
1151	.all()
1152	.await?
1153	.results::<JobRow>()?;
1154	for other in others.iter().filter(\|other\| !mine.contains(&other.id)) {
1155	if *cancel \|\| matches!(other.status.as_str(), "queued" \| "waiting") {
1156	self.stop_job(other, "A newer job in the same concurrency group replaced it.").await?;
1157	if !moved.contains(&other.run_id) {
1158	moved.push(other.run_id.clone());
1159	}
1160	}
1161	}
1162	}
1163	for other_run in moved.iter().filter(\|id\| **id != run.id) {
1164	Box::pin(self.advance(other_run)).await?;
1165	}
1166	Ok(())
1167	}
1168
1169	/// A job that calls a reusable workflow, in the repository or another
1170	/// (reach.rs): once for each combination of its matrix, as on GitHub,
1171	/// each with the inputs and secrets it passes read with that
1172	/// combination's `matrix`. Each call waits (`holds_call`) until its
1173	/// matrix's `max-parallel` and its concurrency group have room, when
1174	/// `start_call` puts the workflow's jobs in the run under it.
1175	#[allow(clippy::too_many_arguments)]
1176	async fn call_workflow(
1177	&self,
1178	run: &RunRow,
1179	job: &workflow::Job,
1180	row: &JobRow,
1181	uses: &str,
1182	contexts: &Map<String, Value>,
1183	combinations: &[Map<String, Value>],
1184	caller: Option<&str>,
1185	) -> Result<()> {
1186	let depth = row.call().and_then(\|c\| c["depth"].as_u64()).unwrap_or(0) + 1;
1187	if depth > MAX_CALL_DEPTH {
1188	return self.fail_job(row, &call_allowed(depth, &[], uses).unwrap_err()).await;
1189	}
1190	let (file, source, origin) = match self.called_workflow(run, row, uses).await? {
1191	Ok(found) => found,
1192	Err(why) => return self.fail_job(row, &why).await,
1193	};
1194	let jobs = self.job_rows(&run.id).await?;
1195	// Each call so far, by the workflow it called.
1196	let called_paths: Vec<String> = jobs
1197	.iter()
1198	.filter(\|other\| other.call_prefix().is_some())
1199	.filter_map(JobRow::call)
1200	.filter_map(\|call\| call["path"].as_str().map(str::to_owned))
1201	.collect();
1202	let called_already: Vec<&str> = called_paths.iter().map(String::as_str).collect();
1203	if let Err(why) = call_allowed(depth, &called_already, &file) {
1204	return self.fail_job(row, &why).await;
1205	}
1206	let called = match workflow::parse(&source) {
1207	Ok(called) => called,
1208	Err(problem) => return self.fail_job(row, &format!("`{file}` does not read: {problem}")).await,
1209	};
1210	let Some(trigger) = called.trigger("workflow_call") else {
1211	return self.fail_job(row, &format!("`{file}` cannot be called: it has no `on: workflow_call`.")).await;
1212	};
1213	// Secrets: none but the job's token unless `secrets:` passes them,
1214	// by name or with `inherit`; read when each job starts (job_spec).
1215	let outer = row.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c.get("secrets").cloned());
1216	let total = combinations.len();
1217	let mut groups: Vec<(String, bool)> = Vec::new();
1218	let mut statements = Vec::new();
1219	for (index, combination) in combinations.iter().enumerate() {
1220	let mut contexts = contexts.clone();
1221	contexts.insert("matrix".into(), Value::Object(combination.clone()));
1222	contexts.insert(
1223	"strategy".into(),
1224	json!({ "fail-fast": job.fail_fast, "job-index": index, "job-total": total, "max-parallel": job.max_parallel.unwrap_or(total as u32) }),
1225	);
1226	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
1227	let name = shown_name(job, caller, combination, &scope);
1228	// Inputs: what `with:` passes, as the called workflow types them.
1229	let given = match job.raw.get("with") {
1230	Some(with) => match expr::interpolate_value(with, &scope) {
1231	Ok(Value::Object(given)) => Ok(given),
1232	Ok(_) => Ok(Map::new()),
1233	Err(problem) => Err(format!("Its `with` does not read: {problem}")),
1234	},
1235	None => Ok(Map::new()),
1236	};
1237	let inputs = given.and_then(\|given\| crate::reach::call_inputs(&file, &trigger.inputs, &given));
1238	let secrets = crate::reach::secrets_plan(&job.raw, &contexts, outer.clone());
1239	let problem = match &inputs {
1240	Err(why) => Some(why.clone()),
1241	Ok(_) => crate::reach::missing_secrets(&called.raw, &secrets)
1242	.first()
1243	.map(\|name\| format!("`{file}` needs the secret `{name}`: pass it under `secrets:`, or use `secrets: inherit`."))
1244	.or_else(\|\| {
1245	crate::reach::undeclared_secrets(&called.raw, &secrets)
1246	.first()
1247	.map(\|name\| format!("Invalid secret: `{name}` is not defined in `{file}`."))
1248	}),
1249	};
1250	// Its own concurrency group, which it holds while its workflow runs.
1251	let group = job
1252	.concurrency
1253	.as_ref()
1254	.and_then(\|c\| expr::interpolate(&c.group, &scope).ok())
1255	.map(\|group\| group.trim().to_owned())
1256	.filter(\|group\| !group.is_empty());
1257	let cancel = job
1258	.concurrency
1259	.as_ref()
1260	.and_then(\|c\| expr::interpolate_value(&c.cancel_in_progress, &scope).ok())
1261	.is_some_and(\|value\| expr::truthy(&value));
1262	if problem.is_none()
1263	&& let Some(group) = &group
1264	&& !groups.iter().any(\|(known, _)\| known == group)
1265	{
1266	groups.push((group.clone(), cancel));
1267	}
1268	let call = json!({
1269	"role": "caller", "path": file, "source": source, "origin": origin, "depth": depth,
1270	"prefix": call_prefix(&row.key, job.matrix.is_some().then_some(index)), "title": name,
1271	"inputs": inputs.unwrap_or_default(), "secrets": secrets, "fail_fast": job.fail_fast,
1272	});
1273	let (status, conclusion, finished) = match &problem {
1274	Some(_) => ("completed", Some("failure"), Some(now())),
1275	None => ("waiting", None, None),
1276	};
1277	let values: Vec<worker::wasm_bindgen::JsValue> = vec![
1278	name.as_str().into(),
1279	serde_json::to_string(combination)?.into(),
1280	status.into(),
1281	optional(conclusion),
1282	optional(problem.as_deref()),
1283	job.max_parallel.map(\|max\| max.max(1)).map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1284	optional(finished.as_deref()),
1285	optional(group.as_deref()),
1286	u32::from(cancel).into(),
1287	serde_json::to_string(&call)?.into(),
1288	];
1289	if index == 0 {
1290	let mut bound = values;
1291	bound.push(row.id.as_str().into());
1292	statements.push(
1293	self.db
1294	.prepare(
1295	"UPDATE jobs SET name = ?, matrix = ?, status = ?, conclusion = ?, reason = ?, max_parallel = ?, finished_at = ?,
1296	concurrency_group = ?, cancel_in_progress = ?, call = ? WHERE id = ?",
1297	)
1298	.bind(&bound)?,
1299	);
1300	} else {
1301	let mut bound: Vec<worker::wasm_bindgen::JsValue> = vec![
1302	new_id("job", now_ms()).into(),
1303	row.run_id.as_str().into(),
1304	row.repo_id.as_str().into(),
1305	row.namespace.as_str().into(),
1306	row.key.as_str().into(),
1307	(index as u32).into(),
1308	row.needs.as_str().into(),
1309	];
1310	bound.extend(values);
1311	statements.push(
1312	self.db
1313	.prepare(
1314	"INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, name, matrix, status, conclusion, reason,
1315	max_parallel, finished_at, concurrency_group, cancel_in_progress, call)
1316	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1317	)
1318	.bind(&bound)?,
1319	);
1320	}
1321	}
1322	self.db.batch(statements).await?;
1323	self.replace_in_groups(run, &row.key, &groups).await
1324	}
1325
1326	/// Whether another job of `row`'s concurrency group runs, or calls a
1327	/// workflow, so that `row` waits for it.
1328	async fn group_busy(&self, row: &JobRow) -> Result<bool> {
1329	let Some(group) = &row.concurrency_group else { return Ok(false) };
1330	let busy = self
1331	.db
1332	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('in_progress', 'calling') AND id != ?")
1333	.bind(&[row.repo_id.as_str().into(), group.as_str().into(), row.id.as_str().into()])?
1334	.first::<Count>(None)
1335	.await?
1336	.map_or(0, \|count\| count.n);
1337	Ok(busy > 0)
1338	}
1339
1340	/// A call that waited (`holds_call`) is made: the called workflow's
1341	/// jobs join the run under it, named under its name and keyed under its
1342	/// prefix. False when it no longer waits.
1343	async fn start_call(&self, row: &JobRow) -> Result<bool> {
1344	let Some(call) = row.call() else { return Ok(false) };
1345	let Some(prefix) = row.call_prefix() else { return Ok(false) };
1346	let Some(called) = call["source"].as_str().and_then(\|source\| workflow::parse(source).ok()) else { return Ok(false) };
1347	let file = call["path"].as_str().unwrap_or_default();
1348	let started = self
1349	.db
1350	.prepare("UPDATE jobs SET status = 'calling', reason = ?, started_at = ? WHERE id = ? AND status = 'waiting' RETURNING id")
1351	.bind(&[format!("Calls `{file}`.").into(), now().into(), row.id.as_str().into()])?
1352	.first::<Value>(None)
1353	.await?;
1354	if started.is_none() {
1355	return Ok(false);
1356	}
1357	let mut statements = Vec::new();
1358	for called_job in &called.jobs {
1359	let needs: Vec<String> = called_job.needs.iter().map(\|n\| format!("{prefix}/{n}")).collect();
1360	let callee = json!({
1361	"role": "callee", "parent": prefix, "title": row.name, "job": called_job.id, "path": file,
1362	"source": call["source"], "inputs": call["inputs"], "depth": call["depth"], "origin": call["origin"], "secrets": call["secrets"],
1363	});
1364	statements.push(
1365	self.db
1366	.prepare("INSERT INTO jobs (id, run_id, repo_id, namespace, key, name, needs, status, call) VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting', ?)")
1367	.bind(&[
1368	new_id("job", now_ms()).into(),
1369	row.run_id.as_str().into(),
1370	row.repo_id.as_str().into(),
1371	row.namespace.as_str().into(),
1372	format!("{prefix}/{}", called_job.id).into(),
1373	format!("{} / {}", row.name, called_job.name.clone().unwrap_or(called_job.id.clone())).into(),
1374	serde_json::to_string(&needs)?.into(),
1375	serde_json::to_string(&callee)?.into(),
1376	])?,
1377	);
1378	}
1379	if !statements.is_empty() {
1380	self.db.batch(statements).await?;
1381	}
1382	Ok(true)
1383	}
1384
1385	/// The runs with a call waiting in `group`, but `run_id`, moved along
1386	/// now that a job of the group has finished.
1387	async fn wake_group(&self, repo_id: &str, group: &str, run_id: &str) -> Result<()> {
1388	#[derive(Deserialize)]
1389	struct RunId {
1390	run_id: String,
1391	}
1392	let waiting = self
1393	.db
1394	.prepare("SELECT DISTINCT run_id FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status = 'waiting' AND run_id != ?")
1395	.bind(&[repo_id.into(), group.into(), run_id.into()])?
1396	.all()
1397	.await?
1398	.results::<RunId>()?;
1399	for other in waiting {
1400	Box::pin(self.advance(&other.run_id)).await?;
1401	}
1402	Ok(())
1403	}
1404
1405	/// A job that called a workflow, finished with its jobs: their result,
1406	/// and the outputs the workflow declares. A failed call of a fail-fast
1407	/// matrix stops the matrix's other calls.
1408	async fn finish_call(&self, run: &RunRow, row: &JobRow, children: &[&JobRow]) -> Result<()> {
1409	let call = row.call().unwrap_or_default();
1410	let called = call["source"].as_str().and_then(\|s\| workflow::parse(s).ok());
1411	let mut jobs_context = Map::new();
1412	let mut by_key: std::collections::BTreeMap<String, Vec<&JobRow>> = std::collections::BTreeMap::new();
1413	for child in children {
1414	by_key.entry(child.key.clone()).or_default().push(child);
1415	}
1416	for (key, rows) in &by_key {
1417	let mut outputs = Map::new();
1418	for child in rows {
1419	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&child.outputs) {
1420	outputs.extend(more);
1421	}
1422	}
1423	let id = key.rsplit('/').next().unwrap_or(key);
1424	jobs_context.insert(id.to_owned(), json!({ "result": key_result(rows), "outputs": outputs }));
1425	}
1426	let inputs = children.first().and_then(\|c\| c.call()).map(\|c\| c["inputs"].clone()).unwrap_or(json!({}));
1427	let mut contexts = Map::new();
1428	contexts.insert("jobs".into(), Value::Object(jobs_context));
1429	contexts.insert("inputs".into(), inputs);
1430	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
1431	let mut outputs = Map::new();
1432	if let Some(Value::Object(declared)) = called.as_ref().map(\|w\| {
1433	let on = w.raw.get("on").or_else(\|\| w.raw.get("true")).cloned().unwrap_or(Value::Null);
1434	on.get("workflow_call").and_then(\|c\| c.get("outputs")).cloned().unwrap_or(Value::Null)
1435	}) {
1436	for (name, spec) in declared {
1437	if let Some(value) = spec.get("value") {
1438	let value = expr::interpolate_value(value, &scope).unwrap_or(Value::Null);
1439	outputs.insert(name, Value::String(expr::to_text(&value)));
1440	}
1441	}
1442	}
1443	let conclusion = key_result(children);
1444	let finished = self
1445	.db
1446	.prepare("UPDATE jobs SET status = 'completed', conclusion = ?, outputs = ?, finished_at = ? WHERE id = ? AND status = 'calling' RETURNING id")
1447	.bind(&[conclusion.into(), serde_json::to_string(&outputs)?.into(), now().into(), row.id.as_str().into()])?
1448	.first::<Value>(None)
1449	.await?;
1450	if finished.is_none() {
1451	return Ok(());
1452	}
1453	if conclusion == "failure" && row.matrix.as_deref().is_some_and(\|m\| m != "{}") && call["fail_fast"] != json!(false) {
1454	let siblings = self
1455	.db
1456	.prepare("SELECT * FROM jobs WHERE run_id = ? AND key = ? AND status != 'completed'")
1457	.bind(&[row.run_id.as_str().into(), row.key.as_str().into()])?
1458	.all()
1459	.await?
1460	.results::<JobRow>()?;
1461	for sibling in siblings {
1462	self.stop_job(&sibling, "Another job of its matrix failed, and the matrix is fail-fast.").await?;
1463	}
1464	}
1465	if let Some(group) = &row.concurrency_group {
1466	self.wake_group(&run.repo_id, group, &run.id).await?;
1467	}
1468	Ok(())
1469	}
1470
1471	/// A file's text at a commit, if it is there.
1472	pub(crate) async fn read_file(&self, path: &RepoPath, ws: &g1t_contracts::User, sha: &str, file: &str) -> Result<Option<String>> {
1473	let blob: Outcome<g1t_contracts::repos::BlobView> = g1t_kit::call(
1474	&self.repos,
1475	"blob",
1476	&g1t_contracts::repos::BlobArgs {
1477	path: path.clone(),
1478	viewer: Some(ws.clone()),
1479	git_ref: sha.to_owned(),
1480	file_path: file.to_owned(),
1481	},
1482	)
1483	.await?;
1484	Ok(match blob {
1485	Outcome::Ok(view) => view.text,
1486	Outcome::Fail(_) => None,
1487	})
1488	}
1489
1490	async fn skip_job(&self, row: &JobRow, reason: Option<&str>) -> Result<()> {
1491	self.db
1492	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'skipped', reason = ?, finished_at = ? WHERE id = ?")
1493	.bind(&[optional(reason), now().into(), row.id.as_str().into()])?
1494	.run()
1495	.await?;
1496	Ok(())
1497	}
1498
1499	async fn fail_job(&self, row: &JobRow, reason: &str) -> Result<()> {
1500	self.db
1501	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'failure', reason = ?, finished_at = ? WHERE id = ? AND status != 'completed'")
1502	.bind(&[reason.into(), now().into(), row.id.as_str().into()])?
1503	.run()
1504	.await?;
1505	Ok(())
1506	}
1507
1508	/// Starts queued jobs, oldest first, while their workspace has room.
1509	pub async fn start_queued(&self) -> Result<()> {
1510	let queued = self
1511	.db
1512	// Self-hosted jobs are taken by their runners (runners.rs).
1513	.prepare("SELECT * FROM jobs WHERE status = 'queued' AND labels IS NULL ORDER BY rowid LIMIT 50")
1514	.all()
1515	.await?
1516	.results::<JobRow>()?;
1517	let mut running: std::collections::HashMap<String, u32> = std::collections::HashMap::new();
1518	for job in queued {
1519	let in_workspace = match running.get(&job.namespace) {
1520	Some(n) => *n,
1521	None => {
1522	let n = self
1523	.db
1524	// Only g1t's own sandboxes count against the workspace's room.
1525	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE status = 'in_progress' AND namespace = ? AND runner_id IS NULL")
1526	.bind(&[job.namespace.as_str().into()])?
1527	.first::<Count>(None)
1528	.await?
1529	.map_or(0, \|count\| count.n);
1530	running.insert(job.namespace.clone(), n);
1531	n
1532	}
1533	};
1534	if in_workspace >= RUNNING_PER_WORKSPACE {
1535	continue;
1536	}
1537	if let Some(max) = job.max_parallel {
1538	let siblings = self
1539	.db
1540	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE run_id = ? AND key = ? AND status = 'in_progress'")
1541	.bind(&[job.run_id.as_str().into(), job.key.as_str().into()])?
1542	.first::<Count>(None)
1543	.await?
1544	.map_or(0, \|count\| count.n);
1545	if siblings >= max {
1546	continue;
1547	}
1548	}
1549	// One job of a concurrency group runs at a time.
1550	if let Some(group) = &job.concurrency_group {
1551	let running = self
1552	.db
1553	.prepare("SELECT COUNT(*) AS n FROM jobs WHERE repo_id = ? AND concurrency_group = ? AND status IN ('in_progress', 'calling') AND id != ?")
1554	.bind(&[job.repo_id.as_str().into(), group.as_str().into(), job.id.as_str().into()])?
1555	.first::<Count>(None)
1556	.await?
1557	.map_or(0, \|count\| count.n);
1558	if running > 0 {
1559	continue;
1560	}
1561	}
1562	let token = random_hex(24);
1563	let at = now();
1564	let claimed = self
1565	.db
1566	.prepare(
1567	"UPDATE jobs SET status = 'in_progress', token_hash = ?, started_at = ?, seen_at = ? WHERE id = ? AND status = 'queued' RETURNING id",
1568	)
1569	.bind(&[sha256_hex(&token).into(), at.as_str().into(), at.as_str().into(), job.id.as_str().into()])?
1570	.first::<Value>(None)
1571	.await?;
1572	if claimed.is_none() {
1573	continue;
1574	}
1575	running.insert(job.namespace.clone(), in_workspace + 1);
1576	self.db
1577	.prepare("UPDATE runs SET status = 'in_progress', started_at = COALESCE(started_at, ?) WHERE id = ? AND status = 'queued'")
1578	.bind(&[at.as_str().into(), job.run_id.as_str().into()])?
1579	.run()
1580	.await?;
1581	let run = self.run_row(&job.run_id).await?;
1582	let repo: RepoPath = run.as_ref().map(\|run\| repo_path(&run.repo)).unwrap_or(RepoPath {
1583	namespace: job.namespace.clone(),
1584	name: String::new(),
1585	});
1586	// Its environment and the machine it asked for, for the runner.
1587	let details = run.as_ref().map(\|run\| start_details(run, &job)).unwrap_or_default();
1588	let started: Outcome<Value> = g1t_kit::call(
1589	&self.runner,
1590	"start_actions_job",
1591	&StartJobArgs {
1592	job: job.id.clone(),
1593	token,
1594	repo,
1595	timeout_minutes: job.timeout_minutes,
1596	workflow: run.as_ref().map(\|run\| run.path.clone()),
1597	environment: details.environment,
1598	trusted: run.as_ref().is_some_and(\|run\| run.trusted != 0),
1599	instance: details.instance,
1600	},
1601	)
1602	.await
1603	.unwrap_or_else(\|error\| fail(FailureCode::Conflict, format!("The runner could not be reached: {error}")));
1604	if let Outcome::Fail(refused) = started {
1605	// Billing would not pay for a scheduled run's job: the
1606	// schedule waits rather than making runs only to refuse them.
1607	if refused.code == FailureCode::PaymentRequired
1608	&& let Some(run) = run.as_ref().filter(\|run\| run.event == "schedule")
1609	{
1610	self.db
1611	.prepare("UPDATE workflows SET schedule_refused_until = ? WHERE id = ?")
1612	.bind(&[rfc3339(now_ms() + crate::SCHEDULE_REFUSED_MS).into(), run.workflow_id.as_str().into()])?
1613	.run()
1614	.await?;
1615	}
1616	Box::pin(self.finish_job(&job.id, "failure", Some(&refused.message), None)).await?;
1617	} else {
1618	self.job_started(&job.id).await?;
1619	}
1620	}
1621	Ok(())
1622	}
1623
1624	/// Finishes a job and moves its run along.
1625	pub async fn finish_job(&self, job_id: &str, conclusion: &str, reason: Option<&str>, outputs: Option<&Map<String, Value>>) -> Result<()> {
1626	let finished = self
1627	.db
1628	.prepare(
1629	"UPDATE jobs SET status = 'completed', conclusion = ?, reason = COALESCE(?, reason), outputs = COALESCE(?, outputs),
1630	finished_at = ?, token_hash = NULL WHERE id = ? AND status != 'completed' RETURNING *",
1631	)
1632	.bind(&[
1633	conclusion.into(),
1634	optional(reason),
1635	outputs.map(\|o\| serde_json::to_string(o).unwrap_or_default()).as_deref().map_or(worker::wasm_bindgen::JsValue::NULL, Into::into),
1636	now().into(),
1637	job_id.into(),
1638	])?
1639	.first::<JobRow>(None)
1640	.await?;
1641	let Some(job) = finished else { return Ok(()) };
1642	// Its G1T_TOKEN stops working with it.
1643	self.revoke_job_tokens(&job.id).await;
1644	// A job that deploys failed: so did its run's deployment, now.
1645	if conclusion == "failure" && job.continue_on_error == 0 && job.started_at.is_some()
1646	&& let Some(run) = self.run_row(&job.run_id).await?
1647	&& let Some(env) = deploys_to(&run, &job)
1648	{
1649	self.report_deployment(&run, &env, "failure", false).await;
1650	}
1651	// A self-hosted runner's job: the runner is free again, and its time
1652	// is recorded, at nothing.
1653	if job.runner_id.is_some() {
1654	self.released(&job).await?;
1655	}
1656	// Steps still marked as going are not going any more.
1657	let mut steps: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
1658	let mut touched = false;
1659	for step in steps.iter_mut() {
1660	if step["status"] != "completed" {
1661	let was_running = step["status"] == "in_progress";
1662	step["status"] = json!("completed");
1663	step["conclusion"] = json!(if was_running { conclusion } else { "skipped" });
1664	touched = true;
1665	}
1666	}
1667	if touched {
1668	self.db
1669	.prepare("UPDATE jobs SET steps = ? WHERE id = ?")
1670	.bind(&[serde_json::to_string(&steps)?.into(), job.id.as_str().into()])?
1671	.run()
1672	.await?;
1673	}
1674	// fail-fast: one failed combination stops the rest of its matrix.
1675	if conclusion == "failure" && job.continue_on_error == 0 && job.matrix.as_deref().is_some_and(\|m\| m != "{}") {
1676	let run = self.run_row(&job.run_id).await?;
1677	// As its workflow, or the workflow it is a job of, says.
1678	let fail_fast = run.as_ref().and_then(\|run\| job_spec(run, &job)).is_none_or(\|j\| j.fail_fast);
1679	if fail_fast {
1680	let siblings = self
1681	.db
1682	.prepare("SELECT * FROM jobs WHERE run_id = ? AND key = ? AND status != 'completed'")
1683	.bind(&[job.run_id.as_str().into(), job.key.as_str().into()])?
1684	.all()
1685	.await?
1686	.results::<JobRow>()?;
1687	for sibling in siblings {
1688	self.stop_job(&sibling, "Another job of its matrix failed, and the matrix is fail-fast.").await?;
1689	}
1690	}
1691	}
1692	// A call waiting in its concurrency group, in another run, may start.
1693	if let Some(group) = &job.concurrency_group {
1694	self.wake_group(&job.repo_id, group, &job.run_id).await?;
1695	}
1696	Box::pin(self.advance(&job.run_id)).await
1697	}
1698
1699	/// Cancels a job. One that is running is told to stop (the answer to
1700	/// its next report), and runs its `if: always()` and `cancelled()`
1701	/// steps and its post steps before it ends `cancelled`; one that has
1702	/// not started is cancelled at once.
1703	async fn stop_job(&self, job: &JobRow, reason: &str) -> Result<()> {
1704	// A job calling a workflow stops with that workflow's jobs.
1705	if job.status == "calling"
1706	&& let Some(prefix) = job.call_prefix()
1707	{
1708	let under = format!("{prefix}/");
1709	for child in self.job_rows(&job.run_id).await?.iter().filter(\|child\| child.key.starts_with(&under) && child.status != "completed") {
1710	Box::pin(self.stop_job(child, reason)).await?;
1711	}
1712	}
1713	if job.status == "in_progress" && job.started_at.is_some() {
1714	self.db
1715	.prepare("UPDATE jobs SET cancel_requested_at = COALESCE(cancel_requested_at, ?), reason = ? WHERE id = ? AND status = 'in_progress'")
1716	.bind(&[now().into(), reason.into(), job.id.as_str().into()])?
1717	.run()
1718	.await?;
1719	return Ok(());
1720	}
1721	self.hard_stop(job, reason).await
1722	}
1723
1724	/// Cancels a job outright, stopping its sandbox if it has one.
1725	async fn hard_stop(&self, job: &JobRow, reason: &str) -> Result<()> {
1726	// A self-hosted runner hears it was cancelled on its next poll.
1727	if job.status == "in_progress" && job.runner_id.is_none() {
1728	let _: Result<Value> = g1t_kit::call(&self.runner, "stop_actions_job", &json!({ "job": job.id })).await;
1729	}
1730	let stopped = self
1731	.db
1732	.prepare("UPDATE jobs SET status = 'completed', conclusion = 'cancelled', reason = ?, finished_at = ?, token_hash = NULL WHERE id = ? AND status != 'completed' RETURNING *")
1733	.bind(&[reason.into(), now().into(), job.id.as_str().into()])?
1734	.first::<JobRow>(None)
1735	.await?;
1736	if stopped.as_ref().is_some_and(\|row\| row.started_at.is_some()) {
1737	self.revoke_job_tokens(&job.id).await;
1738	}
1739	if let Some(stopped) = stopped.filter(\|row\| row.runner_id.is_some()) {
1740	self.released(&stopped).await?;
1741	}
1742	Ok(())
1743	}
1744
1745	/// Ends a job's tokens at once. A failure is logged: the token expires
1746	/// on its own soon after the job's time limit.
1747	async fn revoke_job_tokens(&self, job_id: &str) {
1748	let revoked: Result<bool> =
1749	g1t_kit::call(&self.identity, "revoke_job_tokens", &RevokeJobTokensArgs { job_id: job_id.to_owned() }).await;
1750	if let Err(error) = revoked {
1751	worker::console_error!("actions: the tokens of job {job_id} were not revoked: {error}");
1752	}
1753	}
1754
1755	/// Finishes the run when every job has.
1756	async fn finish_if_done(&self, run_id: &str) -> Result<()> {
1757	let Some(run) = self.run_row(run_id).await? else { return Ok(()) };
1758	if matches!(run.status.as_str(), "completed" \| "pending" \| "action_required") {
1759	return Ok(());
1760	}
1761	let jobs = self.job_rows(run_id).await?;
1762	if !jobs.iter().all(\|job\| job.status == "completed") {
1763	return Ok(());
1764	}
1765	self.finish_run(&run, None).await
1766	}
1767
1768	async fn finish_run(&self, run: &RunRow, error: Option<&str>) -> Result<()> {
1769	let jobs = self.job_rows(&run.id).await?;
1770	let rows: Vec<&JobRow> = jobs.iter().collect();
1771	let conclusion = if error.is_some() {
1772	"failure"
1773	} else if run.conclusion.as_deref() == Some("cancelled") {
1774	"cancelled"
1775	} else if rows.is_empty() {
1776	"skipped"
1777	} else {
1778	key_result(&rows)
1779	};
1780	let done = self
1781	.db
1782	.prepare("UPDATE runs SET status = 'completed', conclusion = ?, error = COALESCE(?, error), finished_at = ? WHERE id = ? AND status != 'completed' RETURNING id")
1783	.bind(&[conclusion.into(), optional(error), now().into(), run.id.as_str().into()])?
1784	.first::<Value>(None)
1785	.await?;
1786	if done.is_none() {
1787	return Ok(());
1788	}
1789	self.report_status(run, conclusion).await?;
1790	self.settle_deployments(run, &jobs, conclusion == "cancelled").await;
1791	let published: Result<()> = g1t_kit::call(
1792	&self.events,
1793	"publish",
1794	&g1t_contracts::events::Publish {
1795	events: vec![g1t_contracts::events::NewEvent {
1796	kind: "workflow.completed",
1797	source: "actions",
1798	repo_id: Some(run.repo_id.clone()),
1799	actor: run.actor_id.clone(),
1800	data: g1t_contracts::events::WorkflowEvent {
1801	run_id: run.id.clone(),
1802	repo_id: run.repo_id.clone(),
1803	workflow: run.name.clone(),
1804	path: run.path.clone(),
1805	number: run.number,
1806	event: run.event.clone(),
1807	conclusion: conclusion.to_owned(),
1808	git_ref: run.git_ref.clone(),
1809	sha: run.sha.clone(),
1810	pull: run.pull,
1811	},
1812	}],
1813	},
1814	)
1815	.await;
1816	if let Err(error) = published {
1817	worker::console_error!("actions: could not publish workflow.completed: {error}");
1818	}
1819	// The next run waiting in its concurrency group.
1820	if let Some(group) = &run.concurrency_group {
1821	let next = self
1822	.db
1823	.prepare("SELECT * FROM runs WHERE repo_id = ? AND concurrency_group = ? AND status = 'pending' ORDER BY id LIMIT 1")
1824	.bind(&[run.repo_id.as_str().into(), group.as_str().into()])?
1825	.first::<RunRow>(None)
1826	.await?;
1827	if let Some(next) = next {
1828	self.db.prepare("UPDATE runs SET status = 'queued' WHERE id = ?").bind(&[next.id.as_str().into()])?.run().await?;
1829	Box::pin(self.advance(&next.id)).await?;
1830	}
1831	}
1832	Ok(())
1833	}
1834
1835	/// Tells the deployments service how a run's deployment to `env` stands:
1836	/// made when its first job naming the environment starts, failed when one
1837	/// of them fails, and settled (`last`) when the run finishes. Never fails
1838	/// the run: a deployment that could not be recorded is logged.
1839	async fn report_deployment(&self, run: &RunRow, env: &JobEnvironment, state: &str, last: bool) {
1840	let path = repo_path(&run.repo);
1841	let reported: Result<Value> = g1t_kit::call(
1842	&self.deployments,
1843	"actions_deployment",
1844	&json!({
1845	"repoId": run.repo_id,
1846	"repo": { "namespace": path.namespace, "name": path.name },
1847	"runId": run.id,
1848	"attempt": run.attempt,
1849	"runUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1850	"environment": env.name,
1851	"url": env.url,
1852	"ref": run.git_ref,
1853	"sha": run.sha,
1854	"state": state,
1855	"final": last,
1856	"creator": run.actor,
1857	"workflow": run.name,
1858	}),
1859	)
1860	.await;
1861	if let Err(error) = reported {
1862	worker::console_error!("actions: deployment to {} not recorded for run {}: {error}", env.name, run.id);
1863	}
1864	}
1865
1866	/// A job that deploys has started: its run's deployment to the
1867	/// environment is under way.
1868	pub(crate) async fn job_started(&self, job_id: &str) -> Result<()> {
1869	let Some(job) = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[job_id.into()])?.first::<JobRow>(None).await? else {
1870	return Ok(());
1871	};
1872	let Some(run) = self.run_row(&job.run_id).await? else { return Ok(()) };
1873	if let Some(env) = deploys_to(&run, &job) {
1874	self.report_deployment(&run, &env, "in_progress", false).await;
1875	}
1876	Ok(())
1877	}
1878
1879	/// The run is over: each environment its jobs deployed to takes the
1880	/// outcome of those jobs (`deployment_outcome`).
1881	async fn settle_deployments(&self, run: &RunRow, jobs: &[JobRow], cancelled: bool) {
1882	let mut seen: Vec<(JobEnvironment, Vec<Option<String>>)> = Vec::new();
1883	for job in jobs {
1884	let Some(env) = deploys_to(run, job) else { continue };
1885	let conclusion = if cancelled && job.conclusion.as_deref() != Some("skipped") && job.started_at.is_some() {
1886	Some("cancelled".to_owned())
1887	} else if job.started_at.is_none() {
1888	Some("skipped".to_owned())
1889	} else {
1890	job.conclusion.clone()
1891	};
1892	match seen.iter_mut().find(\|(known, _)\| known.name.eq_ignore_ascii_case(&env.name)) {
1893	Some((known, conclusions)) => {
1894	if known.url.is_none() {
1895	known.url = env.url.clone();
1896	}
1897	conclusions.push(conclusion);
1898	}
1899	None => seen.push((env, vec![conclusion])),
1900	}
1901	}
1902	for (env, conclusions) in seen {
1903	if let Some(state) = deployment_outcome(&conclusions) {
1904	self.report_deployment(run, &env, state, true).await;
1905	}
1906	}
1907	}
1908
1909	/// Tells the pull request (or commit) how the run went, as a status.
1910	async fn report_status(&self, run: &RunRow, conclusion: &str) -> Result<()> {
1911	let state = match conclusion {
1912	"success" \| "skipped" => "success",
1913	"cancelled" => "error",
1914	_ => "failure",
1915	};
1916	let _: Result<Value> = g1t_kit::call(
1917	&self.work,
1918	"set_commit_status",
1919	&json!({
1920	"repoId": run.repo_id,
1921	"sha": run.sha,
1922	"context": format!("{} / {}", run.name, run.event),
1923	"state": state,
1924	"description": format!("{} {}", run.name, match conclusion {
1925	"success" => "passed",
1926	"skipped" => "was skipped",
1927	"cancelled" => "was cancelled",
1928	_ => "failed",
1929	}),
1930	"targetUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1931	"source": "actions",
1932	}),
1933	)
1934	.await;
1935	Ok(())
1936	}
1937
1938	/// Tells the pull request a run has started on its head.
1939	pub async fn report_pending(&self, run: &RunRow) -> Result<()> {
1940	let _: Result<Value> = g1t_kit::call(
1941	&self.work,
1942	"set_commit_status",
1943	&json!({
1944	"repoId": run.repo_id,
1945	"sha": run.sha,
1946	"context": format!("{} / {}", run.name, run.event),
1947	"state": "pending",
1948	"description": if run.status == "action_required" {
1949	format!("{} is waiting for approval", run.name)
1950	} else {
1951	format!("{} is running", run.name)
1952	},
1953	"targetUrl": format!("{SITE}/{}/actions/runs/{}", run.repo, run.id),
1954	"source": "actions",
1955	}),
1956	)
1957	.await;
1958	Ok(())
1959	}
1960
1961	/// Cancels a run: its waiting and queued jobs at once, and its running
1962	/// ones gracefully (`stop_job`), or outright when `force`.
1963	pub async fn cancel_run(&self, run: &RunRow, reason: &str, force: bool) -> Result<()> {
1964	self.db
1965	.prepare("UPDATE runs SET conclusion = 'cancelled' WHERE id = ? AND status != 'completed'")
1966	.bind(&[run.id.as_str().into()])?
1967	.run()
1968	.await?;
1969	for job in self.job_rows(&run.id).await?.iter().filter(\|job\| job.status != "completed") {
1970	if force {
1971	self.hard_stop(job, reason).await?;
1972	} else {
1973	self.stop_job(job, reason).await?;
1974	}
1975	}
1976	if run.status == "pending" \|\| run.status == "action_required" {
1977	self.db.prepare("UPDATE runs SET status = 'queued' WHERE id = ?").bind(&[run.id.as_str().into()])?.run().await?;
1978	}
1979	self.advance(&run.id).await
1980	}
1981
1982	/// Cancels every run of the repository that has not finished, for
1983	/// `repo.deleted` and `repo.archived`.
1984	pub async fn stop_runs(&self, repo_id: &str) -> Result<()> {
1985	let runs = self
1986	.db
1987	.prepare("SELECT * FROM runs WHERE repo_id = ? AND status != 'completed'")
1988	.bind(&[repo_id.into()])?
1989	.all()
1990	.await?
1991	.results::<RunRow>()?;
1992	for run in runs {
1993	self.cancel_run(&run, "The repository was archived or deleted.", true).await?;
1994	}
1995	Ok(())
1996	}
1997
1998	pub async fn cancel(&self, a: RunActionArgs) -> Result<Outcome<WorkflowRun>> {
1999	if let Outcome::Fail(refused) = self.may(&a.actor, &a.repo, Capability::Run).await? {
2000	return Ok(Outcome::Fail(refused));
2001	}
2002	let run = check!(self.run_in(&a.repo, &a.id).await?);
2003	if run.status == "completed" {
2004	return Ok(fail(FailureCode::Conflict, "The run has already finished."));
2005	}
2006	// Cancelling a run that is already cancelling stops its jobs
2007	// outright, without waiting for their cleanup steps.
2008	let force = a.force \|\| run.conclusion.as_deref() == Some("cancelled");
2009	let reason = if force {
2010	format!("{} stopped the run without waiting for its cleanup steps.", a.actor.username)
2011	} else {
2012	format!("{} cancelled the run.", a.actor.username)
2013	};
2014	self.cancel_run(&run, &reason, force).await?;
2015	self.run_summary(&run.id).await
2016	}
2017
2018	/// Runs again, as a new attempt: every job, with `failed_only` those
2019	/// that did not succeed, or with `job` that one; each with the jobs that
2020	/// need them. The attempt that ends is kept, its jobs and their logs and
2021	/// summaries with it (`job_attempts`, `run_attempts`).
2022	pub async fn rerun(&self, a: RunActionArgs) -> Result<Outcome<WorkflowRun>> {
2023	if let Outcome::Fail(refused) = self.may(&a.actor, &a.repo, Capability::Run).await? {
2024	return Ok(Outcome::Fail(refused));
2025	}
2026	// A job alone (`POST …/jobs/{job}/rerun`) names its run.
2027	let run_id = match (&a.job, a.id.is_empty()) {
2028	(Some(job), true) => {
2029	#[derive(Deserialize)]
2030	struct Of {
2031	run_id: String,
2032	}
2033	let of = self.db.prepare("SELECT run_id FROM jobs WHERE id = ?").bind(&[job.as_str().into()])?.first::<Of>(None).await?;
2034	match of {
2035	Some(of) => of.run_id,
2036	None => return Ok(fail(FailureCode::NotFound, "No such job.")),
2037	}
2038	}
2039	_ => a.id.clone(),
2040	};
2041	let run = check!(self.run_in(&a.repo, &run_id).await?);
2042	if run.status != "completed" {
2043	return Ok(fail(FailureCode::Conflict, "The run is still going: cancel it first."));
2044	}
2045	// Nothing starts again on an archived repository.
2046	match self.visible_repo(&a.repo, &Some(a.actor.clone())).await? {
2047	Some(repo) if repo.archived() => {
2048	return Ok(fail(FailureCode::Forbidden, g1t_contracts::repos::archived_message(&repo.namespace, &repo.name)));
2049	}
2050	Some(_) => {}
2051	None => return Ok(fail(FailureCode::NotFound, "There is no such repository.")),
2052	}
2053	if run.error.is_some() {
2054	return Ok(fail(FailureCode::Conflict, "This run never started: fix the workflow file and push again."));
2055	}
2056	let jobs = self.job_rows(&run.id).await?;
2057	let workflow = workflow::parse(&run.source).ok();
2058	let target = match &a.job {
2059	Some(id) => match jobs.iter().find(\|job\| &job.id == id) {
2060	Some(job) => Some(top_key(&job.key).to_owned()),
2061	None => return Ok(fail(FailureCode::NotFound, "That job is not in the run's latest attempt.")),
2062	},
2063	None => None,
2064	};
2065	let which = match (&target, a.failed_only) {
2066	(Some(key), _) => Rerun::Job(key),
2067	(None, true) => Rerun::Failed,
2068	(None, false) => Rerun::All,
2069	};
2070	let order = workflow.as_ref().map(\|w\| w.job_order()).unwrap_or_default();
2071	let again = rerun_keys(
2072	&order,
2073	\|key\| jobs.iter().find(\|j\| j.key == key).map(JobRow::needs).unwrap_or_default(),
2074	\|key\| jobs.iter().filter(\|j\| j.key == key).all(\|j\| j.conclusion.as_deref() == Some("success")),
2075	which,
2076	);
2077	if again.is_empty() {
2078	return Ok(fail(FailureCode::Conflict, "Every job succeeded: there is nothing to run again."));
2079	}
2080	// The jobs that run again, a called workflow's with the job calling it.
2081	let rerun_ids: Vec<&str> = jobs.iter().filter(\|job\| again.iter().any(\|key\| key == top_key(&job.key))).map(\|job\| job.id.as_str()).collect();
2082	let ids = serde_json::to_string(&rerun_ids)?;
2083	let ended = run.attempt.to_string();
2084	let ended = ended.as_str();
2085	let mut statements = vec![
2086	// The attempt that ends, as it ended.
2087	self.db
2088	.prepare(
2089	"INSERT OR REPLACE INTO run_attempts (run_id, attempt, repo_id, conclusion, actor, debug, started_at, finished_at)
2090	SELECT id, attempt, repo_id, conclusion, COALESCE(triggering_actor, actor), debug, started_at, finished_at FROM runs WHERE id = ?",
2091	)
2092	.bind(&[run.id.as_str().into()])?,
2093	// Earlier attempts that showed a job's logs from where it ran
2094	// then now find them where they move to.
2095	self.db
2096	.prepare("UPDATE job_attempts SET log_id = log_id \|\| '.' \|\| ?1 WHERE run_id = ?2 AND log_id IN (SELECT value FROM json_each(?3))")
2097	.bind(&[ended.into(), run.id.as_str().into(), ids.as_str().into()])?,
2098	// Each of its jobs: one that runs again keeps its logs under
2099	// `{id}.{attempt}`; one left alone is still the live job's.
2100	self.db
2101	.prepare(
2102	"INSERT OR REPLACE INTO job_attempts (id, run_id, repo_id, attempt, job_id, log_id, key, ordinal, name, needs, status, conclusion,
2103	steps, annotations, reason, environment, labels, runner_name, started_at, finished_at)
2104	SELECT id \|\| '.' \|\| ?1, run_id, repo_id, ?1, id,
2105	CASE WHEN id IN (SELECT value FROM json_each(?3)) THEN id \|\| '.' \|\| ?1 ELSE id END,
2106	key, ordinal, name, needs, status, conclusion, steps, annotations, reason, environment, labels, runner_name, started_at, finished_at
2107	FROM jobs WHERE run_id = ?2 ORDER BY rowid",
2108	)
2109	.bind(&[ended.into(), run.id.as_str().into(), ids.as_str().into()])?,
2110	self.db
2111	.prepare("UPDATE logs SET job_id = job_id \|\| '.' \|\| ?1 WHERE job_id IN (SELECT value FROM json_each(?2))")
2112	.bind(&[ended.into(), ids.as_str().into()])?,
2113	self.db
2114	.prepare("UPDATE job_summaries SET job_id = job_id \|\| '.' \|\| ?1 WHERE job_id IN (SELECT value FROM json_each(?2))")
2115	.bind(&[ended.into(), ids.as_str().into()])?,
2116	];
2117	for key in &again {
2118	statements.push(self.db.prepare("DELETE FROM jobs WHERE run_id = ? AND key = ? AND ordinal > 0").bind(&[run.id.as_str().into(), key.as_str().into()])?);
2119	// The jobs of a workflow it called (each matrix combination's
2120	// under `key[i]/`) are made again when it calls it again.
2121	statements.push(
2122	self.db
2123	.prepare("DELETE FROM jobs WHERE run_id = ? AND (key LIKE ? OR key LIKE ?)")
2124	.bind(&[run.id.as_str().into(), format!("{key}/%").into(), format!("{key}[%").into()])?,
2125	);
2126	statements.push(
2127	self.db
2128	.prepare(
2129	"UPDATE jobs SET status = 'waiting', conclusion = NULL, steps = '[]', annotations = '[]', outputs = '{}', reason = NULL,
2130	matrix = NULL, call = NULL, token_hash = NULL, seen_at = NULL, started_at = NULL, finished_at = NULL,
2131	labels = NULL, queued_at = NULL, runner_id = NULL, runner_name = NULL, environment = NULL,
2132	concurrency_group = NULL, cancel_in_progress = 0, cancel_requested_at = NULL WHERE run_id = ? AND key = ?",
2133	)
2134	.bind(&[run.id.as_str().into(), key.as_str().into()])?,
2135	);
2136	}
2137	statements.push(
2138	self.db
2139	.prepare(
2140	"UPDATE runs SET status = 'queued', conclusion = NULL, attempt = attempt + 1, started_at = NULL, finished_at = NULL,
2141	triggering_actor = ?, debug = ? WHERE id = ?",
2142	)
2143	.bind(&[a.actor.username.as_str().into(), u32::from(a.debug).into(), run.id.as_str().into()])?,
2144	);
2145	self.db.batch(statements).await?;
2146	if let Some(run) = self.run_row(&run.id).await? {
2147	self.report_pending(&run).await?;
2148	}
2149	self.advance(&run.id).await?;
2150	self.run_summary(&run.id).await
2151	}
2152
2153	pub async fn run_in(&self, repo: &RepoPath, id: &str) -> Result<Outcome<RunRow>> {
2154	let row = self
2155	.db
2156	.prepare("SELECT * FROM runs WHERE id = ? AND lower(repo) = lower(?)")
2157	.bind(&[id.into(), format!("{}/{}", repo.namespace, repo.name).into()])?
2158	.first::<RunRow>(None)
2159	.await?;
2160	Ok(row.map_or_else(\|\| fail(FailureCode::NotFound, "No such run."), Outcome::Ok))
2161	}
2162
2163	// --- The sandbox's side -----------------------------------------------------
2164
2165	pub(crate) async fn job_for_token(&self, a: &JobCallArgs) -> Result<Outcome<JobRow>> {
2166	let job = self.db.prepare("SELECT * FROM jobs WHERE id = ?").bind(&[a.job.as_str().into()])?.first::<JobRow>(None).await?;
2167	Ok(match job {
2168	Some(job) if job.status == "in_progress" && job.token_hash.as_deref().is_some_and(\|hash\| same(hash, &sha256_hex(&a.token))) => {
2169	Outcome::Ok(job)
2170	}
2171	_ => fail(FailureCode::Unauthenticated, "That job is not running, or the token is not its."),
2172	})
2173	}
2174
2175	/// `job_auth`: which run and repository a running job's token is for,
2176	/// so the API can keep its artifacts and cache.
2177	pub async fn job_auth(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2178	let job = check!(self.job_for_token(&a).await?);
2179	Ok(Outcome::Ok(json!({ "run": job.run_id, "repoId": job.repo_id })))
2180	}
2181
2182	/// `job_spec`: everything the sandbox needs to run the job.
2183	pub async fn job_spec(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2184	let job = check!(self.job_for_token(&a).await?);
2185	let Some(run) = self.run_row(&job.run_id).await? else {
2186	return Ok(fail(FailureCode::NotFound, "No such run."));
2187	};
2188	let Ok(caller) = workflow::parse(&run.source) else {
2189	return Ok(fail(FailureCode::Invalid, "The workflow no longer reads."));
2190	};
2191	// A called workflow's job runs as that workflow defines it.
2192	let callee = job.callee();
2193	let (workflow, spec, call_inputs) = match callee {
2194	Some((called, spec, call)) => (called, spec, Some(call["inputs"].clone())),
2195	None => match caller.jobs.iter().find(\|j\| j.id == job.key) {
2196	Some(spec) => (caller.clone(), spec.clone(), None),
2197	None => return Ok(fail(FailureCode::NotFound, "The job is not in the workflow.")),
2198	},
2199	};
2200	let spec = &spec;
2201	let repo = repo_path(&run.repo);
2202	let trusted = run.trusted != 0;
2203	// The job's `environment:`, by name, as read when its needs were done
2204	// (an expression included), once the environment's protection rules
2205	// let it start: entries with a value for it give that value instead
2206	// of their default, as GitHub's environment secrets do.
2207	let environment: Option<String> = job.environment.clone();
2208	// What its token may do: its `permissions:` (a called workflow's
2209	// jobs no more than the job that calls it), else the repository's
2210	// default; read-only for a pull request from outside.
2211	// The repository's default, its workspace's taken in, and whether
2212	// its jobs may open and approve pull requests.
2213	let (default, pull_requests) = self.token_policy(&run.repo_id).await?;
2214	let mut permissions = spec.permissions(&workflow, default);
2215	if let Some(parent) = &job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c["parent"].as_str().map(str::to_owned)) {
2216	let top = top_key(parent);
2217	if let Some(caller_job) = caller.jobs.iter().find(\|j\| j.id == top) {
2218	permissions = permissions.capped_by(&caller_job.permissions(&caller, default));
2219	}
2220	}
2221	if !trusted {
2222	permissions = permissions.read_only();
2223	}
2224	// G1T_TOKEN, and GITHUB_TOKEN as its alias: a token of the
2225	// workspace's that reaches this repository only, with the scopes
2226	// its permissions give, until the job ends.
2227	let token = match self.workspace_actor(&repo.namespace).await? {
2228	Some(workspace) => {
2229	let created: CreatedAccessToken = g1t_kit::call(
2230	&self.identity,
2231	"create_job_token",
2232	&CreateJobTokenArgs {
2233	workspace,
2234	repo: repo.clone(),
2235	run_id: run.id.clone(),
2236	job_id: job.id.clone(),
2237	name: format!("G1T_TOKEN for {} run {}", run.repo, run.number),
2238	ttl_seconds: u64::from(job.timeout_minutes) * 60 + 600,
2239	scopes: permissions.scopes().into_iter().map(str::to_owned).collect(),
2240	pull_requests: pull_requests && trusted,
2241	},
2242	)
2243	.await?;
2244	created.token
2245	}
2246	None => String::new(),
2247	};
2248	// A run that is not trusted (a pull request from outside the
2249	// workspace) gets no secrets and an empty token.
2250	let passed = job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c.get("secrets").cloned());
2251	let mut secrets = match (trusted, passed) {
2252	(false, _) => Map::new(),
2253	(true, None) => self.secrets_for(&run.repo_id, &run.repo, environment.as_deref(), true).await?,
2254	// A called workflow's job: what its callers passed it (reach.rs),
2255	// and its own environment's secrets over them.
2256	(true, Some(plan)) => {
2257	let base = self.secrets_for(&run.repo_id, &run.repo, None, true).await?;
2258	let vars = self.variables_for(&run.repo_id, &run.repo, None, true).await?;
2259	let github = run.info().context(&job.key, "", run.action.as_deref());
2260	let mut passed = crate::reach::resolve_secrets(&plan, &base, &github, &vars);
2261	if let Some(name) = environment.as_deref() {
2262	let own = self.secrets_for(&run.repo_id, &run.repo, Some(name), true).await?;
2263	for (key, value) in own {
2264	if base.get(&key) != Some(&value) {
2265	passed.insert(key, value);
2266	}
2267	}
2268	}
2269	passed
2270	}
2271	};
2272	secrets.insert("G1T_TOKEN".into(), Value::String(token.clone()));
2273	secrets.insert("GITHUB_TOKEN".into(), Value::String(token.clone()));
2274	// Each secret as it is, a line at a time, base64 and JSON-escaped.
2275	let mut masks: Vec<String> = g1t_actions::mask::all_variants(secrets.values().filter_map(\|v\| v.as_str()));
2276	let vars = self.variables_for(&run.repo_id, &run.repo, environment.as_deref(), trusted).await?;
2277	// The toolkit's runtime token (runtime.rs), for as long as the job
2278	// may run; the API puts it and the toolkit's addresses in the
2279	// job's variables.
2280	let runtime_token = crate::runtime::runtime_token(
2281	&job.id,
2282	&job.run_id,
2283	job.token_hash.as_deref().unwrap_or_default(),
2284	now_ms() / 1000,
2285	u64::from(job.timeout_minutes) * 60 + 600,
2286	);
2287	masks.push(runtime_token.clone());
2288	let retention_days = self.retention_setting(&run.repo_id).await?;
2289
2290	let jobs = self.job_rows(&run.id).await?;
2291	let mut needs = Map::new();
2292	// In a called workflow, its jobs' keys sit under the job that called it.
2293	let parent = job.call().filter(\|c\| c["role"] == "callee").and_then(\|c\| c["parent"].as_str().map(str::to_owned));
2294	for need in &spec.needs {
2295	let key = match &parent {
2296	Some(parent) => format!("{parent}/{need}"),
2297	None => need.clone(),
2298	};
2299	let rows: Vec<&JobRow> = jobs.iter().filter(\|row\| row.key == key).collect();
2300	let mut outputs = Map::new();
2301	for row in &rows {
2302	if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&row.outputs) {
2303	outputs.extend(more);
2304	}
2305	}
2306	needs.insert(need.clone(), json!({ "result": key_result(&rows), "outputs": outputs }));
2307	}
2308	let siblings = jobs.iter().filter(\|row\| row.key == job.key).count();
2309	let matrix: Value = job.matrix.as_deref().and_then(\|m\| serde_json::from_str(m).ok()).unwrap_or(json!({}));
2310	let info = run.info();
2311	let mut github = info.context(&job.key, &token, run.action.as_deref());
2312	github["token"] = json!(token);
2313	github["retention_days"] = json!(retention_days);
2314	// On a self-hosted runner, `runner` and `RUNNER_*` describe that
2315	// machine rather than g1t's sandbox.
2316	let mut variables = info.variables(&job.key);
2317	variables.insert("GITHUB_RETENTION_DAYS".into(), json!(retention_days.to_string()));
2318	let mut runner = match &job.runner_id {
2319	Some(id) => self.runner_context_for(id, &mut variables).await?,
2320	None => runner_context(),
2321	};
2322	// A re-run with debug logging: what GitHub sets for one.
2323	if run.debug != 0 {
2324	debug_logging(&mut variables, &mut runner);
2325	}
2326
2327	// Where to check out: a pull request's fork, or the repository.
2328	let clone_url = match run.pull {
2329	Some(number) if run.event.starts_with("pull_request") && run.event != "pull_request_target" => {
2330	let located: Outcome<g1t_contracts::work::PullDetail> = g1t_kit::call(
2331	&self.work,
2332	"get_pull",
2333	&g1t_contracts::work::ViewArgs {
2334	repo: repo.clone(),
2335	number,
2336	viewer: self.workspace_actor(&repo.namespace).await?,
2337	after_seq: 0,
2338	},
2339	)
2340	.await?;
2341	match located {
2342	Outcome::Ok(detail) => match detail.pull.fork {
2343	Some(fork) => format!("{SITE}/{}/{}.git", fork.namespace, fork.name),
2344	None => format!("{SITE}/{}.git", run.repo),
2345	},
2346	Outcome::Fail(_) => format!("{SITE}/{}.git", run.repo),
2347	}
2348	}
2349	_ => format!("{SITE}/{}.git", run.repo),
2350	};
2351
2352	Ok(Outcome::Ok(json!({
2353	"job": job.id,
2354	"run": run.id,
2355	"key": job.key,
2356	"name": job.name,
2357	"spec": spec.raw,
2358	"workflow": {
2359	"env": workflow.env,
2360	"defaults": workflow.raw.get("defaults").cloned().unwrap_or(Value::Null),
2361	},
2362	"github": github,
2363	"variables": variables,
2364	"event": info.event,
2365	"contexts": {
2366	"vars": vars,
2367	"secrets": secrets,
2368	"inputs": call_inputs.unwrap_or_else(\|\| Value::Object(run.inputs())),
2369	"matrix": matrix,
2370	"needs": needs,
2371	"strategy": {
2372	"fail-fast": spec.fail_fast,
2373	"job-index": job.ordinal,
2374	"job-total": siblings,
2375	"max-parallel": spec.max_parallel.unwrap_or(siblings as u32),
2376	},
2377	"runner": runner,
2378	},
2379	"checkout": {
2380	"repository": run.repo,
2381	"url": clone_url,
2382	"sha": run.sha,
2383	"ref": run.git_ref,
2384	"token": token,
2385	},
2386	"timeoutMinutes": job.timeout_minutes,
2387	"masks": masks,
2388	// As the job's log lists them at its start.
2389	"permissions": permissions.listed().into_iter().map(\|(name, access)\| (name.to_owned(), json!(access.as_str()))).collect::<Map<String, Value>>(),
2390	// Whether the job may ask for an OIDC token decides whether it
2391	// is told where to.
2392	"runtime": {
2393	"token": runtime_token,
2394	"idToken": self.oidc_allowed(&run, &job),
2395	},
2396	})))
2397	}
2398
2399	/// `job_report`: the sandbox telling how the job is going.
2400	pub async fn job_report(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
2401	let job = check!(self.job_for_token(&a).await?);
2402	let report = &a.report;
2403	let at = now();
2404	match report["kind"].as_str().unwrap_or_default() {
2405	"steps" => {
2406	// The list can grow as the job goes (post steps), so steps
2407	// already reported keep where they stand.
2408	let known: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
2409	let steps: Vec<Value> = report["steps"]
2410	.as_array()
2411	.map(\|names\| {
2412	names
2413	.iter()
2414	.enumerate()
2415	.map(\|(i, name)\| match known.get(i) {
2416	Some(step) if step["status"] != "queued" => step.clone(),
2417	_ => json!({ "number": i + 1, "name": expr::to_text(name), "status": "queued", "conclusion": null, "startedAt": null, "finishedAt": null }),
2418	})
2419	.collect()
2420	})
2421	.unwrap_or_default();
2422	self.db
2423	.prepare("UPDATE jobs SET steps = ?, seen_at = ? WHERE id = ?")
2424	.bind(&[serde_json::to_string(&steps)?.into(), at.as_str().into(), job.id.as_str().into()])?
2425	.run()
2426	.await?;
2427	}
2428	"step" => {
2429	let number = report["number"].as_u64().unwrap_or(0) as usize;
2430	let mut steps: Vec<Value> = serde_json::from_str(&job.steps).unwrap_or_default();
2431	if let Some(step) = number.checked_sub(1).and_then(\|i\| steps.get_mut(i)) {
2432	let status = report["status"].as_str().unwrap_or("in_progress");
2433	step["status"] = json!(status);
2434	if status == "in_progress" {
2435	step["startedAt"] = json!(at);
2436	}
2437	if status == "completed" {
2438	step["finishedAt"] = json!(at);
2439	step["conclusion"] = report["conclusion"].clone();
2440	}
2441	if let Some(name) = report["name"].as_str() {
2442	step["name"] = json!(name);
2443	}
2444	}
2445	self.db
2446	.prepare("UPDATE jobs SET steps = ?, seen_at = ? WHERE id = ?")
2447	.bind(&[serde_json::to_string(&steps)?.into(), at.as_str().into(), job.id.as_str().into()])?
2448	.run()
2449	.await?;
2450	}
2451	"log" => {
2452	let mut text = report["text"].as_str().unwrap_or_default().to_owned();
2453	if text.len() > MAX_CHUNK_BYTES {
2454	let mut cut = MAX_CHUNK_BYTES;
2455	while !text.is_char_boundary(cut) {
2456	cut -= 1;
2457	}
2458	text.truncate(cut);
2459	}
2460	#[derive(Deserialize)]
2461	struct Size {
2462	n: Option<f64>,
2463	}
2464	let size = self
2465	.db
2466	.prepare("SELECT SUM(LENGTH(text)) AS n FROM logs WHERE job_id = ?")
2467	.bind(&[job.id.as_str().into()])?
2468	.first::<Size>(None)
2469	.await?;
2470	let used = size.and_then(\|s\| s.n).unwrap_or(0.0) as usize;
2471	if used < MAX_LOG_BYTES {
2472	if used + text.len() >= MAX_LOG_BYTES {
2473	text.push_str("\n… The log reached its limit of 4 MB; the rest is not kept.\n");
2474	}
2475	// The next `seq` is taken in the insert itself, so two
2476	// batches at once (a report sent again while the first
2477	// was still being written) never reach for the same one
2478	// and lose a chunk to the primary key.
2479	self.db
2480	.prepare(LOG_INSERT)
2481	.bind(&[
2482	job.id.as_str().into(),
2483	(report["step"].as_u64().unwrap_or(0) as u32).into(),
2484	text.into(),
2485	report["chunk"].as_u64().map_or(worker::wasm_bindgen::JsValue::NULL, \|n\| (n as f64).into()),
2486	])?
2487	.run()
2488	.await?;
2489	}
2490	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2491	}
2492	"summary" => {
2493	// `$GITHUB_STEP_SUMMARY`, masked by the runner: added to the
2494	// step's summary, up to 1 MiB a step and 20 steps a job.
2495	let step = report["step"].as_u64().unwrap_or(0) as u32;
2496	let markdown = report["markdown"].as_str().unwrap_or_default();
2497	#[derive(Deserialize)]
2498	struct Held {
2499	steps: u32,
2500	mine: Option<f64>,
2501	}
2502	let held = self
2503	.db
2504	.prepare("SELECT COUNT(*) AS steps, MAX(CASE WHEN step = ? THEN LENGTH(markdown) END) AS mine FROM job_summaries WHERE job_id = ?")
2505	.bind(&[step.into(), job.id.as_str().into()])?
2506	.first::<Held>(None)
2507	.await?;
2508	let (steps, mine) = held.map_or((0, None), \|held\| (held.steps, held.mine.map(\|n\| n as usize)));
2509	if summary_fits(steps, mine, markdown.len()) {
2510	self.db
2511	.prepare(
2512	"INSERT INTO job_summaries (job_id, step, markdown) VALUES (?, ?, ?)
2513	ON CONFLICT (job_id, step) DO UPDATE SET markdown = job_summaries.markdown \|\| excluded.markdown",
2514	)
2515	.bind(&[job.id.as_str().into(), step.into(), markdown.into()])?
2516	.run()
2517	.await?;
2518	}
2519	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2520	}
2521	"annotation" => {
2522	let mut annotations: Vec<Value> = serde_json::from_str(&job.annotations).unwrap_or_default();
2523	if annotations.len() < MAX_ANNOTATIONS {
2524	annotations.push(json!({
2525	"level": report["level"].as_str().unwrap_or("notice"),
2526	"message": report["message"].as_str().unwrap_or_default().chars().take(4000).collect::<String>(),
2527	"title": report["title"],
2528	"file": report["file"],
2529	"line": report["line"],
2530	}));
2531	self.db
2532	.prepare("UPDATE jobs SET annotations = ?, seen_at = ? WHERE id = ?")
2533	.bind(&[serde_json::to_string(&annotations)?.into(), at.as_str().into(), job.id.as_str().into()])?
2534	.run()
2535	.await?;
2536	}
2537	}
2538	// Nothing to tell; the answer says whether to stop.
2539	"ping" => {
2540	self.db.prepare("UPDATE jobs SET seen_at = ? WHERE id = ?").bind(&[at.into(), job.id.as_str().into()])?.run().await?;
2541	}
2542	"done" => {
2543	// A job told to stop ends cancelled, however its cleanup went.
2544	let conclusion = if job.cancel_requested_at.is_some() {
2545	"cancelled"
2546	} else {
2547	report["conclusion"]
2548	.as_str()
2549	.filter(\|c\| matches!(*c, "success" \| "failure" \| "cancelled"))
2550	.unwrap_or("failure")
2551	};
2552	let outputs = report["outputs"].as_object().cloned();
2553	Box::pin(self.finish_job(&job.id, conclusion, report["reason"].as_str(), outputs.as_ref())).await?;
2554	}
2555	other => return Ok(fail(FailureCode::Invalid, format!("There is no report called `{other}`."))),
2556	}
2557	// `cancelled`: the run was cancelled, and the runner should stop
2558	// the step it is on and run only its cleanup steps.
2559	Ok(Outcome::Ok(json!({ "ok": true, "cancelled": job.cancel_requested_at.is_some() })))
2560	}
2561
2562	// --- Every minute ---------------------------------------------------------------
2563
2564	pub async fn on_minute(&self, now_ms: u64) -> Result<()> {
2565	let minute = now_ms / 60_000 * 60_000;
2566	// Staff (or billing's usage watcher) paused scheduled runs across
2567	// g1t: this minute's schedules are skipped, not queued for later.
2568	// Kept 30 seconds in the isolate (g1t_kit::pause).
2569	if g1t_kit::pause::paused(&self.billing, g1t_contracts::billing::PauseLevel::Schedules).await {
2570	worker::console_log!("actions: schedules are paused across g1t; skipped this minute's");
2571	} else if let Err(error) = self.run_schedules(minute).await {
2572	worker::console_error!("actions: schedules failed: {error}");
2573	}
2574	// Jobs whose sandbox went quiet or ran past their time.
2575	let running = self.db.prepare("SELECT * FROM jobs WHERE status = 'in_progress'").all().await?.results::<JobRow>()?;
2576	for job in running {
2577	// Times in g1t's format compare as text.
2578	let before = \|ms: u64\| rfc3339(now_ms.saturating_sub(ms));
2579	let silent = job.seen_at.as_deref().is_some_and(\|seen\| seen < before(SILENT_MS).as_str());
2580	let limit = (u64::from(job.timeout_minutes) * 60 + 120) * 1000;
2581	let over = job.started_at.as_deref().is_some_and(\|started\| started < before(limit).as_str());
2582	let cancel_overdue = job.cancel_requested_at.as_deref().is_some_and(\|at\| at < before(CANCEL_GRACE_MS).as_str());
2583	if cancel_overdue {
2584	// Cancelled, and still going after its grace period.
2585	self.hard_stop(&job, "It was cancelled, and did not finish its cleanup steps within 5 minutes.").await?;
2586	self.advance(&job.run_id).await?;
2587	} else if over {
2588	let reason = format!("It ran longer than its time limit of {} minutes.", job.timeout_minutes);
2589	// A self-hosted runner is told to stop on its next poll.
2590	if job.runner_id.is_none() {
2591	let _: Result<Value> = g1t_kit::call(&self.runner, "stop_actions_job", &json!({ "job": job.id })).await;
2592	}
2593	self.finish_job(&job.id, "failure", Some(&reason), None).await?;
2594	} else if silent {
2595	let reason = match &job.runner_name {
2596	Some(name) => format!("The self-hosted runner {name} stopped answering."),
2597	None => "The runner stopped answering.".to_owned(),
2598	};
2599	self.finish_job(&job.id, "failure", Some(&reason), None).await?;
2600	}
2601	}
2602	// Calls of reusable workflows waiting in a concurrency group, in
2603	// case the job they waited for was cancelled in another run.
2604	#[derive(Deserialize)]
2605	struct RunId {
2606	run_id: String,
2607	}
2608	let held = self
2609	.db
2610	.prepare("SELECT DISTINCT run_id FROM jobs WHERE status = 'waiting' AND concurrency_group IS NOT NULL LIMIT 20")
2611	.all()
2612	.await?
2613	.results::<RunId>()?;
2614	for run in held {
2615	self.advance(&run.run_id).await?;
2616	}
2617	if let Err(error) = self.sweep_runners(now_ms).await {
2618	worker::console_error!("actions: the runners' sweep failed: {error}");
2619	}
2620	// Jobs held at an environment whose wait timer has run out.
2621	if let Err(error) = self.release_gates().await {
2622	worker::console_error!("actions: environments' gates failed: {error}");
2623	}
2624	// Once an hour: the cache's expired entries, and its storage.
2625	if (now_ms / 60_000) % 60 == 7
2626	&& let Err(error) = self.sweep_cache(now_ms).await
2627	{
2628	worker::console_error!("actions: the cache's sweep failed: {error}");
2629	}
2630	// And the archives of GitHub's actions no job uses any more.
2631	if (now_ms / 60_000) % 60 == 47
2632	&& let Err(error) = self.sweep_action_archives(now_ms).await
2633	{
2634	worker::console_error!("actions: the action archives' sweep failed: {error}");
2635	}
2636	// And artifacts past their time, and the toolkit's abandoned parts.
2637	if (now_ms / 60_000) % 60 == 37 {
2638	if let Err(error) = self.sweep_artifacts(now_ms).await {
2639	worker::console_error!("actions: the artifacts' sweep failed: {error}");
2640	}
2641	if let Err(error) = self.sweep_blob_parts(now_ms).await {
2642	worker::console_error!("actions: the blob parts' sweep failed: {error}");
2643	}
2644	}
2645	self.start_queued().await
2646	}
2647	}
2648
2649
2650	#[cfg(test)]
2651	mod log_chunks {
2652	use rusqlite::{Connection, params};
2653
2654	use super::LOG_INSERT;
2655
2656	/// The logs table as the migrations leave it.
2657	fn database() -> Connection {
2658	let db = Connection::open_in_memory().unwrap();
2659	for name in ["0001_init.sql", "0016_log_chunks.sql"] {
2660	let path = format!("{}/migrations/{name}", env!("CARGO_MANIFEST_DIR"));
2661	db.execute_batch(&std::fs::read_to_string(path).unwrap()).unwrap();
2662	}
2663	db
2664	}
2665
2666	fn log(db: &Connection, job: &str) -> Vec<(i64, i64, String)> {
2667	let mut rows = db.prepare("SELECT seq, step, text FROM logs WHERE job_id = ? ORDER BY seq").unwrap();
2668	rows.query_map([job], \|row\| Ok((row.get(0)?, row.get(1)?, row.get(2)?))).unwrap().map(Result::unwrap).collect()
2669	}
2670
2671	#[test]
2672	fn a_batch_sent_again_is_kept_once() {
2673	let db = database();
2674	let insert = \|job: &str, step: i64, text: &str, chunk: Option<i64>\| db.execute(LOG_INSERT, params![job, step, text, chunk]).unwrap();
2675	assert_eq!(insert("job_1", 0, "one\n", Some(1)), 1);
2676	// The runner did not hear the answer and sent the batch again.
2677	assert_eq!(insert("job_1", 0, "one\n", Some(1)), 0);
2678	assert_eq!(insert("job_1", 1, "two\n", Some(2)), 1);
2679	// Another job numbers its batches from 1 too.
2680	assert_eq!(insert("job_2", 0, "other\n", Some(1)), 1);
2681	// A runner that numbers nothing has every batch kept, as before.
2682	assert_eq!(insert("job_3", 0, "a\n", None), 1);
2683	assert_eq!(insert("job_3", 0, "a\n", None), 1);
2684	assert_eq!(log(&db, "job_1"), [(1, 0, "one\n".into()), (2, 1, "two\n".into())]);
2685	assert_eq!(log(&db, "job_2"), [(1, 0, "other\n".into())]);
2686	assert_eq!(log(&db, "job_3").len(), 2);
2687	// A job run again moves its log aside (`{id}.{attempt}`), and the
2688	// new attempt's runner numbers from 1 again.
2689	db.execute("UPDATE logs SET job_id = job_id \|\| '.1' WHERE job_id = 'job_1'", []).unwrap();
2690	assert_eq!(insert("job_1", 0, "again\n", Some(1)), 1);
2691	assert_eq!(log(&db, "job_1"), [(1, 0, "again\n".into())]);
2692	assert_eq!(log(&db, "job_1.1").len(), 2);
2693	}
2694	}
2695
2696	#[cfg(test)]
2697	mod stopping {
2698	use super::stops_runs;
2699	use g1t_contracts::events::Event;
2700	use serde_json::{Value, json};
2701
2702	fn event(kind: &str, data: Value) -> Event {
2703	Event {
2704	id: "evt_1".into(),
2705	kind: kind.into(),
2706	source: "repos".into(),
2707	time: "2026-10-05T00:00:00Z".into(),
2708	repo_id: Some("rep_1".into()),
2709	actor: None,
2710	data,
2711	}
2712	}
2713
2714	#[test]
2715	fn deleting_or_archiving_stops_runs() {
2716	assert_eq!(stops_runs(&event("repo.deleted", json!({ "repoId": "rep_1" }))).as_deref(), Some("rep_1"));
2717	assert_eq!(stops_runs(&event("repo.archived", json!({ "archived": true }))).as_deref(), Some("rep_1"));
2718	assert_eq!(stops_runs(&event("repo.unarchived", json!({ "archived": false }))), None);
2719	assert_eq!(stops_runs(&event("repo.restored", json!({}))), None);
2720	assert_eq!(stops_runs(&event("git.push", json!({}))), None);
2721	}
2722	}
2723
2724	#[cfg(test)]
2725	mod status_of_needs {
2726	use std::collections::HashMap;
2727
2728	use super::ancestor_failed;
2729
2730	/// check -> plan -> (migrate) -> core -> edge, as deploy.yml has them,
2731	/// and a job that needs only the last.
2732	fn graph() -> HashMap<&'static str, Vec<&'static str>> {
2733	HashMap::from([
2734	("check", vec![]),
2735	("plan", vec!["check"]),
2736	("migrate", vec!["plan"]),
2737	("core", vec!["plan", "migrate"]),
2738	("edge", vec!["plan", "migrate", "core"]),
2739	("notify", vec!["edge"]),
2740	])
2741	}
2742
2743	#[test]
2744	fn a_failure_is_seen_however_far_back() {
2745	let needs = graph();
2746	let failed = \|which: &'static str\| move \|key: &str\| key == which;
2747	// check failed; plan, and everything after, was skipped for it.
2748	assert!(ancestor_failed(&needs, "notify", failed("check")));
2749	assert!(ancestor_failed(&needs, "core", failed("check")));
2750	assert!(ancestor_failed(&needs, "edge", failed("core")));
2751	// Nothing before a job failed: a skipped migrate is not a failure.
2752	assert!(!ancestor_failed(&needs, "edge", \|_\| false));
2753	assert!(!ancestor_failed(&needs, "core", failed("edge")));
2754	assert!(!ancestor_failed(&needs, "check", failed("check")));
2755	}
2756
2757	#[test]
2758	fn cycles_and_unknown_keys_end() {
2759	let needs = HashMap::from([("a", vec!["b"]), ("b", vec!["a"])]);
2760	assert!(!ancestor_failed(&needs, "a", \|_\| false));
2761	assert!(!ancestor_failed(&needs, "missing", \|_\| true));
2762	}
2763	}
2764
2765	#[cfg(test)]
2766	mod deployments {
2767	use serde_json::{Map, Value, json};
2768
2769	use super::{JobEnvironment, deployment_outcome, environment_of};
2770
2771	#[test]
2772	fn a_jobs_environment_is_read_for_deployments() {
2773	let contexts: Map<String, Value> = serde_json::from_value(json!({
2774	"github": { "ref_name": "main", "repository": "acme/web" },
2775	"inputs": { "target": "staging" },
2776	"matrix": {},
2777	}))
2778	.unwrap();
2779	let read = \|raw: Value\| environment_of(&raw, &contexts);
2780	assert_eq!(read(json!({})), None);
2781	assert_eq!(
2782	read(json!({ "environment": "production" })),
2783	Some(JobEnvironment { name: "production".into(), url: None, deploys: true })
2784	);
2785	assert_eq!(
2786	read(json!({ "environment": { "name": "production", "url": "https://g1t.sh" } })),
2787	Some(JobEnvironment { name: "production".into(), url: Some("https://g1t.sh".into()), deploys: true })
2788	);
2789	// Expressions are filled in from the run.
2790	assert_eq!(
2791	read(json!({ "environment": { "name": "${{ inputs.target }}", "url": "https://${{ github.ref_name }}.example.com" } })),
2792	Some(JobEnvironment { name: "staging".into(), url: Some("https://main.example.com".into()), deploys: true })
2793	);
2794	// Secrets only: no deployment.
2795	assert!(!read(json!({ "environment": { "name": "production", "deployment": false } })).unwrap().deploys);
2796	// The same, said in the job's g1t settings.
2797	assert!(!read(json!({ "environment": "production", "g1t": { "deployment": false } })).unwrap().deploys);
2798	assert!(!read(json!({ "environment": { "name": "production" }, "g1t": { "deployment": false } })).unwrap().deploys);
2799	// Only http(s) addresses.
2800	assert_eq!(read(json!({ "environment": { "name": "production", "url": "javascript:alert(1)" } })).unwrap().url, None);
2801	}
2802
2803	#[test]
2804	fn a_runs_outcome_for_an_environment() {
2805	let of = \|list: &[&str]\| deployment_outcome(&list.iter().map(\|c\| Some((*c).to_owned())).collect::<Vec<_>>());
2806	assert_eq!(of(&["success", "skipped"]), Some("success"));
2807	assert_eq!(of(&["success", "failure"]), Some("failure"));
2808	assert_eq!(of(&["success", "cancelled"]), Some("error"));
2809	assert_eq!(of(&["skipped"]), None);
2810	assert_eq!(deployment_outcome(&[None]), None);
2811	}
2812	}
2813
2814	#[cfg(test)]
2815	mod reruns {
2816	use serde_json::{Map, Value, json};
2817
2818	use super::{MAX_SUMMARIES, MAX_SUMMARY_BYTES, Rerun, Scope, Status, call_allowed, call_prefix, debug_logging, rerun_keys, shown_name, summary_fits, top_key};
2819
2820	/// build ← test ← deploy, and lint on its own.
2821	fn keys(which: Rerun, failed: &[&str]) -> Vec<String> {
2822	let order = ["build", "lint", "test", "deploy"];
2823	let needs = \|key: &str\| -> Vec<String> {
2824	match key {
2825	"test" => vec!["build".into()],
2826	"deploy" => vec!["test".into()],
2827	_ => Vec::new(),
2828	}
2829	};
2830	rerun_keys(&order, needs, \|key\| !failed.contains(&key), which)
2831	}
2832
2833	#[test]
2834	fn a_rerun_takes_the_jobs_it_names_and_those_that_need_them() {
2835	assert_eq!(keys(Rerun::All, &[]), ["build", "lint", "test", "deploy"]);
2836	assert_eq!(keys(Rerun::Failed, &["test"]), ["test", "deploy"]);
2837	assert_eq!(keys(Rerun::Failed, &["lint"]), ["lint"]);
2838	assert!(keys(Rerun::Failed, &[]).is_empty());
2839	// One job, whatever it came to, and what depends on it.
2840	assert_eq!(keys(Rerun::Job("build"), &[]), ["build", "test", "deploy"]);
2841	assert_eq!(keys(Rerun::Job("lint"), &["test"]), ["lint"]);
2842	assert_eq!(keys(Rerun::Job("deploy"), &[]), ["deploy"]);
2843	assert!(keys(Rerun::Job("missing"), &[]).is_empty());
2844	}
2845
2846	#[test]
2847	fn a_called_workflows_jobs_run_again_with_the_job_that_calls_it() {
2848	assert_eq!(top_key("build/test"), "build");
2849	assert_eq!(top_key("build/inner/test"), "build");
2850	assert_eq!(top_key("lint"), "lint");
2851	// A matrix's calls each have their own jobs, under `key[i]/`.
2852	assert_eq!(top_key("build[1]/test"), "build");
2853	assert_eq!(top_key("build/inner[0]/test"), "build");
2854	assert_eq!(call_prefix("build", Some(1)), "build[1]");
2855	assert_eq!(call_prefix("build/inner", None), "build/inner");
2856	}
2857
2858	#[test]
2859	fn a_run_connects_ten_levels_of_workflows_and_fifty_reusable_ones() {
2860	assert!(call_allowed(1, &[], "a.yml").is_ok());
2861	assert!(call_allowed(9, &[], "a.yml").is_ok());
2862	let deep = call_allowed(10, &[], "a.yml").unwrap_err();
2863	assert!(deep.contains("at most ten") && deep.contains("nine levels"), "{deep}");
2864	let names: Vec<String> = (0..50).map(\|n\| format!("w{n}.yml")).collect();
2865	let fifty: Vec<&str> = names.iter().map(String::as_str).collect();
2866	// A workflow called again, as by each combination of a matrix, is one.
2867	assert!(call_allowed(1, &fifty, "w3.yml").is_ok());
2868	assert!(call_allowed(1, &fifty, "another.yml").unwrap_err().contains("at most 50 different"));
2869	}
2870
2871	#[test]
2872	fn a_matrixs_calls_are_named_by_the_caller_and_its_values() {
2873	let workflow = super::workflow::parse(
2874	"on: push\njobs:\n build:\n name: Build\n strategy:\n matrix: { os: [linux], node: [20] }\n uses: ./.g1t/workflows/b.yml\n plain:\n uses: ./.g1t/workflows/b.yml\n test:\n name: Test ${{ matrix.os }}\n strategy:\n matrix: { os: [linux] }\n runs-on: ubuntu-latest\n steps: [{ run: 'true' }]",
2875	)
2876	.unwrap();
2877	let job = \|id: &str\| workflow.jobs.iter().find(\|job\| job.id == id).unwrap();
2878	let combination: Map<String, Value> = serde_json::from_value(json!({ "os": "linux", "node": 20 })).unwrap();
2879	let mut contexts = Map::new();
2880	contexts.insert("matrix".into(), Value::Object(combination.clone()));
2881	let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
2882	assert_eq!(shown_name(job("build"), None, &combination, &scope), "Build (linux, 20)");
2883	assert_eq!(shown_name(job("plain"), None, &Map::new(), &scope), "plain");
2884	// A called workflow's job, under the call that made it.
2885	assert_eq!(shown_name(job("plain"), Some("Build (linux, 20)"), &Map::new(), &scope), "Build (linux, 20) / plain");
2886	assert_eq!(shown_name(job("test"), Some("Build"), &combination, &scope), "Build / Test linux");
2887	}
2888
2889	#[test]
2890	fn a_job_keeps_twenty_steps_summaries_of_a_mebibyte_each() {
2891	assert!(summary_fits(0, None, 10));
2892	assert!(!summary_fits(0, None, 0), "nothing to keep");
2893	assert!(!summary_fits(MAX_SUMMARIES, None, 10), "a twenty-first step's summary is dropped");
2894	assert!(summary_fits(MAX_SUMMARIES, Some(10), 10), "a step that has one may add to it");
2895	assert!(summary_fits(1, Some(MAX_SUMMARY_BYTES - 10), 10));
2896	assert!(!summary_fits(1, Some(MAX_SUMMARY_BYTES - 10), 11));
2897	assert!(!summary_fits(0, None, MAX_SUMMARY_BYTES + 1));
2898	}
2899
2900	#[test]
2901	fn a_debug_rerun_sets_what_github_sets() {
2902	let mut variables = Map::new();
2903	let mut runner = json!({ "name": "g1t", "debug": "" });
2904	debug_logging(&mut variables, &mut runner);
2905	assert_eq!(variables["RUNNER_DEBUG"], "1");
2906	assert_eq!(variables["ACTIONS_STEP_DEBUG"], "true");
2907	assert_eq!(variables["ACTIONS_RUNNER_DEBUG"], "true");
2908	assert_eq!(runner["debug"], Value::String("1".into()));
2909	assert_eq!(runner["name"], "g1t");
2910	}
2911	}