Commit

Actions: reusable workflows in the repository

A job with `uses: ./.g1t/workflows/x.yml` calls that workflow: its jobs join the run under the caller ("caller / job"), read the caller's `with:` as their inputs (with the called workflow's defaults and required inputs checked), and run as that workflow defines them. The caller finishes with their result and the outputs the workflow declares under on.workflow_call.outputs, so jobs that need it read them. Calls nest up to four deep. A workflow still saying ./.github/workflows/... is found under .g1t/. Re-running a caller makes its called jobs again. Workflows in other repositories are still not called, and say so.

syntaqxcommitted Parentdbce826Browse files
6 files+273−170/6 viewed
+2−1
3737 | `run:` with `bash`, `sh`, `python` or a custom shell | The same. |
3838 | JavaScript actions (`uses: owner/repo@v7`) | Fetched from GitHub and run as they are, on Node 24, the runtime current actions declare. |
3939 | Composite actions | The same. |
40+| Reusable workflows in the repository (`jobs.<id>.uses: ./.g1t/workflows/build.yml`) | The same: `with:` inputs, `on.workflow_call` outputs, and nesting up to four deep. `./.github/workflows/…` finds the workflow under `.g1t/` after the move. Their jobs run with the repository's secrets. |
4041 | `actions/checkout` | Checks out from g1t, with `ref`, `fetch-depth`, `path`, `repository`, `token` and `submodules`. |
4142 | `GITHUB_OUTPUT`, `GITHUB_ENV`, `GITHUB_PATH`, `GITHUB_STATE`, `GITHUB_STEP_SUMMARY` | The same. |
4243 | `::error::`, `::warning::`, `::notice::`, `::group::`, `::add-mask::` | The same: errors and warnings become annotations on the run. |
5253 - **Windows and macOS runners.** Jobs run on Linux; a job with
5354 `runs-on: windows-latest` or `macos-latest` fails, and says so.
5455 - **Docker** container actions, `services:` containers and `container:`.
55−- **Reusable workflows** (`uses:` on a job).
56+- **Reusable workflows from other repositories** (`uses: owner/repo/.github/workflows/x.yml@v1`); ones in the same repository work.
5657 - **The toolkit's own cache.** Actions that cache through GitHub's service
5758 themselves, such as `actions/setup-node` with `cache: npm`, run without
5859 it. Use `actions/cache` for the same effect.
+1−0
4040 if (status === "in_progress") return "Running";
4141 if (status === "pending") return "Waiting for its concurrency group";
4242 if (status === "waiting") return "Waiting for the jobs it needs";
43+ if (status === "calling") return "Running the workflow it calls";
4344 if (status !== "completed") return "Queued";
4445 return { success: "Succeeded", failure: "Failed", cancelled: "Cancelled", skipped: "Skipped" }[conclusion ?? "skipped"];
4546 }
+6−2
412412 ),
413413 _ => (None, true, None),
414414 };
415− if uses.is_some() {
416− note(Severity::Unsupported, Some(id), "Reusable workflows (`uses:` on a job) are not called on g1t yet, so this job fails.".to_owned());
415+ if uses.as_deref().is_some_and(|uses| !uses.starts_with("./")) {
416+ note(
417+ Severity::Unsupported,
418+ Some(id),
419+ "Reusable workflows from other repositories are not called on g1t yet, so this job fails; ones in this repository (`./.g1t/workflows/…`) are.".to_owned(),
420+ );
417421 }
418422 jobs.push(Job {
419423 id: id.clone(),
+2−1
8181 key: string;
8282 name: string;
8383 needs: string[];
84− status: "waiting" | "queued" | "in_progress" | "completed";
84+ /** `calling`: running the reusable workflow it calls, whose jobs follow it. */
85+ status: "waiting" | "queued" | "in_progress" | "calling" | "completed";
8586 conclusion: Conclusion | null;
8687 steps: StepState[];
8788 annotations: Annotation[];
+5−0
1+-- Reusable workflows: a job that calls a workflow in the repository, and
2+-- the jobs of that workflow, which join the run. As JSON:
3+-- the caller: {"role": "caller", "path", "source"};
4+-- a called job: {"role": "callee", "parent", "job", "path", "source", "inputs", "depth"}.
5+ALTER TABLE jobs ADD COLUMN call TEXT;
+257−13
136136 pub timeout_minutes: u32,
137137 pub continue_on_error: u32,
138138 pub max_parallel: Option<u32>,
139+ /// Set for a job that calls a reusable workflow, and for that
140+ /// workflow's jobs (see migration 0002).
141+ pub call: Option<String>,
139142 pub seen_at: Option<String>,
140143 pub started_at: Option<String>,
141144 pub finished_at: Option<String>,
145148 pub fn needs(&self) -> Vec<String> {
146149 serde_json::from_str(&self.needs).unwrap_or_default()
147150 }
151+
152+ pub fn call(&self) -> Option<Value> {
153+ self.call.as_deref().and_then(|call| serde_json::from_str(call).ok())
154+ }
155+
156+ /// For a job of a called workflow: that workflow, the job's own id in
157+ /// it, and the job.
158+ pub fn callee(&self) -> Option<(Workflow, workflow::Job, Value)> {
159+ let call = self.call().filter(|call| call["role"] == "callee")?;
160+ let called = workflow::parse(call["source"].as_str()?).ok()?;
161+ let job = called.jobs.iter().find(|job| call["job"].as_str() == Some(job.id.as_str()))?.clone();
162+ Some((called, job, call))
163+ }
148164 }
149165
166+/// A job to decide on: its key, its definition, and what it needs, as
167+/// (name in `needs`, key of the jobs).
168+type Unit = (String, workflow::Job, Vec<(String, String)>);
169+
170+/// How deep reusable workflows may call one another, as on GitHub.
171+const MAX_CALL_DEPTH: u64 = 4;
172+
150173 /// What the jobs of one key came to, for `needs.<key>`.
151174 fn key_result(rows: &[&JobRow]) -> &'static str {
152175 let failed = |row: &&&JobRow| row.conclusion.as_deref() == Some("failure") && row.continue_on_error == 0;
398421 };
399422 let jobs = self.job_rows(run_id).await?;
400423 let mut changed = false;
401− for job in &workflow.jobs {
402− let rows: Vec<&JobRow> = jobs.iter().filter(|row| row.key == job.id).collect();
424+ // What to decide on: the workflow's jobs, and the jobs of the
425+ // workflows they call, each with its needs as (name, key).
426+ let mut units: Vec<Unit> = workflow
427+ .jobs
428+ .iter()
429+ .map(|job| (job.id.clone(), job.clone(), job.needs.iter().map(|n| (n.clone(), n.clone())).collect()))
430+ .collect();
431+ let mut seen = std::collections::HashSet::new();
432+ for row in &jobs {
433+ if !seen.insert(row.key.clone()) {
434+ continue;
435+ }
436+ if let Some((_, job, call)) = row.callee() {
437+ let parent = call["parent"].as_str().unwrap_or_default().to_owned();
438+ let needs = job.needs.iter().map(|n| (n.clone(), format!("{parent}/{n}"))).collect();
439+ units.push((row.key.clone(), job, needs));
440+ }
441+ }
442+ for (key, job, needs) in &units {
443+ let rows: Vec<&JobRow> = jobs.iter().filter(|row| &row.key == key).collect();
403444 if rows.is_empty() || !rows.iter().all(|row| row.status == "waiting") {
404445 continue;
405446 }
406447 let needed: Vec<(&String, Vec<&JobRow>)> =
407− job.needs.iter().map(|need| (need, jobs.iter().filter(|row| &row.key == need).collect())).collect();
448+ needs.iter().map(|(name, need)| (name, jobs.iter().filter(|row| &row.key == need).collect())).collect();
408449 if !needed.iter().all(|(_, rows)| rows.iter().all(|row| row.status == "completed")) {
409450 continue;
410451 }
411452 self.decide(&run, job, rows[0], &needed).await?;
412453 changed = true;
413454 }
455+ // A job that called a workflow finishes with that workflow's jobs.
456+ for row in jobs.iter().filter(|row| row.status == "calling") {
457+ let children: Vec<&JobRow> = jobs
458+ .iter()
459+ .filter(|child| child.call().is_some_and(|call| call["role"] == "callee" && call["parent"].as_str() == Some(row.key.as_str())))
460+ .collect();
461+ if !children.is_empty() && children.iter().all(|child| child.status == "completed") {
462+ self.finish_call(row, &children).await?;
463+ changed = true;
464+ }
465+ }
414466 if !changed {
415467 break;
416468 }
424476 async fn decide(&self, run: &RunRow, job: &workflow::Job, row: &JobRow, needed: &[(&String, Vec<&JobRow>)]) -> Result<()> {
425477 let vars = self.variables_for(&run.repo_id, &repo_path(&run.repo).namespace).await?;
426478 let mut contexts = Self::base_contexts(run, &vars, &job.id);
479+ // A called workflow's jobs read the inputs they were called with.
480+ let call = row.call();
481+ let parent = call.as_ref().filter(|c| c["role"] == "callee").and_then(|c| c["parent"].as_str().map(str::to_owned));
482+ if let Some(call) = call.as_ref().filter(|c| c["role"] == "callee") {
483+ contexts.insert("inputs".into(), call["inputs"].clone());
484+ }
427485 let mut needs = Map::new();
428486 let mut status = if run.conclusion.as_deref() == Some("cancelled") { Status::Cancelled } else { Status::Success };
429487 for (key, rows) in needed {
451509 Ok(false) => return self.skip_job(row, None).await,
452510 Err(problem) => return self.fail_job(row, &format!("Its `if` does not read: {problem}")).await,
453511 }
454− if job.uses.is_some() {
455− return self.fail_job(row, "Reusable workflows (`uses:` on a job) are not called on g1t yet.").await;
512+ if let Some(uses) = &job.uses {
513+ return self.call_workflow(run, job, row, uses, &scope).await;
456514 }
457515
458516 // Its matrix, which may come from a needed job's outputs.
486544 hash_files: None,
487545 };
488546 let base_name = job.name.clone().unwrap_or(job.id.clone());
547+ // A called workflow's job is shown under the job that called it.
548+ let base_name = match &parent {
549+ Some(parent) => format!("{} / {base_name}", parent.replace('/', " / ")),
550+ None => base_name,
551+ };
489552 let name = if expr::has_expression(&base_name) {
490553 expr::interpolate(&base_name, &scope).unwrap_or(base_name)
491554 } else if job.matrix.is_some() {
554617 row.key.as_str().into(),
555618 (index as u32).into(),
556619 row.needs.as_str().into(),
620+ optional(row.call.as_deref()),
557621 ];
558622 bound.extend(values);
559623 statements.push(
560624 self.db
561625 .prepare(
562− "INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, name, matrix, status, conclusion, reason,
626+ "INSERT INTO jobs (id, run_id, repo_id, namespace, key, ordinal, needs, call, name, matrix, status, conclusion, reason,
563627 timeout_minutes, continue_on_error, max_parallel, finished_at)
564− VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
628+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
565629 )
566630 .bind(&bound)?,
567631 );
571635 Ok(())
572636 }
573637
638+ /// A job that calls a reusable workflow in the repository: that
639+ /// workflow's jobs join the run under it, with the inputs it passes.
640+ async fn call_workflow(&self, run: &RunRow, job: &workflow::Job, row: &JobRow, uses: &str, scope: &Scope<'_>) -> Result<()> {
641+ let Some(local) = uses.strip_prefix("./") else {
642+ return self
643+ .fail_job(row, "Reusable workflows from other repositories are not called on g1t yet; ones in this repository (`./.g1t/workflows/…`) are.")
644+ .await;
645+ };
646+ let depth = row.call().and_then(|c| c["depth"].as_u64()).unwrap_or(0) + 1;
647+ if depth > MAX_CALL_DEPTH {
648+ return self.fail_job(row, &format!("Reusable workflows call each other more than {MAX_CALL_DEPTH} deep.")).await;
649+ }
650+ let local = local.split('@').next().unwrap_or(local).to_owned();
651+ let path = repo_path(&run.repo);
652+ let Some(ws) = self.workspace_actor(&path.namespace).await? else {
653+ return self.fail_job(row, "The workspace is gone.").await;
654+ };
655+ // A repository moved from GitHub keeps saying `.github/…`.
656+ let mut found = self.read_file(&path, &ws, &run.sha, &local).await?.map(|text| (local.clone(), text));
657+ if found.is_none()
658+ && let Some(rest) = local.strip_prefix(".github/")
659+ {
660+ let moved = format!(".g1t/{rest}");
661+ found = self.read_file(&path, &ws, &run.sha, &moved).await?.map(|text| (moved, text));
662+ }
663+ let Some((file, source)) = found else {
664+ return self.fail_job(row, &format!("`{uses}` is not in the repository at this commit.")).await;
665+ };
666+ let called = match workflow::parse(&source) {
667+ Ok(called) => called,
668+ Err(problem) => return self.fail_job(row, &format!("`{file}` does not read: {problem}")).await,
669+ };
670+ let Some(trigger) = called.trigger("workflow_call") else {
671+ return self.fail_job(row, &format!("`{file}` cannot be called: it has no `on: workflow_call`.")).await;
672+ };
673+ // Inputs: what the caller passes, else the called workflow's defaults.
674+ let given = match job.raw.get("with") {
675+ Some(with) => match expr::interpolate_value(with, scope) {
676+ Ok(Value::Object(given)) => given,
677+ Ok(_) => Map::new(),
678+ Err(problem) => return self.fail_job(row, &format!("Its `with` does not read: {problem}")).await,
679+ },
680+ None => Map::new(),
681+ };
682+ let mut inputs = Map::new();
683+ for (name, spec) in &trigger.inputs {
684+ let value = given.get(name).cloned().or_else(|| spec.get("default").cloned()).unwrap_or(Value::Null);
685+ if value.is_null() && spec.get("required").and_then(Value::as_bool) == Some(true) {
686+ return self.fail_job(row, &format!("`{file}` needs the input `{name}`.")).await;
687+ }
688+ inputs.insert(name.clone(), value);
689+ }
690+ for (name, value) in given {
691+ inputs.entry(name).or_insert(value);
692+ }
693+ let mut statements = Vec::new();
694+ for called_job in &called.jobs {
695+ let needs: Vec<String> = called_job.needs.iter().map(|n| format!("{}/{n}", row.key)).collect();
696+ let call = json!({
697+ "role": "callee", "parent": row.key, "job": called_job.id, "path": file,
698+ "source": source, "inputs": inputs, "depth": depth,
699+ });
700+ statements.push(
701+ self.db
702+ .prepare("INSERT INTO jobs (id, run_id, repo_id, namespace, key, name, needs, status, call) VALUES (?, ?, ?, ?, ?, ?, ?, 'waiting', ?)")
703+ .bind(&[
704+ new_id("job", now_ms()).into(),
705+ row.run_id.as_str().into(),
706+ row.repo_id.as_str().into(),
707+ row.namespace.as_str().into(),
708+ format!("{}/{}", row.key, called_job.id).into(),
709+ format!("{} / {}", row.name, called_job.name.clone().unwrap_or(called_job.id.clone())).into(),
710+ serde_json::to_string(&needs)?.into(),
711+ serde_json::to_string(&call)?.into(),
712+ ])?,
713+ );
714+ }
715+ statements.push(
716+ self.db
717+ .prepare("UPDATE jobs SET status = 'calling', call = ?, reason = ?, started_at = ? WHERE id = ?")
718+ .bind(&[
719+ serde_json::to_string(&json!({ "role": "caller", "path": file, "source": source }))?.into(),
720+ format!("Calls `{file}`.").into(),
721+ now().into(),
722+ row.id.as_str().into(),
723+ ])?,
724+ );
725+ self.db.batch(statements).await?;
726+ Ok(())
727+ }
728+
729+ /// A job that called a workflow, finished with its jobs: their result,
730+ /// and the outputs the workflow declares.
731+ async fn finish_call(&self, row: &JobRow, children: &[&JobRow]) -> Result<()> {
732+ let call = row.call().unwrap_or_default();
733+ let called = call["source"].as_str().and_then(|s| workflow::parse(s).ok());
734+ let mut jobs_context = Map::new();
735+ let mut by_key: std::collections::BTreeMap<String, Vec<&JobRow>> = std::collections::BTreeMap::new();
736+ for child in children {
737+ by_key.entry(child.key.clone()).or_default().push(child);
738+ }
739+ for (key, rows) in &by_key {
740+ let mut outputs = Map::new();
741+ for child in rows {
742+ if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&child.outputs) {
743+ outputs.extend(more);
744+ }
745+ }
746+ let id = key.rsplit('/').next().unwrap_or(key);
747+ jobs_context.insert(id.to_owned(), json!({ "result": key_result(rows), "outputs": outputs }));
748+ }
749+ let inputs = children.first().and_then(|c| c.call()).map(|c| c["inputs"].clone()).unwrap_or(json!({}));
750+ let mut contexts = Map::new();
751+ contexts.insert("jobs".into(), Value::Object(jobs_context));
752+ contexts.insert("inputs".into(), inputs);
753+ let scope = Scope { contexts: &contexts, status: Status::Success, hash_files: None };
754+ let mut outputs = Map::new();
755+ if let Some(Value::Object(declared)) = called.as_ref().map(|w| {
756+ let on = w.raw.get("on").or_else(|| w.raw.get("true")).cloned().unwrap_or(Value::Null);
757+ on.get("workflow_call").and_then(|c| c.get("outputs")).cloned().unwrap_or(Value::Null)
758+ }) {
759+ for (name, spec) in declared {
760+ if let Some(value) = spec.get("value") {
761+ let value = expr::interpolate_value(value, &scope).unwrap_or(Value::Null);
762+ outputs.insert(name, Value::String(expr::to_text(&value)));
763+ }
764+ }
765+ }
766+ let conclusion = key_result(children);
767+ self.db
768+ .prepare("UPDATE jobs SET status = 'completed', conclusion = ?, outputs = ?, finished_at = ? WHERE id = ? AND status = 'calling'")
769+ .bind(&[conclusion.into(), serde_json::to_string(&outputs)?.into(), now().into(), row.id.as_str().into()])?
770+ .run()
771+ .await?;
772+ Ok(())
773+ }
774+
775+ /// A file's text at a commit, if it is there.
776+ async fn read_file(&self, path: &RepoPath, ws: &g1t_contracts::User, sha: &str, file: &str) -> Result<Option<String>> {
777+ let blob: Outcome<g1t_contracts::repos::BlobView> = g1t_kit::call(
778+ &self.repos,
779+ "blob",
780+ &g1t_contracts::repos::BlobArgs {
781+ path: path.clone(),
782+ viewer: Some(ws.clone()),
783+ git_ref: sha.to_owned(),
784+ file_path: file.to_owned(),
785+ },
786+ )
787+ .await?;
788+ Ok(match blob {
789+ Outcome::Ok(view) => view.text,
790+ Outcome::Fail(_) => None,
791+ })
792+ }
793+
574794 async fn skip_job(&self, row: &JobRow, reason: Option<&str>) -> Result<()> {
575795 self.db
576796 .prepare("UPDATE jobs SET status = 'completed', conclusion = 'skipped', reason = ?, finished_at = ? WHERE id = ?")
9291149 for key in &again {
9301150 statements.push(self.db.prepare("DELETE FROM logs WHERE job_id IN (SELECT id FROM jobs WHERE run_id = ? AND key = ?)").bind(&[run.id.as_str().into(), key.as_str().into()])?);
9311151 statements.push(self.db.prepare("DELETE FROM jobs WHERE run_id = ? AND key = ? AND ordinal > 0").bind(&[run.id.as_str().into(), key.as_str().into()])?);
1152+ // The jobs of a workflow it called are made again when it calls it again.
9321153 statements.push(
9331154 self.db
1155+ .prepare("DELETE FROM logs WHERE job_id IN (SELECT id FROM jobs WHERE run_id = ? AND key LIKE ?)")
1156+ .bind(&[run.id.as_str().into(), format!("{key}/%").into()])?,
1157+ );
1158+ statements.push(
1159+ self.db
1160+ .prepare("DELETE FROM jobs WHERE run_id = ? AND key LIKE ?")
1161+ .bind(&[run.id.as_str().into(), format!("{key}/%").into()])?,
1162+ );
1163+ statements.push(
1164+ self.db
9341165 .prepare(
9351166 "UPDATE jobs SET status = 'waiting', conclusion = NULL, steps = '[]', annotations = '[]', outputs = '{}', reason = NULL,
936− matrix = NULL, token_hash = NULL, seen_at = NULL, started_at = NULL, finished_at = NULL WHERE run_id = ? AND key = ?",
1167+ matrix = NULL, call = NULL, token_hash = NULL, seen_at = NULL, started_at = NULL, finished_at = NULL WHERE run_id = ? AND key = ?",
9371168 )
9381169 .bind(&[run.id.as_str().into(), key.as_str().into()])?,
9391170 );
9861217 let Some(run) = self.run_row(&job.run_id).await? else {
9871218 return Ok(fail(FailureCode::NotFound, "No such run."));
9881219 };
989− let Ok(workflow) = workflow::parse(&run.source) else {
1220+ let Ok(caller) = workflow::parse(&run.source) else {
9901221 return Ok(fail(FailureCode::Invalid, "The workflow no longer reads."));
9911222 };
992− let Some(spec) = workflow.jobs.iter().find(|j| j.id == job.key) else {
993− return Ok(fail(FailureCode::NotFound, "The job is not in the workflow."));
1223+ // A called workflow's job runs as that workflow defines it.
1224+ let callee = job.callee();
1225+ let (workflow, spec, call_inputs) = match callee {
1226+ Some((called, spec, call)) => (called, spec, Some(call["inputs"].clone())),
1227+ None => match caller.jobs.iter().find(|j| j.id == job.key) {
1228+ Some(spec) => (caller.clone(), spec.clone(), None),
1229+ None => return Ok(fail(FailureCode::NotFound, "The job is not in the workflow.")),
1230+ },
9941231 };
1232+ let spec = &spec;
9951233 let repo = repo_path(&run.repo);
9961234 let trusted = run.trusted != 0;
9971235 // GITHUB_TOKEN: the workspace's, for as long as the job may run.
10221260
10231261 let jobs = self.job_rows(&run.id).await?;
10241262 let mut needs = Map::new();
1263+ // In a called workflow, its jobs' keys sit under the job that called it.
1264+ let parent = job.call().filter(|c| c["role"] == "callee").and_then(|c| c["parent"].as_str().map(str::to_owned));
10251265 for need in &spec.needs {
1026− let rows: Vec<&JobRow> = jobs.iter().filter(|row| &row.key == need).collect();
1266+ let key = match &parent {
1267+ Some(parent) => format!("{parent}/{need}"),
1268+ None => need.clone(),
1269+ };
1270+ let rows: Vec<&JobRow> = jobs.iter().filter(|row| row.key == key).collect();
10271271 let mut outputs = Map::new();
10281272 for row in &rows {
10291273 if let Ok(Value::Object(more)) = serde_json::from_str::<Value>(&row.outputs) {
10791323 "contexts": {
10801324 "vars": vars,
10811325 "secrets": secrets,
1082− "inputs": run.inputs(),
1326+ "inputs": call_inputs.unwrap_or_else(|| Value::Object(run.inputs())),
10831327 "matrix": matrix,
10841328 "needs": needs,
10851329 "strategy": {