A g1t.page build opened by a workflow job no longer stays Building for 45 minutes when the job is cancelled. Whenever a job that ran ends, with any conclusion, the actions service tells deployments (`workflow_job_ended`), which fails every build that job opened with `start_from_workflow` and left queued or building, saying how the job ended ("The workflow job was cancelled before the build finished."), with the commit's `g1t / deploy` check and `deployment.failed` as for any failed build; a build the job finished is left alone, and the sweep still catches one whose job's end was never heard. Builds are found by an index on their job. The Deployments guide says so, with the message in its troubleshooting table.
- Claude-Session:
- https://claude.ai/code/session_01PnwTRSNaC7vEHTAuBzjhrn
7 files+87−20/7 viewed
| 271 | 271 | uploads exactly as [What deploys](#what-deploys) says; its output is in | |
| 272 | 272 | the step's log and on the deployment's page. When the build fails, or | |
| 273 | 273 | g1t does not take what it built, the step fails and the deployment says | |
| 274 | − | why. | |
| 274 | + | why. When the job ends before the build has finished, because it was | |
| 275 | + | cancelled, failed in an earlier step or timed out, the deployment fails | |
| 276 | + | as the job ends, saying how it ended, rather than staying **Building**. | |
| 275 | 277 | ||
| 276 | 278 | - **Secrets.** A pull request from a fork or from someone without the | |
| 277 | 279 | Write role, and a run that had to be | |
| ⋯ | |||
| 620 | 622 | | g1t/deploy says "The job's token needs `deployments: write`" | Add `deployments: write` to the workflow's or the job's `permissions:`. A pull request's run from outside gets a read-only token and cannot deploy. | | |
| 621 | 623 | | g1t/deploy says "Name the project to build" | The repository has several projects with Deployments on: give `project:`. | | |
| 622 | 624 | | g1t/deploy says "g1t.page deploys branches and pull requests" | The run is for a tag. Run the workflow on a branch, such as on a push to `main`. | | |
| 625 | + | | "The workflow job was cancelled before the build finished" | The job that opened the build (`g1t/deploy`) was cancelled, or ended another way the message names, before `g1t-runner deploy` reported. Run the job again. | | |
| 623 | 626 | | "The build did not finish in 45 minutes" | The build stopped reporting: a build stops at 30 minutes, and one not heard from after 45 is failed. Make the build faster, or build less for previews with **Build command**. | | |
| 624 | 627 | ||
| 625 | 628 | ## Running your own g1t | |
| 1680 | 1680 | let Some(job) = finished else { return Ok(()) }; | |
| 1681 | 1681 | // Its G1T_TOKEN stops working with it. | |
| 1682 | 1682 | self.revoke_job_tokens(&job.id).await; | |
| 1683 | + | // A g1t.page build it opened and did not finish fails now, not at | |
| 1684 | + | // the deployments sweep. Only a job that ran can have opened one. | |
| 1685 | + | if job.started_at.is_some() { | |
| 1686 | + | self.end_job_builds(&job.id, conclusion).await; | |
| 1687 | + | } | |
| 1683 | 1688 | // A job that deploys failed: so did its run's deployment, now. | |
| 1684 | 1689 | if conclusion == "failure" && job.continue_on_error == 0 && job.started_at.is_some() | |
| 1685 | 1690 | && let Some(run) = self.run_row(&job.run_id).await? | |
| ⋯ | |||
| 1774 | 1779 | .await?; | |
| 1775 | 1780 | if stopped.as_ref().is_some_and(|row| row.started_at.is_some()) { | |
| 1776 | 1781 | self.revoke_job_tokens(&job.id).await; | |
| 1782 | + | self.end_job_builds(&job.id, "cancelled").await; | |
| 1777 | 1783 | } | |
| 1778 | 1784 | if let Some(stopped) = stopped.filter(|row| row.runner_id.is_some()) { | |
| 1779 | 1785 | self.released(&stopped).await?; | |
| ⋯ | |||
| 1871 | 1877 | Ok(()) | |
| 1872 | 1878 | } | |
| 1873 | 1879 | ||
| 1880 | + | /// Tells the deployments service that a job ended (`workflow_job_ended`), | |
| 1881 | + | /// so a build it opened with `start_from_workflow` and left open fails | |
| 1882 | + | /// at once. Never fails the job: an answer that does not come is | |
| 1883 | + | /// logged, and the sweep fails the build later. | |
| 1884 | + | async fn end_job_builds(&self, job_id: &str, conclusion: &str) { | |
| 1885 | + | let ended: Result<Value> = | |
| 1886 | + | g1t_kit::call(&self.deployments, "workflow_job_ended", &json!({ "jobId": job_id, "conclusion": conclusion })).await; | |
| 1887 | + | if let Err(error) = ended { | |
| 1888 | + | worker::console_log!("actions: could not tell deployments that job {job_id} ended: {error}"); | |
| 1889 | + | } | |
| 1890 | + | } | |
| 1891 | + | ||
| 1874 | 1892 | /// Tells the deployments service how a run's deployment to `env` stands: | |
| 1875 | 1893 | /// made when its first job naming the environment starts, failed when one | |
| 1876 | 1894 | /// of them fails, and settled (`last`) when the run finishes. Never fails | |
| 1 | + | -- A workflow job's open builds are found by its id when it ends | |
| 2 | + | -- (`workflow_job_ended`), to be failed if the job did not finish them. | |
| 3 | + | CREATE INDEX IF NOT EXISTS deployments_job ON deployments (job_id) WHERE job_id IS NOT NULL; |
| 91 | 91 | import { commitMissing, MAX_IDENTICAL_FAILURES, missingCommitMessage, retryDecision, type PastBuild } from "./retries"; | |
| 92 | 92 | import { appHost, appUrl, label, uniqueLabel } from "./names"; | |
| 93 | 93 | import { RepoDeployments, sourceOf } from "./repo-deployments"; | |
| 94 | − | import { NEEDS_WRITE, commitOf, jobOf, projectNamed, readOnlyAllowed, runTrusted, targetOf, type WorkflowBuild } from "./workflow-builds"; | |
| 94 | + | import { NEEDS_WRITE, commitOf, jobEndedMessage, jobOf, projectNamed, readOnlyAllowed, runTrusted, targetOf, type WorkflowBuild } from "./workflow-builds"; | |
| 95 | 95 | ||
| 96 | 96 | type Env = { | |
| 97 | 97 | DB: D1Database; | |
| ⋯ | |||
| 1030 | 1030 | }); | |
| 1031 | 1031 | } | |
| 1032 | 1032 | ||
| 1033 | + | /** | |
| 1034 | + | * For the actions service, when a workflow job ends (`finish_job`), with | |
| 1035 | + | * any conclusion: a build the job opened and did not finish or fail is | |
| 1036 | + | * failed now, saying how the job ended, rather than left building until | |
| 1037 | + | * the sweep's 45 minutes. A build the job finished is left as it is. | |
| 1038 | + | */ | |
| 1039 | + | async workflowJobEnded(a: { jobId?: unknown; conclusion?: unknown }): Promise<Result<number>> { | |
| 1040 | + | if (typeof a.jobId !== "string" || !a.jobId) return fail("invalid", "No job."); | |
| 1041 | + | const open = await this.db | |
| 1042 | + | .prepare("SELECT id FROM deployments WHERE job_id = ? AND status IN ('queued', 'building')") | |
| 1043 | + | .bind(a.jobId) | |
| 1044 | + | .all<{ id: string }>(); | |
| 1045 | + | const message = jobEndedMessage(typeof a.conclusion === "string" ? a.conclusion : ""); | |
| 1046 | + | for (const { id } of open.results) await this.finishFailed(id, message, null, null); | |
| 1047 | + | return ok(open.results.length); | |
| 1048 | + | } | |
| 1049 | + | ||
| 1033 | 1050 | private async deployProduction(project: Project, commit: string | null, createdBy: string): Promise<Result<Deployment> | null> { | |
| 1034 | 1051 | const settings = await this.settingsRow(project.id); | |
| 1035 | 1052 | if (!settings?.enabled || !settings.production || settings.repo_deleted_at || settings.workspace_deleted_at) return null; | |
| ⋯ | |||
| 2429 | 2446 | // For the API: a g1t Actions job opening a build it runs itself. | |
| 2430 | 2447 | case "start_from_workflow": | |
| 2431 | 2448 | return service.startFromWorkflow(args); | |
| 2449 | + | // For the actions service: a job ended, so its open builds end too. | |
| 2450 | + | case "workflow_job_ended": | |
| 2451 | + | return service.workflowJobEnded(args); | |
| 2432 | 2452 | case "take_down": | |
| 2433 | 2453 | return service.takeDown(args); | |
| 2434 | 2454 | case "overview": | |
| 304 | 304 | assert.equal(await refused({}), "conflict"); | |
| 305 | 305 | assert.ok(!w.state.calls.includes("RUNNER start_deploy")); | |
| 306 | 306 | }); | |
| 307 | + | ||
| 308 | + | test("a job that ends without finishing its build fails the build at once, and leaves a finished one be", async () => { | |
| 309 | + | const build = (await open()).value; | |
| 310 | + | const ended = await post("/rpc/workflow_job_ended", { jobId: "job_1", conclusion: "cancelled" }); | |
| 311 | + | assert.deepEqual(ended, { ok: true, value: 1 }); | |
| 312 | + | assert.deepEqual(rows("SELECT id, status, error FROM deployments"), [ | |
| 313 | + | { id: build.deploy_id, status: "failed", error: "The workflow job was cancelled before the build finished." }, | |
| 314 | + | ]); | |
| 315 | + | // Another job's build, and one that is no longer open, are left alone. | |
| 316 | + | w.state.sqlite.prepare("UPDATE deployments SET job_id = 'job_2', status = 'building', error = NULL").run(); | |
| 317 | + | assert.deepEqual(await post("/rpc/workflow_job_ended", { jobId: "job_1", conclusion: "success" }), { ok: true, value: 0 }); | |
| 318 | + | assert.deepEqual(rows("SELECT status FROM deployments"), [{ status: "building" }]); | |
| 319 | + | w.state.sqlite.prepare("UPDATE deployments SET status = 'ready'").run(); | |
| 320 | + | assert.deepEqual(await post("/rpc/workflow_job_ended", { jobId: "job_2", conclusion: "success" }), { ok: true, value: 0 }); | |
| 321 | + | assert.deepEqual(rows("SELECT status FROM deployments"), [{ status: "ready" }]); | |
| 322 | + | assert.equal((await post("/rpc/workflow_job_ended", {})).error.code, "invalid"); | |
| 323 | + | }); |
| 103 | 103 | assert.ok(!readOnlyAllowed(own, { kind: "pull", number: 4 })); | |
| 104 | 104 | assert.ok(!readOnlyAllowed(managed, { error: "a tag" })); | |
| 105 | 105 | }); | |
| 106 | + | ||
| 107 | + | test("a build its job left open says how the job ended", async () => { | |
| 108 | + | const { jobEndedMessage } = await import("./workflow-builds.ts"); | |
| 109 | + | assert.equal(jobEndedMessage("cancelled"), "The workflow job was cancelled before the build finished."); | |
| 110 | + | assert.equal(jobEndedMessage("failure"), "The workflow job failed before the build finished."); | |
| 111 | + | assert.equal(jobEndedMessage("success"), "The workflow job ended without finishing the build."); | |
| 112 | + | }); |
| 103 | 103 | } | |
| 104 | 104 | ||
| 105 | 105 | /** | |
| 106 | + | * Why a build its workflow job left open failed, from how the job ended | |
| 107 | + | * (its conclusion: `cancelled`, `failure`, `success`, …). | |
| 108 | + | */ | |
| 109 | + | export function jobEndedMessage(conclusion: string): string { | |
| 110 | + | switch (conclusion) { | |
| 111 | + | case "cancelled": | |
| 112 | + | return "The workflow job was cancelled before the build finished."; | |
| 113 | + | case "failure": | |
| 114 | + | return "The workflow job failed before the build finished."; | |
| 115 | + | case "timed_out": | |
| 116 | + | return "The workflow job timed out before the build finished."; | |
| 117 | + | default: | |
| 118 | + | return "The workflow job ended without finishing the build."; | |
| 119 | + | } | |
| 120 | + | } | |
| 121 | + | ||
| 122 | + | /** | |
| 106 | 123 | * The project to build: the one named, or the repository's one project | |
| 107 | 124 | * with Deployments on when none is named. | |
| 108 | 125 | */ |
Comments
Sign in to comment.