g1t/services/deployments/src/retries.ts
Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Usage, Billing settings and prepaid AI credit; fixes from the UX audit | 1 | /** |
| 2 | * When a build that keeps failing is tried again, and when it stops being | |
| 3 | * tried. The sweep builds an app again under its new name after a move | |
| 4 | * (see `followMoves`); a build that failed the same way a few times will | |
| 5 | * fail that way again, and one whose commit is gone (force-pushed over, | |
| 6 | * or its branch deleted) never builds, so neither is tried again until | |
| 7 | * something changes: a push, which builds on its own, or a redeploy. | |
| 8 | * | |
| 9 | * Kept free of imports so its tests run on Node as they are. | |
| 10 | */ | |
| 11 | ||
| 12 | /** The same failure, this many times in a row, and a rebuild is not tried again. */ | |
| 13 | export const MAX_IDENTICAL_FAILURES = 3; | |
| 14 | ||
| 15 | /** What git says when the commit asked for is not in the repository. */ | |
| 16 | const MISSING_COMMIT = /reference is not a tree|not our ref|bad object|unknown revision|no such commit|commit no longer exists/i; | |
| 17 | ||
| 18 | /** Whether a build failed because its commit is no longer in the repository. */ | |
| 19 | export function commitMissing(error: string | null | undefined): boolean { | |
| 20 | return !!error && MISSING_COMMIT.test(error); | |
| 21 | } | |
| 22 | ||
| 23 | /** | |
| 24 | * What a build whose commit is gone says, in place of git's own words: | |
| 25 | * the pull request's head was force-pushed over or its branch deleted. | |
| 26 | */ | |
| 27 | export function missingCommitMessage(build: { kind: string; number: number | null }): string { | |
| 28 | if (build.kind === "preview" && build.number != null) { | |
| 29 | return `This pull request's commit no longer exists. Push again, or close pull request #${build.number}.`; | |
| 30 | } | |
| 31 | if (build.kind === "preview") return "This branch's commit no longer exists. Push to the branch again."; | |
| 32 | return "This commit no longer exists in the repository. Push to the default branch again."; | |
| 33 | } | |
| 34 | ||
| 35 | /** One earlier build of an app, newest first, as far as retrying needs it. */ | |
| 36 | export type PastBuild = { status: string; error: string | null; commit_sha: string; created_at: string }; | |
| 37 | ||
| 38 | export type RetryDecision = | |
| 39 | /** Build it now. */ | |
| 40 | | { kind: "build" } | |
| 41 | /** Tried too recently: wait for a later sweep. */ | |
| 42 | | { kind: "wait"; until: number } | |
| 43 | /** Not tried again: its commit is gone, or it failed the same way too often. */ | |
| 44 | | { kind: "stop"; reason: "missing_commit" | "failing" }; | |
| 45 | ||
| 46 | /** | |
| 47 | * Whether to build an app again, from its builds newest first. A build | |
| 48 | * under way is left to finish (the caller checks that first). The run of | |
| 49 | * identical failures at the head of `history` (same commit, same error) | |
| 50 | * decides: a missing commit stops at once; `MAX_IDENTICAL_FAILURES` of | |
| 51 | * them stop; fewer wait `afterMs`, doubled for each failure after the | |
| 52 | * first. With `backoff` off (an event's delivery) only stopping applies. | |
| 53 | */ | |
| 54 | export function retryDecision( | |
| 55 | history: PastBuild[], | |
| 56 | now: number, | |
| 57 | afterMs: number, | |
| 58 | options: { backoff?: boolean; max?: number } = {}, | |
| 59 | ): RetryDecision { | |
| 60 | const last = history[0]; | |
| 61 | if (!last) return { kind: "build" }; | |
| 62 | const max = options.max ?? MAX_IDENTICAL_FAILURES; | |
| 63 | if (last.status === "failed") { | |
| 64 | if (commitMissing(last.error)) return { kind: "stop", reason: "missing_commit" }; | |
| 65 | let run = 0; | |
| 66 | for (const build of history) { | |
| 67 | if (build.status !== "failed" || build.error !== last.error || build.commit_sha !== last.commit_sha) break; | |
| 68 | run++; | |
| 69 | } | |
| 70 | if (run >= max) return { kind: "stop", reason: "failing" }; | |
| 71 | if (!options.backoff) return { kind: "build" }; | |
| 72 | const until = Date.parse(last.created_at) + afterMs * 2 ** (run - 1); | |
| 73 | return until <= now ? { kind: "build" } : { kind: "wait", until }; | |
| 74 | } | |
| 75 | if (!options.backoff) return { kind: "build" }; | |
| 76 | // Refused or skipped: tried again after `afterMs`, as before. | |
| 77 | const until = Date.parse(last.created_at) + afterMs; | |
| 78 | return until <= now ? { kind: "build" } : { kind: "wait", until }; | |
| 79 | } |