Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp

Commit

A deploy no longer fails because Cloudflare's API stumbled once, and when one does fail it says why. Each of the last few deploys failed at one service's `wrangler deploy` (the runner, then agents and models), and the log showed only the trace of the request that failed, cut off before its reason. Now a `wrangler deploy` that fails for anything but a mistake in what is being deployed (a build that fails, a configuration that does not read) is tried again after 20 seconds and then a minute, which is harmless since deploying the same version again changes nothing, and each failure is reported by Wrangler's own error block, which carries the reason, rather than its last lines.

syntaqxcommitted Parent4f03002Browse files
3 files+80−20/3 viewed
+13−2
3131 annotation,
3232 apiAuth,
3333 applyMigrations,
34+ DEPLOY_RETRY_WAITS_MS,
3435 dockerAvailable,
3536 exec,
3637 jsonFrom,
3738 lastLines,
3839 pendingMigrations,
3940 readLive,
41+ retryableDeploy,
4042 versionFrom,
4143 wrangler,
4244 wranglerEnv,
45+ wranglerError,
4346 } from "./deploy/cloudflare.mjs";
4447 import {
4548 baseInputs,
334337 if (!rebuildImage && !rebuildBase && !decision.image) args.push("--containers-rollout", "none");
335338 result.note = image.note;
336339 }
337− const deployed = await wrangler(args, { cwd, onLine: out.line });
338− if (deployed.code !== 0) throw new Error(`wrangler deploy failed:\n${lastLines(deployed.out)}`);
340+ // Tried again after a wait when Cloudflare, not what is deployed, is
341+ // why it failed: deploying the same version again is harmless.
342+ let deployed = await wrangler(args, { cwd, onLine: out.line });
343+ for (const wait of DEPLOY_RETRY_WAITS_MS) {
344+ if (deployed.code === 0 || dryRun || !retryableDeploy(deployed.out)) break;
345+ out.line(`wrangler deploy failed; trying again in ${wait / 1000}s:\n${wranglerError(deployed.out)}`);
346+ await new Promise((resolve) => setTimeout(resolve, wait));
347+ deployed = await wrangler(args, { cwd, onLine: out.line });
348+ }
349+ if (deployed.code !== 0) throw new Error(`wrangler deploy failed:\n${wranglerError(deployed.out)}`);
339350 result.version = dryRun ? "(dry run)" : versionFrom(deployed.out);
340351 result.ok = true;
341352 } catch (error) {
+34−0
365365 export function lastLines(text, count = 12) {
366366 return text.trim().split(/\r?\n/).slice(-count).join("\n");
367367 }
368+
369+/**
370+ * What a failed Wrangler command said went wrong: its last `✘ [ERROR]`
371+ * block, which says why, rather than its last lines, which after an API
372+ * error are the trace of the request that failed and cut its reason off.
373+ * The last lines when there is no such block.
374+ */
375+export function wranglerError(out, count = 20) {
376+ const lines = out.trim().split(/\r?\n/);
377+ let at = -1;
378+ for (let i = lines.length - 1; i >= 0; i--) {
379+ if (/\[ERROR\]/.test(lines[i])) {
380+ at = i;
381+ break;
382+ }
383+ }
384+ return at < 0 ? lastLines(out, count) : lines.slice(at, at + count).join("\n");
385+}
386+
387+/**
388+ * Whether a failed `wrangler deploy` is worth trying again: anything but a
389+ * mistake in what is deployed (a build that fails, a configuration that
390+ * does not read), which fails the same way every time. Cloudflare's API
391+ * answering an error, timing out or dropping the connection, which it does
392+ * now and then when many Workers deploy one after another, is not.
393+ */
394+export function retryableDeploy(out) {
395+ const error = wranglerError(out);
396+ const ours = /Build failed|Could not resolve|No loader is configured|is not exported by|Unexpected token|Processing .*wrangler\.jsonc? configuration|is not a valid|Missing entry-point|The entry-point file .* was not found/i;
397+ return !ours.test(error);
398+}
399+
400+/** The waits before the second and third tries of a `wrangler deploy` that failed. */
401+export const DEPLOY_RETRY_WAITS_MS = [20_000, 60_000];
+33−0
3636 pendingFrom,
3737 pendingMigrations,
3838 readLive,
39+ retryableDeploy,
3940 versionFrom,
41+ wranglerError,
4042 } from "./cloudflare.mjs";
4143 import { changedNames, parseCargoLock, parseNpmLock, reaches } from "./lockfiles.mjs";
4244 import { decide, git, planJson, pool } from "./plan.mjs";
812814 assert.equal(settle, null);
813815 assert.equal(called, false);
814816 });
817+
818+// What a deploy said when Cloudflare's API failed under it: the reason is
819+// in the ✘ [ERROR] block, and the last lines are only the request's trace.
820+const API_FAILED = [
821+ "Uploaded g1t-runner (12.31 sec)",
822+ "✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/g1t-runner/versions) failed.",
823+ "",
824+ " Service unavailable [code: 10013]",
825+ "",
826+ " If you think this is a bug, please open an issue at: https://github.com/cloudflare/workers-sdk/issues/new/choose",
827+ "",
828+ ' "result": {',
829+ ' "id": "g1t-runner",',
830+ " GET /accounts/abc/workers/services/g1t-runner -> 200 OK",
831+ " Cloudflare Ray ID: a49663530d5eebe3-SJC",
832+].join("\n");
833+
834+test("a failed wrangler command is reported by its error, not its trace", () => {
835+ const said = wranglerError(API_FAILED);
836+ assert.match(said, /^✘ \[ERROR\] A request to the Cloudflare API/);
837+ assert.match(said, /Service unavailable \[code: 10013\]/);
838+ // No error block: its last lines, as before.
839+ assert.equal(wranglerError("one\ntwo\nthree", 2), "two\nthree");
840+});
841+
842+test("Cloudflare failing is tried again; a mistake in what is deployed is not", () => {
843+ assert.equal(retryableDeploy(API_FAILED), true);
844+ assert.equal(retryableDeploy("✘ [ERROR] Build failed with 1 error:\n src/index.ts:3:7: ERROR: Could not resolve \"./nope\""), false);
845+ assert.equal(retryableDeploy("✘ [ERROR] Processing wrangler.jsonc configuration:\n - \"main\" is not a valid path"), false);
846+ assert.equal(retryableDeploy("fetch failed: ECONNRESET"), true);
847+});

Comments

Sign in to comment.