flagon-io/g1t

public

Runner: each sweep starts a few of the queued nightly backups

A backup's sandbox holds only its job's token; it has a 60-minute time cap, is never metered to the workspace, and one that dies before it reports is failed back to the repos service, which tries again later. BACKUPS_PER_SWEEP (4) and BACKUPS_RUNNING (6) pace it; 0 turns it off.

syntaqxcommitted 13h agoParent839004fBrowse files

6 files+175−60/6 viewed

+3−0View BlameViewed

292	292		commitFile: (repo, actor, file) => call("commit_file", { repo, actor, ...file }),
293	293		land: (sourceId, actor, branch) => call("land", { sourceId, actor, branch }),
294	294		compare: (repoId, viewer, base, head) => call("compare", { repoId, viewer, base, head }),
	295	+	claimBackups: (limit, maxRunning) => call("claim_backups", { limit, maxRunning }),
	296	+	// The sandbox's own calls are snake_case (they come through the API).
	297	+	failBackup: (jobId, token, error) => call("backup_fail", { job_id: jobId, token, error }),
295	298		};
296	299		}
297	300

+24−0View BlameViewed

277	277		* branch with the point where it left the default branch.
278	278		*/
279	279		compare(repoId: string, viewer: Viewer, base?: string \| null, head?: string \| null): Promise<Result<Comparison>>;
	280	+
	281	+	/**
	282	+	* Services only, for the runner's sweep: up to `limit` queued nightly
	283	+	* backups, each now running with a token of its own, so long as no more
	284	+	* than `maxRunning` are then running. Empty when backups are off.
	285	+	*/
	286	+	claimBackups(limit: number, maxRunning: number): Promise<BackupClaim[]>;
	287	+
	288	+	/**
	289	+	* Services only: a backup's sandbox stopped before it reported, so the
	290	+	* job is tried again later. Refused harmlessly once it has reported.
	291	+	*/
	292	+	failBackup(jobId: string, token: string, error: string): Promise<Result<boolean>>;
280	293		}
281	294
	295	+	/**
	296	+	* A nightly backup to start (`g1t_contracts::backups`): the sandbox is
	297	+	* given the job's id and token, and nothing else.
	298	+	*/
	299	+	export type BackupClaim = {
	300	+	jobId: string;
	301	+	token: string;
	302	+	repoId: string;
	303	+	path: RepoPath;
	304	+	};
	305	+
282	306		/** Lines `start` to `end` (inclusive, from 1) last changed by `commit`. */
283	307		export type BlameRange = { start: number; end: number; commit: string };
284	308

+36−0View BlameViewed

1	+	import assert from "node:assert/strict";
2	+	import { test } from "node:test";
3	+
4	+	import type { BackupClaim } from "@g1t/contracts";
5	+
6	+	import { backupEnv, backupPace, backupSandboxName } from "./backup.ts";
7	+
8	+	const claim: BackupClaim = {
9	+	jobId: "bkp_01",
10	+	token: "secret-token",
11	+	repoId: "repo_1",
12	+	path: { namespace: "acme", name: "rocket" },
13	+	};
14	+
15	+	test("the sandbox gets its job and token, and no git credential", () => {
16	+	const env = backupEnv(claim, "https://api.g1t.sh");
17	+	assert.deepEqual(env, {
18	+	MODE: "backup",
19	+	G1T_API: "https://api.g1t.sh",
20	+	BACKUP_JOB: "bkp_01",
21	+	BACKUP_TOKEN: "secret-token",
22	+	});
23	+	assert.ok(!("G1T_TOKEN" in env) && !("GIT_REMOTE" in env));
24	+	});
25	+
26	+	test("one sandbox per job", () => {
27	+	assert.equal(backupSandboxName(claim), "backup-bkp_01");
28	+	assert.notEqual(backupSandboxName({ ...claim, jobId: "bkp_02" }), backupSandboxName(claim));
29	+	});
30	+
31	+	test("the pace comes from the variables, and 0 turns backups off", () => {
32	+	assert.deepEqual(backupPace(undefined, undefined), { perSweep: 4, running: 6 });
33	+	assert.deepEqual(backupPace("2", "3"), { perSweep: 2, running: 3 });
34	+	assert.deepEqual(backupPace("0", "6"), { perSweep: 0, running: 6 });
35	+	assert.deepEqual(backupPace("many", "-1"), { perSweep: 4, running: 6 });
36	+	});

+48−0View BlameViewed

1	+	/**
2	+	* Nightly backups (`RunnerService.startBackups`): each sweep claims a few
3	+	* of the backups the repos service queued and starts a sandbox for each,
4	+	* which runs the runner's `backup` mode (crates/runner backup.rs). The
5	+	* flow is in `crates/contracts/src/backups.rs`. Pure, so it is tested on
6	+	* its own.
7	+	*/
8	+	import type { BackupClaim } from "@g1t/contracts";
9	+
10	+	/** A backup's time cap: a clone and a bundle of at most 1 GB. */
11	+	export const BACKUP_MINUTES = 60;
12	+
13	+	/** How many backups one sweep starts, and how many may run at once. */
14	+	export type BackupPace = { perSweep: number; running: number };
15	+
16	+	const DEFAULT_PACE: BackupPace = { perSweep: 4, running: 6 };
17	+
18	+	function count(value: string \| undefined, fallback: number): number {
19	+	if (value === undefined \|\| value.trim() === "") return fallback;
20	+	const parsed = Number(value);
21	+	return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
22	+	}
23	+
24	+	/**
25	+	* BACKUPS_PER_SWEEP and BACKUPS_RUNNING, or their defaults. `0` per sweep
26	+	* starts none: backups are off.
27	+	*/
28	+	export function backupPace(perSweep: string \| undefined, running: string \| undefined): BackupPace {
29	+	return {
30	+	perSweep: count(perSweep, DEFAULT_PACE.perSweep),
31	+	running: count(running, DEFAULT_PACE.running),
32	+	};
33	+	}
34	+
35	+	/** One sandbox per job: asking twice starts nothing twice. */
36	+	export function backupSandboxName(claim: BackupClaim): string {
37	+	return `backup-${claim.jobId}`;
38	+	}
39	+
40	+	/** What the sandbox is started with: its job, and nothing that reads git. */
41	+	export function backupEnv(claim: BackupClaim, api: string): Record<string, string> {
42	+	return {
43	+	MODE: "backup",
44	+	G1T_API: api,
45	+	BACKUP_JOB: claim.jobId,
46	+	BACKUP_TOKEN: claim.token,
47	+	};
48	+	}

+57−4View BlameViewed

77	77		import { hostedOpen } from "./hosted";
78	78		import { delegateInput, noModelMessage, notStarted, queued, started } from "./delegate";
79	79		import { BUMP_MINUTES, BUMP_TOKEN_TTL_SECONDS, bumpEnv, bumpProblem, bumpSandboxName, systemActor } from "./bump";
	80	+	import { BACKUP_MINUTES, backupEnv, backupPace, backupSandboxName } from "./backup";
80	81		import { type ProjectSurroundings, readableSurroundings } from "./surroundings";
81	82		import { holdCredentials, pushGrant, remotePath, revokeCredentials, runCredential } from "./credentials";
82	83		import { buildMentionPrompt, describeThread, handleMention, planMention } from "./mentions";
⋯
180	181		* either way.
181	182		*/
182	183		ABUSE_WATCH?: string;
	184	+	/**
	185	+	* Nightly backups (backup.ts): how many queued backups one sweep starts
	186	+	* (`0`: none, backups off here), and how many may run at once.
	187	+	*/
	188	+	BACKUPS_PER_SWEEP?: string;
	189	+	BACKUPS_RUNNING?: string;
183	190		}
184	191
185	192		/**
⋯
226	233		* its branch. The security service opens the pull request when it hears
227	234		* the push, so a failure has no one to tell.
228	235		*/
229		−	\| { kind: "bump"; repo: RepoPath; branch: string };
	236	+	\| { kind: "bump"; repo: RepoPath; branch: string }
	237	+	/**
	238	+	* A repository's nightly backup: a bundle cut and sent to the repos
	239	+	* service. g1t's own work, never charged to the workspace.
	240	+	*/
	241	+	\| { kind: "backup"; jobId: string; token: string };
230	242		/**
231	243		* Whose sandbox time it is, reported when the sandbox stops, and the
232	244		* machine it ran on when it was not the standard one.
⋯
284	296		return "workflow";
285	297		case "deploy":
286	298		return "deploy";
	299	+	// Not metered: a backup is g1t's own cost.
	300	+	case "backup":
	301	+	return null;
287	302		default:
288	303		return "agent";
289	304		}
⋯
449	464		if (build) delete harness.GUARDRAILS;
450	465		const watch: Record<string, string> = this.env.ABUSE_WATCH === "off" ? { G1T_ABUSE: "off" } : {};
451	466		await this.start({ envVars: { ...vars, ...harness, ...watch }, enableInternet: !restricted });
452		−	if (guard) {
453		−	await this.ctx.storage.put("timeCap", guard.minutes);
454		−	await this.schedule(guard.minutes * 60 + ALARM_GRACE_SECONDS, "timeUp");
	467	+	// A backup has no guardrails, but still a time cap.
	468	+	const cap = guard?.minutes ?? (run.kind === "backup" ? BACKUP_MINUTES : null);
	469	+	if (cap) {
	470	+	await this.ctx.storage.put("timeCap", cap);
	471	+	await this.schedule(cap * 60 + ALARM_GRACE_SECONDS, "timeUp");
455	472		}
456	473		} catch (error) {
457	474		await revokeCredentials(this.env.IDENTITY, this.ctx.storage, this.env.INTEGRATIONS);
⋯
677	694		}
678	695		// Nothing was pushed, so no pull request opens; why is in its log.
679	696		if (run.kind === "bump") return;
	697	+	if (run.kind === "backup") {
	698	+	// Refused harmlessly if the sandbox reported before it stopped; the
	699	+	// job is otherwise tried again later tonight.
	700	+	await reposClient(this.env.REPOS)
	701	+	.failBackup(run.jobId, run.token, why ?? `The sandbox exited with ${exitCode}.`)
	702	+	.catch((error: unknown) => console.log("backup failure not reported", run.jobId, String(error)));
	703	+	return;
	704	+	}
680	705		if (run.kind === "deploy") {
681	706		// Refused harmlessly if the build reported its end before it stopped.
682	707		await this.env.DEPLOYMENTS.fetch(`https://deployments/jobs/${run.deployId}/fail`, {
⋯
1880	1905		await this.drainWaits();
1881	1906		await this.advanceAll();
1882	1907		await this.startReady();
	1908	+	await this.startBackups().catch((error: unknown) => console.log("backups not started", String(error)));
	1909	+	}
	1910	+
	1911	+	/**
	1912	+	* Starts a few of the nightly backups the repos service queued, each in
	1913	+	* a sandbox of its own that holds only its job's token: the sandbox asks
	1914	+	* for a read-only git credential itself, when it is ready to clone. No
	1915	+	* plan is asked and nothing is metered: backups are g1t's own work.
	1916	+	*/
	1917	+	private async startBackups(): Promise<void> {
	1918	+	const pace = backupPace(this.env.BACKUPS_PER_SWEEP, this.env.BACKUPS_RUNNING);
	1919	+	if (pace.perSweep === 0) return;
	1920	+	const repos = reposClient(this.env.REPOS);
	1921	+	for (const claim of await repos.claimBackups(pace.perSweep, pace.running)) {
	1922	+	try {
	1923	+	const sandbox = this.env.SANDBOX.get(this.env.SANDBOX.idFromName(backupSandboxName(claim)));
	1924	+	await sandbox.run({
	1925	+	kind: "backup",
	1926	+	jobId: claim.jobId,
	1927	+	token: claim.token,
	1928	+	// For `abuse.flagged`: whose repository it was.
	1929	+	owner: { workspace: claim.path.namespace, repo: `${claim.path.namespace}/${claim.path.name}` },
	1930	+	envVars: backupEnv(claim, "https://api.g1t.sh"),
	1931	+	});
	1932	+	} catch (error) {
	1933	+	await repos.failBackup(claim.jobId, claim.token, `The sandbox could not start: ${String(error)}`).catch(() => null);
	1934	+	}
	1935	+	}
1883	1936		}
1884	1937
1885	1938		/**

+7−2View BlameViewed

63	63		{ "binding": "EVENTS", "service": "g1t-events" }
64	64		],
65	65		// A sweep for lifecycle steps whose trigger was missed or whose sandbox
66		−	// died before reporting.
	66	+	// died before reporting, which also starts queued nightly backups.
67	67		"triggers": { "crons": ["/5 * * *"] },
68	68		// Events it reacts to: a pull request ready for review, or its head moving.
69	69		"queues": {
⋯
98	98		// Sandboxes stop themselves when they look like they are mining
99	99		// (crates/runner abuse.rs). "off" turns the CPU watch off; miners
100	100		// named in commands are refused either way.
101		−	"ABUSE_WATCH": "on"
	101	+	"ABUSE_WATCH": "on",
	102	+	// Nightly backups (src/backup.ts): each sweep starts this many of the
	103	+	// backups the repos service queued, with at most BACKUPS_RUNNING at
	104	+	// once. "0" starts none.
	105	+	"BACKUPS_PER_SWEEP": "4",
	106	+	"BACKUPS_RUNNING": "6"
102	107		},
103	108		"observability": { "enabled": true }
104	109		}