flagon-io/g1t

public

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh

git agents cloudflare rust forge mcp

Commit

Every npm install in g1t's workflows starts from npm's downloads for the whole lockfile, rather than from Wrangler's alone. All of them restored and saved one key, npm-Linux-<lockfile hash>, and the first job to finish on main (Deploy's check, which installs only the root) saved its 73 MB under it; a key is written once, so every later job restored those 73 MB and downloaded the rest of its install through the runner's egress, where downloads were breaking off. The key is now npm-full-, and only jobs that install the whole lockfile save it: CI's TypeScript and Build jobs, and the desktop release's Linux builds. Every job that installs part of it (Deploy's check, migrate and stages, the mobile, CLI and desktop publish jobs, the runner release) only restores. On main, Deploy's new npm-cache job keeps it: it looks the key up without downloading, and when this lockfile has none it starts from the last lockfile's, downloads every package with scripts off and saves them; nothing waits for it, and a failed download leaves the cache as it was and the run green. A test holds every workflow to one npm key and to saving it only from whole installs.

syntaqxcommitted yesterdayParent1bb793aBrowse files

70	70		timeout-minutes: 20
71	71		steps:
72	72		- uses: actions/checkout@v5
73		−	# npm's downloads, kept between runs: an install reads them first
74		−	# and asks the registry only for what changed, so a dropped
75		−	# connection to it rarely stops the job.
	73	+	# npm's downloads for the whole lockfile, which the npm-cache job
	74	+	# keeps: an install reads them first and asks the registry only for
	75	+	# what changed, so a dropped connection to it rarely stops the job.
	76	+	# Only restored: a job that installs part of the lockfile never
	77	+	# saves, or its part would stand in for the whole under that key.
76	78		- name: Cache npm's downloads
77		−	uses: actions/cache@v4
	79	+	uses: actions/cache/restore@v4
78	80		with:
79	81		path: ~/.npm
80		−	key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
81		−	restore-keys: npm-${{ runner.os }}-
	82	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	83	+	restore-keys: npm-full-${{ runner.os }}-
82	84		- name: Install Wrangler
83	85		run: npm ci --prefer-offline --workspaces=false --no-audit --no-fund
84	86		- name: The manifest matches every wrangler.jsonc
⋯
86	88		- name: The deploy tool's tests
87	89		run: npm run test:deploy
88	90
	91	+	# npm's downloads for the whole lockfile, saved once for each lockfile
	92	+	# on main, which every install of this and the other workflows starts
	93	+	# from. Before, every job saved its own part under one key, and the
	94	+	# first to finish (Wrangler alone, 73 MB) stood in for the rest: each
	95	+	# stage then downloaded hundreds of MB through the runner's egress,
	96	+	# 50 MB tarballs among them. Nothing waits for this job, and a failed
	97	+	# download only leaves the cache as it was.
	98	+	npm-cache:
	99	+	name: npm's downloads
	100	+	runs-on: ubuntu-latest
	101	+	timeout-minutes: 20
	102	+	steps:
	103	+	- uses: actions/checkout@v5
	104	+	- name: Look for this lockfile's downloads
	105	+	id: saved
	106	+	uses: actions/cache/restore@v4
	107	+	with:
	108	+	path: ~/.npm
	109	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	110	+	lookup-only: true
	111	+	- name: Start from the last lockfile's
	112	+	if: ${{ steps.saved.outputs.cache-hit != 'true' }}
	113	+	uses: actions/cache/restore@v4
	114	+	with:
	115	+	path: ~/.npm
	116	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	117	+	restore-keys: npm-full-${{ runner.os }}-
	118	+	- name: Download every package
	119	+	id: download
	120	+	if: ${{ steps.saved.outputs.cache-hit != 'true' }}
	121	+	continue-on-error: true
	122	+	run: npm ci --prefer-offline --ignore-scripts --no-audit --no-fund
	123	+	- name: Save them
	124	+	if: ${{ steps.saved.outputs.cache-hit != 'true' && steps.download.outcome == 'success' }}
	125	+	uses: actions/cache/save@v4
	126	+	with:
	127	+	path: ~/.npm
	128	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	129	+
89	130		# Runs beside check: nothing deploys until both have succeeded.
90	131		plan:
91	132		name: Plan
⋯
138	179		# and asks the registry only for what changed, so a dropped
139	180		# connection to it rarely stops the job.
140	181		- name: Cache npm's downloads
141		−	uses: actions/cache@v4
	182	+	uses: actions/cache/restore@v4
142	183		with:
143	184		path: ~/.npm
144		−	key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
145		−	restore-keys: npm-${{ runner.os }}-
	185	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	186	+	restore-keys: npm-full-${{ runner.os }}-
146	187		- name: Install Wrangler
147	188		run: npm ci --prefer-offline --workspaces=false --no-audit --no-fund
148	189		- name: Apply pending migrations
⋯
246	287		- name: sccache
247	288		if: ${{ matrix.rust \|\| matrix.image }}
248	289		run: bash scripts/sccache.sh install
249		−	# npm's downloads, kept between runs as the check and migrate jobs
250		−	# keep them: the install reads them first (--prefer-offline, in
	290	+	# npm's downloads for the whole lockfile, only restored, as the
	291	+	# check job restores them: the install reads them first (--prefer-offline, in
251	292		# scripts/deploy/stack.mjs) and asks the registry only for what
252	293		# changed, so a connection it drops mid-install rarely stops a stage.
253	294		- name: Cache npm's downloads
254		−	uses: actions/cache@v4
	295	+	uses: actions/cache/restore@v4
255	296		with:
256	297		path: ~/.npm
257		−	key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
258		−	restore-keys: npm-${{ runner.os }}-
	298	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	299	+	restore-keys: npm-full-${{ runner.os }}-
259	300		- name: Install
260	301		run: node scripts/deploy.mjs install --only "${{ matrix.units }}"
261	302		- name: Deploy ${{ matrix.units }}

125	125		let concurrency = deploy.concurrency.as_ref().unwrap();
126	126		assert_eq!(concurrency.group, "deploy-production");
127	127		assert_eq!(concurrency.cancel_in_progress, json!(false));
128		−	assert_eq!(deploy.job_order(), ["check", "plan", "migrate", "core", "edge", "front", "smoke"]);
	128	+	assert_eq!(deploy.job_order(), ["check", "npm-cache", "plan", "migrate", "core", "edge", "front", "smoke"]);
	129	+	// Nothing waits for npm's downloads to be saved.
	130	+	assert!(deploy.jobs.iter().all(\|job\| !job.needs.iter().any(\|n\| n == "npm-cache")));
129	131		// The stages share their steps (a YAML alias), and read the token only
130	132		// where they deploy.
131	133		let steps = \|id: &str\| deploy.jobs.iter().find(\|j\| j.id == id).unwrap().steps.len();
⋯
135	137		assert!(!source.contains("cancel-in-progress: true"));
136	138		}
137	139
	140	+	/// Whether a step's `run` installs the whole lockfile: `npm ci` with no
	141	+	/// workspace narrowing it, and not the deploy tool's install of some units.
	142	+	fn installs_everything(run: &str) -> bool {
	143	+	run.split('\n').any(\|line\| {
	144	+	let line = line.trim();
	145	+	line.starts_with("npm ci") && !line.contains(" -w ") && !line.contains("--workspace")
	146	+	})
	147	+	}
	148	+
	149	+	#[test]
	150	+	fn npm_downloads_are_cached_whole_and_only_whole_installs_save_them() {
	151	+	// Every job's npm cache is the whole lockfile's, under one key. A job
	152	+	// that installs part of it only restores: had it saved, its part would
	153	+	// stand in for the whole under that key (the first job to finish used
	154	+	// to save Wrangler alone, and every later install downloaded the rest).
	155	+	let mut saving = Vec::new();
	156	+	for entry in std::fs::read_dir(workflows_dir()).unwrap() {
	157	+	let path = entry.unwrap().path();
	158	+	if path.extension().and_then(\|e\| e.to_str()) != Some("yml") {
	159	+	continue;
	160	+	}
	161	+	let name = path.file_name().unwrap().to_string_lossy().to_string();
	162	+	for job in read(&name).jobs {
	163	+	let whole = job.steps.iter().any(\|s\| s.run.as_deref().is_some_and(installs_everything));
	164	+	for step in &job.steps {
	165	+	let Some(uses) = step.uses.as_deref() else { continue };
	166	+	if !uses.starts_with("actions/cache") \|\| step.raw["with"]["path"] != json!("~/.npm") {
	167	+	continue;
	168	+	}
	169	+	let key = step.raw["with"]["key"].as_str().unwrap_or_default();
	170	+	assert_eq!(key, "npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}", "{name} {}", job.id);
	171	+	if let Some(prefix) = step.raw["with"]["restore-keys"].as_str() {
	172	+	assert_eq!(prefix, "npm-full-${{ runner.os }}-", "{name} {}", job.id);
	173	+	}
	174	+	let saves = uses.starts_with("actions/cache@") \|\| uses.starts_with("actions/cache/save@");
	175	+	assert!(!saves \|\| whole, "{name} {}: saves npm's cache but installs only part of the lockfile", job.id);
	176	+	if saves {
	177	+	saving.push(format!("{name} {}", job.id));
	178	+	}
	179	+	}
	180	+	}
	181	+	}
	182	+	// main's whole cache is kept by the deploy workflow, beside its check.
	183	+	assert!(saving.contains(&"deploy.yml npm-cache".to_owned()), "{saving:?}");
	184	+	}
	185	+
138	186		#[test]
139	187		fn deploy_stages_follow_one_another() {
140	188		let deploy = read("deploy.yml");

74	74		node scripts/cli-release.mjs sign
75	75		node scripts/cli-release.mjs verify
76	76		- name: Cache npm's downloads
77		−	uses: actions/cache@v4
	77	+	uses: actions/cache/restore@v4
78	78		with:
79	79		path: ~/.npm
80		−	key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
81		−	restore-keys: npm-${{ runner.os }}-
	80	+	key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
	81	+	restore-keys: npm-full-${{ runner.os }}-
82	82		- name: Install Wrangler
83	83		run: npm ci --prefer-offline --workspaces=false --no-audit --no-fund
84	84		- name: Publish to g1t.sh/downloads/cli