flagon-io/g1t

public

Old artifacts in KV are asked for only for a finished run made before the move to R2, after the run is shown, and not at all from 2026-10-22

syntaqxcommitted 3d agoParent2c551c7Browse files

56	56		name: String,
57	57		}
58	58
	59	+	/// Artifacts older runners kept in KV expire 14 days after they were made,
	60	+	/// and none has been made there since artifacts moved to R2 on 2026-10-08:
	61	+	/// from 2026-10-22T00:00Z every one is gone, and KV is not asked (a list is
	62	+	/// the dearest thing KV does). Delete the KV artifact code after that date,
	63	+	/// with its twin in apps/web/app/lib/artifacts.server.ts.
	64	+	const LEGACY_KV_UNTIL_MS: u64 = 1_792_627_200_000;
	65	+
	66	+	/// Whether artifacts kept in KV may still be there at `now`.
	67	+	fn legacy_kv(now: u64) -> bool {
	68	+	now < LEGACY_KV_UNTIL_MS
	69	+	}
	70	+
59	71		fn store(env: &Env) -> Result<KvStore> {
60	72		env.kv("BLOBS")
61	73		}
⋯
404	416		// Artifacts older runners kept in KV, for the days they
405	417		// are still there.
406	418		let legacy_run = run_id.as_deref().unwrap_or(run);
407		−	for (_, meta) in list(kv, &format!("a/{legacy_run}/")).await? {
408		−	if !listed.iter().any(\|a\| a["name"] == meta.name.as_str()) {
409		−	listed.push(json!({ "name": meta.name, "size": meta.size, "format": "tgz" }));
	419	+	if legacy_kv(g1t_kit::now_ms()) {
	420	+	for (_, meta) in list(kv, &format!("a/{legacy_run}/")).await? {
	421	+	if !listed.iter().any(\|a\| a["name"] == meta.name.as_str()) {
	422	+	listed.push(json!({ "name": meta.name, "size": meta.size, "format": "tgz" }));
	423	+	}
410	424		}
411	425		}
412	426		crate::reply(&listed)
⋯
547	561		match found {
548	562		Outcome::Ok(found) => stream(bucket, &found.object, &found.artifact.format).await,
549	563		// One an older runner kept in KV.
550		−	Outcome::Fail(_) => match get(kv, &format!("a/{run}/{name}")).await? {
	564	+	Outcome::Fail(_) if legacy_kv(g1t_kit::now_ms()) => match get(kv, &format!("a/{run}/{name}")).await? {
551	565		Some(bytes) => Response::from_bytes(bytes),
552	566		None => error(404, "No such artifact."),
553	567		},
	568	+	Outcome::Fail(_) => error(404, "No such artifact."),
554	569		}
555	570		}
556	571		_ => error(404, "No such endpoint."),
⋯
612	627		return Response::redirect_with_status(Url::parse(&crate::artifacts::blob_url(&services.addresses.api, &found.blob))?, 302);
613	628		}
614	629		// Kept in KV by an older runner.
	630	+	if !legacy_kv(g1t_kit::now_ms()) {
	631	+	return error(404, "No such artifact, or it has expired.");
	632	+	}
615	633		match get(&store(env)?, &format!("a/{run}/{name}")).await? {
616	634		Some(bytes) => {
617	635		let mut response = Response::from_bytes(bytes)?;
⋯
624	642		}
625	643		}
626	644
	645	+	#[cfg(test)]
	646	+	mod tests {
	647	+	use super::*;
	648	+
	649	+	#[test]
	650	+	fn kv_artifacts_are_not_asked_for_after_they_have_all_expired() {
	651	+	assert_eq!(g1t_contracts::time::rfc3339(LEGACY_KV_UNTIL_MS), "2026-10-22T00:00:00.000Z");
	652	+	assert!(legacy_kv(LEGACY_KV_UNTIL_MS - 1));
	653	+	assert!(!legacy_kv(LEGACY_KV_UNTIL_MS));
	654	+	}
	655	+	}

2	2
3	3		import type { RepoPath, Viewer } from "@g1t/contracts";
4	4
	5	+	import { LEGACY_KV_UNTIL } from "./artifacts";
5	6		import { actions } from "./services.server";
6	7
7	8		/**
8	9		* A run's artifacts, for its page. The actions service lists them (their
9	10		* bytes are in R2, downloaded through the API's signed links); artifacts an
10	11		* older runner kept in KV (`a/{run}/{name}`, its bytes in chunks `…#0`,
11		−	* `…#1`) are listed too until KV expires them.
	12	+	* `…#1`) are listed too until KV expires them, for runs the caller has
	13	+	* already been shown (`legacyArtifactsWorthAsking` in artifacts.ts).
12	14		*/
13	15		export type ArtifactRow = {
14	16		/** The artifact's number; null for one kept in KV. */
⋯
35	37		}));
36	38		}
37	39
	40	+	/** A run's artifacts the actions service keeps, which checks who may see them. */
38	41		export async function listArtifacts(repo: RepoPath, viewer: Viewer, run: string): Promise<ArtifactRow[]> {
39		−	const [kept, legacy] = await Promise.all([
40		−	actions.artifacts(repo, viewer, { run, per_page: 100 }),
41		−	kvArtifacts(run).catch(() => []),
42		−	]);
	42	+	const kept = await actions.artifacts(repo, viewer, { run, per_page: 100 });
43	43		const rows: ArtifactRow[] = kept.ok
44	44		? kept.value.artifacts.map((a) => ({ id: a.id, name: a.name, size: a.size, expiresAt: a.expires_at, createdAt: a.created_at }))
45	45		: [];
	46	+	return rows.sort((a, b) => a.name.localeCompare(b.name));
	47	+	}
	48	+
	49	+	/**
	50	+	* `rows` with the artifacts an older runner kept in KV for `run` added.
	51	+	* Only for a run the viewer was allowed to see. Delete after
	52	+	* 2026-10-22T00:00Z (`LEGACY_KV_UNTIL`).
	53	+	*/
	54	+	export async function withLegacyArtifacts(rows: ArtifactRow[], run: string): Promise<ArtifactRow[]> {
	55	+	if (Date.now() >= LEGACY_KV_UNTIL) return rows;
	56	+	const legacy = await kvArtifacts(run).catch(() => []);
	57	+	const all = [...rows];
46	58		for (const row of legacy) {
47		−	if (!rows.some((r) => r.name === row.name)) rows.push(row);
	59	+	if (!all.some((r) => r.name === row.name)) all.push(row);
48	60		}
49		−	return rows.sort((a, b) => a.name.localeCompare(b.name));
	61	+	return all.sort((a, b) => a.name.localeCompare(b.name));
50	62		}
51	63
52	64		export async function readArtifact(run: string, name: string): Promise<Uint8Array \| null> {
	65	+	// Every artifact kept in KV has expired (artifacts.ts).
	66	+	if (Date.now() >= LEGACY_KV_UNTIL) return null;
53	67		const base = `a/${run}/${name}`;
54	68		const meta = await env.BLOBS.get<Meta>(base, "json");
55	69		if (!meta) return null;

1	1		import assert from "node:assert/strict";
2	2		import { test } from "node:test";
3	3
4		−	import { expiresIn, formatBytes } from "./artifacts.ts";
	4	+	import { expiresIn, formatBytes, legacyArtifactsWorthAsking } from "./artifacts.ts";
5	5
6	6		test("sizes read as KB, MB or GB", () => {
7	7		assert.equal(formatBytes(10), "1 KB");
⋯
18	18		assert.equal(expiresIn("2026-10-08T11:00:00Z", now), "expired");
19	19		assert.equal(expiresIn(null, now), "");
20	20		});
	21	+
	22	+	test("KV is asked for an old, finished run's artifacts only until they have expired", () => {
	23	+	const old = { createdAt: "2026-10-07T12:00:00Z", status: "completed" };
	24	+	const now = Date.parse("2026-10-10T00:00:00Z");
	25	+	assert.ok(legacyArtifactsWorthAsking(old, now));
	26	+	// Still going: its artifacts are in R2, and its page refreshes.
	27	+	assert.ok(!legacyArtifactsWorthAsking({ ...old, status: "in_progress" }, now));
	28	+	// Made after the move to R2.
	29	+	assert.ok(!legacyArtifactsWorthAsking({ ...old, createdAt: "2026-10-09T08:00:00Z" }, now));
	30	+	// Every KV artifact has expired.
	31	+	assert.ok(!legacyArtifactsWorthAsking(old, Date.parse("2026-10-22T00:00:00Z")));
	32	+	});

16	16		if (days === 1) return "expires tomorrow";
17	17		return `expires in ${days} days`;
18	18		}
	19	+
	20	+	/**
	21	+	* Artifacts older runners kept in Workers KV. None has been made there
	22	+	* since artifacts moved to R2 on 2026-10-08, and KV expires each 14 days
	23	+	* after it was made: from 2026-10-22T00:00Z every one is gone. Delete the
	24	+	* KV artifact code (here, artifacts.server.ts, and apps/api/src/blobs.rs)
	25	+	* after that date.
	26	+	*/
	27	+	export const LEGACY_KV_UNTIL = Date.parse("2026-10-22T00:00:00Z");
	28	+	/** Runs made from this time on kept their artifacts in R2 only. */
	29	+	export const LEGACY_KV_BEFORE = Date.parse("2026-10-09T00:00:00Z");
	30	+
	31	+	/**
	32	+	* Whether a run's page asks KV for artifacts an older runner kept there:
	33	+	* only for a finished run made before the move, and only until they have
	34	+	* all expired. A KV list is the dearest thing KV does, and a run still
	35	+	* going refreshes its page every few seconds.
	36	+	*/
	37	+	export function legacyArtifactsWorthAsking(run: { createdAt: string; status: string }, now: number = Date.now()): boolean {
	38	+	return now < LEGACY_KV_UNTIL && run.status === "completed" && Date.parse(run.createdAt) < LEGACY_KV_BEFORE;
	39	+	}

39	39		import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "../../components/ui/dropdown-menu";
40	40		import { Hint } from "../../components/ui/hint";
41	41		import { searchLog } from "../../lib/log-lines";
42		−	import { listArtifacts } from "../../lib/artifacts.server";
43		−	import { expiresIn, formatBytes } from "../../lib/artifacts";
	42	+	import { listArtifacts, withLegacyArtifacts } from "../../lib/artifacts.server";
	43	+	import { expiresIn, formatBytes, legacyArtifactsWorthAsking } from "../../lib/artifacts";
44	44		import { actions } from "../../lib/services.server";
45	45		import { assertSameOrigin, getViewer, requireUser, roleIn, unwrap } from "../../lib/session.server";
46	46		import { accessTo, refusal } from "../../lib/access.server";
⋯
57	57		// An earlier attempt, when one is asked for.
58	58		const asked = Number(new URL(request.url).searchParams.get("attempt") ?? "");
59	59		const attempt = Number.isInteger(asked) && asked > 0 ? asked : undefined;
60		−	const [detail, artifacts, summaries] = await Promise.all([
	60	+	const [detail, kept, summaries] = await Promise.all([
61	61		actions.run(repo, viewer, params.id, attempt).then(unwrap),
62	62		listArtifacts(repo, viewer, params.id).catch(() => []),
63	63		actions
⋯
65	65		.then((found) => (found.ok ? found.value : []))
66	66		.catch((): JobSummary[] => []),
67	67		]);
	68	+	// Artifacts an older runner kept in KV: asked for only once the service
	69	+	// has shown the viewer the run, and never while it is still going.
	70	+	const artifacts = legacyArtifactsWorthAsking(detail.run) ? await withLegacyArtifacts(kept, params.id) : kept;
68	71		// Cancelling and re-running need Write.
69	72		return {
70	73		detail,