flagon-io/g1t

public

Merge the site's CPU: code highlighted once per data centre by content, markdown parsed once per text, and the request handler built once per isolate

syntaqxcommitted 13h agoParents583d576 b64348aBrowse files

8	8		MessageSquareWarning,
9	9		OctagonAlert,
10	10		} from "lucide-react";
	11	+	import type { Root } from "hast";
	12	+	import type { Components } from "react-markdown";
11	13		import { type ReactNode, isValidElement, useEffect, useRef, useState } from "react";
12		−	import ReactMarkdown from "react-markdown";
13	14		import { Link } from "react-router";
14	15		import rehypeRaw from "rehype-raw";
15	16		import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
16	17		import remarkGfm from "remark-gfm";
17	18
18	19		import { Checkbox } from "./ui/checkbox";
	20	+	import { WeightedLru } from "../lib/content-cache";
19	21		import { type AlertKind, G1T_MENTION_HREF, type MarkdownRepo, rehypeAlerts, rehypeReferences } from "../lib/markdown-plugins";
	22	+	import { markdownTree, markdownWeight, renderMarkdownTree } from "../lib/markdown-tree";
20	23		import { imageSource } from "../lib/usercontent";
21	24		import { UserCard } from "./user-card";
22	25
⋯
47	50		},
48	51		};
49	52
	53	+	/**
	54	+	* Parsed markdown, by repository and text: parsing and the plugins are
	55	+	* nine tenths of rendering a README, and a page's markdown is rendered
	56	+	* again on every view (and on every revalidation in the browser). The tree
	57	+	* depends only on the text and the repository its references point into
	58	+	* (lib/markdown-tree.ts). A tree takes about 20 bytes a character of its
	59	+	* text, so 400,000 characters of markdown, about 8 MB, per
	60	+	* isolate or tab.
	61	+	*/
	62	+	const trees = new WeightedLru<{ tree: Root; weight: number }>(400_000, (entry) => entry.weight);
	63	+
	64	+	function treeOf(source: string, repo: MarkdownRepo \| undefined): Root {
	65	+	const key = `${repo ? `${repo.namespace}/${repo.name}` : ""}\n${source}`;
	66	+	const kept = trees.get(key);
	67	+	if (kept) return kept.tree;
	68	+	const tree = markdownTree(source, {
	69	+	remarkPlugins: [remarkGfm],
	70	+	rehypePlugins: [rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]],
	71	+	});
	72	+	trees.set(key, { tree, weight: markdownWeight(source) });
	73	+	return tree;
	74	+	}
	75	+
50	76		const ALERT: Record<AlertKind, { title: string; icon: ReactNode; tone: string }> = {
51	77		note: { title: "Note", icon: <Info size={15} />, tone: "border-info/60 [&_.alert-title]:text-info" },
52	78		tip: { title: "Tip", icon: <Lightbulb size={15} />, tone: "border-success/60 [&_.alert-title]:text-accent" },
⋯
177	203		}) {
178	204		return (
179	205		<div className="prose">
180		−	<ReactMarkdown
181		−	remarkPlugins={[remarkGfm]}
182		−	rehypePlugins={[rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]]}
183		−	components={{
184		−	h1: ({ children }) => <Heading level={1}>{children}</Heading>,
185		−	h2: ({ children }) => <Heading level={2}>{children}</Heading>,
186		−	h3: ({ children }) => <Heading level={3}>{children}</Heading>,
187		−	h4: ({ children }) => <Heading level={4}>{children}</Heading>,
188		−	a({ href = "", children, node }) {
189		−	const ref = (node?.properties as { dataRef?: string } \| undefined)?.dataRef;
190		−	if (ref === "mention") {
191		−	// `@name` may be a person (with a card) or a workspace (none).
192		−	const name = href === G1T_MENTION_HREF ? "g1t" : href.replace(/^\//, "");
193		−	return (
194		−	<UserCard username={name}>
195		−	<Link to={href} prefetch="intent" className="font-medium">
196		−	{children}
197		−	</Link>
198		−	</UserCard>
199		−	);
200		−	}
201		−	if (ref) {
202		−	return (
203		−	<Link
204		−	to={href}
205		−	prefetch="intent"
206		−	className={ref === "commit" ? "font-mono text-[0.9em]" : ref === "mention" ? "font-medium" : ""}
207		−	>
	206	+	{renderMarkdownTree(treeOf(source, repo), {
	207	+	h1: ({ children }) => <Heading level={1}>{children}</Heading>,
	208	+	h2: ({ children }) => <Heading level={2}>{children}</Heading>,
	209	+	h3: ({ children }) => <Heading level={3}>{children}</Heading>,
	210	+	h4: ({ children }) => <Heading level={4}>{children}</Heading>,
	211	+	a({ href = "", children, node }) {
	212	+	const ref = (node?.properties as { dataRef?: string } \| undefined)?.dataRef;
	213	+	if (ref === "mention") {
	214	+	// `@name` may be a person (with a card) or a workspace (none).
	215	+	const name = href === G1T_MENTION_HREF ? "g1t" : href.replace(/^\//, "");
	216	+	return (
	217	+	<UserCard username={name}>
	218	+	<Link to={href} prefetch="intent" className="font-medium">
208	219		{children}
209	220		</Link>
210		−	);
211		−	}
212		−	if (href.startsWith("#")) return <a href={href}>{children}</a>;
213		−	if (isExternal(href)) {
214		−	return (
215		−	<a href={href} rel="noreferrer nofollow ugc" target="_blank">
216		−	{children}
217		−	</a>
218		−	);
219		−	}
220		−	// A link within the site, or relative to the document's folder.
221		−	const to = href.startsWith("/") \|\| !base ? href : `${base}/${href.replace(/^\.\//, "")}`;
222		−	return <Link to={to}>{children}</Link>;
223		−	},
224		−	blockquote({ children, node }) {
225		−	const kind = (node?.properties as { dataAlert?: AlertKind } \| undefined)?.dataAlert;
226		−	if (!kind \|\| !ALERT[kind]) return <blockquote>{children}</blockquote>;
227		−	const alert = ALERT[kind];
	221	+	</UserCard>
	222	+	);
	223	+	}
	224	+	if (ref) {
	225	+	return (
	226	+	<Link
	227	+	to={href}
	228	+	prefetch="intent"
	229	+	className={ref === "commit" ? "font-mono text-[0.9em]" : ref === "mention" ? "font-medium" : ""}
	230	+	>
	231	+	{children}
	232	+	</Link>
	233	+	);
	234	+	}
	235	+	if (href.startsWith("#")) return <a href={href}>{children}</a>;
	236	+	if (isExternal(href)) {
228	237		return (
229		−	<div className={`markdown-alert border-l-2 py-1 pl-4 ${alert.tone}`}>
230		−	<p className="alert-title flex items-center gap-2 text-sm font-medium">
231		−	{alert.icon}
232		−	{alert.title}
233		−	</p>
234		−	<div className="mt-1 [&>+]:mt-3">{children}</div>
235		−	</div>
	238	+	<a href={href} rel="noreferrer nofollow ugc" target="_blank">
	239	+	{children}
	240	+	</a>
236	241		);
237		−	},
238		−	pre({ children }) {
239		−	const code = Array.isArray(children) ? children[0] : children;
240		−	if (isValidElement<{ className?: string; children?: ReactNode }>(code)) {
241		−	const language = /language-([\w+-]+)/.exec(code.props.className ?? "")?.[1] ?? null;
242		−	return <CodeBlock language={language} code={textOf(code.props.children).replace(/\n$/, "")} />;
243		−	}
244		−	return <pre>{children}</pre>;
245		−	},
246		−	input({ type, checked, disabled }) {
247		−	// Task list boxes: shown, not editable.
248		−	return type === "checkbox" ? (
249		−	<Checkbox
250		−	checked={checked === true}
251		−	disabled={disabled !== false}
252		−	aria-label={checked ? "Done" : "Not done"}
253		−	className="mr-1.5 inline-flex translate-y-0.5 disabled:cursor-default disabled:opacity-100"
254		−	/>
255		−	) : null;
256		−	},
257		−	img({ src, alt }) {
258		−	const at = typeof src === "string" ? imageSource(src, rawBase) : undefined;
259		−	return <img src={at} alt={alt ?? ""} loading="lazy" className="inline max-w-full rounded" />;
260		−	},
261		−	}}
262		−	>
263		−	{source}
264		−	</ReactMarkdown>
	242	+	}
	243	+	// A link within the site, or relative to the document's folder.
	244	+	const to = href.startsWith("/") \|\| !base ? href : `${base}/${href.replace(/^\.\//, "")}`;
	245	+	return <Link to={to}>{children}</Link>;
	246	+	},
	247	+	blockquote({ children, node }) {
	248	+	const kind = (node?.properties as { dataAlert?: AlertKind } \| undefined)?.dataAlert;
	249	+	if (!kind \|\| !ALERT[kind]) return <blockquote>{children}</blockquote>;
	250	+	const alert = ALERT[kind];
	251	+	return (
	252	+	<div className={`markdown-alert border-l-2 py-1 pl-4 ${alert.tone}`}>
	253	+	<p className="alert-title flex items-center gap-2 text-sm font-medium">
	254	+	{alert.icon}
	255	+	{alert.title}
	256	+	</p>
	257	+	<div className="mt-1 [&>+]:mt-3">{children}</div>
	258	+	</div>
	259	+	);
	260	+	},
	261	+	pre({ children }) {
	262	+	const code = Array.isArray(children) ? children[0] : children;
	263	+	if (isValidElement<{ className?: string; children?: ReactNode }>(code)) {
	264	+	const language = /language-([\w+-]+)/.exec(code.props.className ?? "")?.[1] ?? null;
	265	+	return <CodeBlock language={language} code={textOf(code.props.children).replace(/\n$/, "")} />;
	266	+	}
	267	+	return <pre>{children}</pre>;
	268	+	},
	269	+	input({ type, checked, disabled }) {
	270	+	// Task list boxes: shown, not editable.
	271	+	return type === "checkbox" ? (
	272	+	<Checkbox
	273	+	checked={checked === true}
	274	+	disabled={disabled !== false}
	275	+	aria-label={checked ? "Done" : "Not done"}
	276	+	className="mr-1.5 inline-flex translate-y-0.5 disabled:cursor-default disabled:opacity-100"
	277	+	/>
	278	+	) : null;
	279	+	},
	280	+	img({ src, alt }) {
	281	+	const at = typeof src === "string" ? imageSource(src, rawBase) : undefined;
	282	+	return <img src={at} alt={alt ?? ""} loading="lazy" className="inline max-w-full rounded" />;
	283	+	},
	284	+	} satisfies Components)}
265	285		</div>
266	286		);
267	287		}

1	+	import assert from "node:assert/strict";
2	+	import { test } from "node:test";
3	+
4	+	import { CONTENT_ORIGIN, type SharedCache, WeightedLru, contentCache, contentKey, weightOfLines } from "./content-cache.ts";
5	+
6	+	test("a content key is a hash of every part, in order, and never of a join that could collide", async () => {
7	+	const key = await contentKey(["typescript", "const a = 1;"]);
8	+	assert.match(key, /^[0-9a-f]{64}$/);
9	+	assert.equal(key, await contentKey(["typescript", "const a = 1;"]));
10	+	assert.notEqual(key, await contentKey(["tsx", "const a = 1;"]));
11	+	assert.notEqual(key, await contentKey(["typescript", "const a = 2;"]));
12	+	// The same characters split differently are different content.
13	+	assert.notEqual(await contentKey(["ab", "c"]), await contentKey(["a", "bc"]));
14	+	assert.notEqual(await contentKey(["a:b"]), await contentKey(["a", "b"]));
15	+	});
16	+
17	+	test("the isolate's memory keeps the most recently used within its weight", () => {
18	+	const lru = new WeightedLru<string>(100, (value) => value.length);
19	+	lru.set("a", "x".repeat(20));
20	+	lru.set("b", "x".repeat(20));
21	+	lru.set("c", "x".repeat(20));
22	+	assert.equal(lru.weight, 60);
23	+	// Reading `a` makes it the newest; `b` is now the oldest.
24	+	assert.ok(lru.get("a"));
25	+	lru.set("d", "x".repeat(20));
26	+	lru.set("e", "x".repeat(20));
27	+	lru.set("f", "x".repeat(20));
28	+	assert.equal(lru.get("b"), undefined);
29	+	assert.ok(lru.get("a"));
30	+	assert.ok(lru.weight <= 100);
31	+	// Replacing an entry does not count it twice.
32	+	lru.set("a", "y");
33	+	assert.equal(lru.get("a"), "y");
34	+	assert.ok(lru.weight <= 100);
35	+	// One entry heavier than a quarter of the whole is never kept.
36	+	lru.set("big", "x".repeat(26));
37	+	assert.equal(lru.get("big"), undefined);
38	+	});
39	+
40	+	test("lines weigh their characters", () => {
41	+	assert.equal(weightOfLines(["ab", null, ""]), 2 + 8 + 0 + 8 + 0 + 8);
42	+	});
43	+
44	+	/** A data centre's cache in memory, as caches.default answers. */
45	+	function fakeShared() {
46	+	const kept = new Map<string, string>();
47	+	const shared: SharedCache = {
48	+	async match(url) {
49	+	const body = kept.get(url);
50	+	return body === undefined ? undefined : new Response(body);
51	+	},
52	+	async put(url, response) {
53	+	kept.set(url, await response.text());
54	+	},
55	+	};
56	+	return { kept, shared };
57	+	}
58	+
59	+	test("an answer is computed once, then read from the isolate, then from the data centre", async () => {
60	+	const { kept, shared } = fakeShared();
61	+	const deferred: Promise<unknown>[] = [];
62	+	const options = {
63	+	name: "test-lines",
64	+	version: "v1",
65	+	maxWeight: 1_000,
66	+	weigh: weightOfLines,
67	+	shared: () => shared,
68	+	defer: (work: Promise<unknown>) => void deferred.push(work),
69	+	};
70	+	const cached = contentCache<string[]>(options);
71	+	let runs = 0;
72	+	const compute = async () => {
73	+	runs += 1;
74	+	return ["<b>a</b>", "b"];
75	+	};
76	+	assert.deepEqual(await cached(["rust", "a\nb"], compute), ["<b>a</b>", "b"]);
77	+	await Promise.all(deferred);
78	+	assert.equal(runs, 1);
79	+	assert.equal(kept.size, 1);
80	+	const [url] = kept.keys();
81	+	assert.ok(url!.startsWith(`${CONTENT_ORIGIN}test-lines/v1/`));
82	+	// The isolate has it.
83	+	assert.deepEqual(await cached(["rust", "a\nb"], compute), ["<b>a</b>", "b"]);
84	+	assert.equal(runs, 1);
85	+	// Another isolate (a fresh cache) finds the data centre's copy.
86	+	const other = contentCache<string[]>(options);
87	+	assert.deepEqual(await other(["rust", "a\nb"], compute), ["<b>a</b>", "b"]);
88	+	assert.equal(runs, 1);
89	+	// Other content, or the same content for another language, is another entry.
90	+	await cached(["rust", "a\nc"], compute);
91	+	await cached(["go", "a\nb"], compute);
92	+	assert.equal(runs, 3);
93	+	});
94	+
95	+	test("a new version never reads what an old one kept", async () => {
96	+	const { shared } = fakeShared();
97	+	const make = (version: string) =>
98	+	contentCache<string[]>({ name: "test", version, maxWeight: 1_000, weigh: weightOfLines, shared: () => shared, defer: () => {} });
99	+	const deferred: Promise<unknown>[] = [];
100	+	const first = contentCache<string[]>({
101	+	name: "test",
102	+	version: "v1",
103	+	maxWeight: 1_000,
104	+	weigh: weightOfLines,
105	+	shared: () => shared,
106	+	defer: (work) => void deferred.push(work),
107	+	});
108	+	await first(["x"], async () => ["old"]);
109	+	await Promise.all(deferred);
110	+	assert.deepEqual(await make("v1")(["x"], async () => ["new"]), ["old"]);
111	+	assert.deepEqual(await make("v2")(["x"], async () => ["new"]), ["new"]);
112	+	});
113	+
114	+	test("nothing to keep is not kept, and a broken shared entry counts as missing", async () => {
115	+	const { kept, shared } = fakeShared();
116	+	const cached = contentCache<string[]>({ name: "t", version: "v1", maxWeight: 1_000, weigh: weightOfLines, shared: () => shared, defer: () => {} });
117	+	let runs = 0;
118	+	assert.equal(await cached(["none"], async () => (runs++, null)), null);
119	+	assert.equal(await cached(["none"], async () => (runs++, null)), null);
120	+	assert.equal(runs, 2);
121	+	assert.equal(kept.size, 0);
122	+
123	+	const url = `${CONTENT_ORIGIN}t/v1/${await contentKey(["t", "v1", "broken"])}`;
124	+	kept.set(url, "{not json");
125	+	assert.deepEqual(await cached(["broken"], async () => ["fresh"]), ["fresh"]);
126	+	});
127	+
128	+	test("without a data centre's cache the isolate still keeps answers", async () => {
129	+	const cached = contentCache<string[]>({ name: "t", version: "v1", maxWeight: 1_000, weigh: weightOfLines, shared: () => null, defer: () => {} });
130	+	let runs = 0;
131	+	await cached(["a"], async () => (runs++, ["x"]));
132	+	await cached(["a"], async () => (runs++, ["x"]));
133	+	assert.equal(runs, 1);
134	+	});

1	+	/**
2	+	* Work done on content that never changes once written: highlighting a
3	+	* file's text, turning a README into a tree. The answer depends on the
4	+	* content alone, so it is kept by a hash of that content and is good for
5	+	* as long as the code that made it is the same (`version`, part of every
6	+	* key). No viewer, path or branch is in the key, so the same text reached
7	+	* by another branch, commit or page is the same entry, and nothing in an
8	+	* entry says whose it is or where it came from.
9	+	*/
10	+
11	+	/**
12	+	* A key for `parts`: SHA-256 over each part, length-prefixed, so
13	+	* `["ab", "c"]` and `["a", "bc"]` never collide. Hex, 64 characters.
14	+	*/
15	+	export async function contentKey(parts: readonly string[]): Promise<string> {
16	+	const text = parts.map((part) => `${part.length}:${part}`).join("");
17	+	const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
18	+	return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
19	+	}
20	+
21	+	/**
22	+	* An isolate's memory of recent answers, bounded by their total weight
23	+	* (characters, roughly) rather than by count, so a few large files cannot
24	+	* hold more than `maxWeight` between them. The least recently used go
25	+	* first. An entry heavier than a quarter of the whole is never kept.
26	+	*/
27	+	export class WeightedLru<V> {
28	+	private readonly entries = new Map<string, { value: V; weight: number }>();
29	+	private total = 0;
30	+	private readonly maxWeight: number;
31	+	private readonly weigh: (value: V) => number;
32	+
33	+	constructor(maxWeight: number, weigh: (value: V) => number) {
34	+	this.maxWeight = maxWeight;
35	+	this.weigh = weigh;
36	+	}
37	+
38	+	get(key: string): V \| undefined {
39	+	const entry = this.entries.get(key);
40	+	if (!entry) return undefined;
41	+	// Most recently used last.
42	+	this.entries.delete(key);
43	+	this.entries.set(key, entry);
44	+	return entry.value;
45	+	}
46	+
47	+	set(key: string, value: V): void {
48	+	const weight = Math.max(1, this.weigh(value));
49	+	this.delete(key);
50	+	if (weight > this.maxWeight / 4) return;
51	+	this.entries.set(key, { value, weight });
52	+	this.total += weight;
53	+	for (const [oldest, entry] of this.entries) {
54	+	if (this.total <= this.maxWeight) break;
55	+	this.entries.delete(oldest);
56	+	this.total -= entry.weight;
57	+	}
58	+	}
59	+
60	+	delete(key: string): void {
61	+	const entry = this.entries.get(key);
62	+	if (!entry) return;
63	+	this.entries.delete(key);
64	+	this.total -= entry.weight;
65	+	}
66	+
67	+	get size(): number {
68	+	return this.entries.size;
69	+	}
70	+
71	+	get weight(): number {
72	+	return this.total;
73	+	}
74	+	}
75	+
76	+	/** The weight of a list of strings: their characters. */
77	+	export function weightOfLines(lines: readonly (string \| null)[]): number {
78	+	let total = 0;
79	+	for (const line of lines) total += (line?.length ?? 0) + 8;
80	+	return total;
81	+	}
82	+
83	+	/** The data centre's cache, as far as a content cache needs it (`caches.default`). */
84	+	export type SharedCache = {
85	+	match(url: string): Promise<Response \| undefined>;
86	+	put(url: string, response: Response): Promise<void>;
87	+	};
88	+
89	+	/** Where shared entries live: a name nothing outside can ask for. */
90	+	export const CONTENT_ORIGIN = "https://content.g1t.internal/";
91	+
92	+	/** A shared entry is good for this long; its key changes when its content or code does. */
93	+	export const SHARED_MAX_AGE_SECONDS = 30 * 24 * 60 * 60;
94	+
95	+	export type ContentCacheOptions<V> = {
96	+	/** What is kept, e.g. `highlight-lines`: part of every key. */
97	+	name: string;
98	+	/** Bumped whenever the code that makes an entry changes what it makes. */
99	+	version: string;
100	+	/** The isolate's share, in `weigh` units. */
101	+	maxWeight: number;
102	+	weigh: (value: V) => number;
103	+	/** The data centre's cache, or null where there is none (tests, local development). */
104	+	shared: () => SharedCache \| null;
105	+	/** Lets a write to the shared cache finish after the answer (`waitUntil`). */
106	+	defer: (work: Promise<unknown>) => void;
107	+	};
108	+
109	+	/**
110	+	* `compute`'s answer for content `parts`, kept in the isolate and in the
111	+	* data centre's cache. The isolate is asked first, then the data centre;
112	+	* only when neither has it is `compute` run, and its answer kept in both.
113	+	* A null answer (nothing to keep: no language, too large, failed) is
114	+	* returned and never kept. A shared entry that cannot be read is treated
115	+	* as missing.
116	+	*/
117	+	export function contentCache<V>(options: ContentCacheOptions<V>) {
118	+	const memory = new WeightedLru<V>(options.maxWeight, options.weigh);
119	+	const cached = async (parts: readonly string[], compute: () => Promise<V \| null>): Promise<V \| null> => {
120	+	const key = await contentKey([options.name, options.version, ...parts]);
121	+	const remembered = memory.get(key);
122	+	if (remembered !== undefined) return remembered;
123	+	const url = `${CONTENT_ORIGIN}${options.name}/${options.version}/${key}`;
124	+	const shared = options.shared();
125	+	if (shared) {
126	+	const found = await shared.match(url).catch(() => undefined);
127	+	if (found) {
128	+	const value = (await found.json().catch(() => undefined)) as V \| undefined;
129	+	if (value !== undefined && value !== null) {
130	+	memory.set(key, value);
131	+	return value;
132	+	}
133	+	}
134	+	}
135	+	const value = await compute();
136	+	if (value === null) return null;
137	+	memory.set(key, value);
138	+	if (shared) {
139	+	const response = new Response(JSON.stringify(value), {
140	+	headers: { "content-type": "application/json", "cache-control": `public, max-age=${SHARED_MAX_AGE_SECONDS}` },
141	+	});
142	+	options.defer(shared.put(url, response).catch(() => undefined));
143	+	}
144	+	return value;
145	+	};
146	+	return Object.assign(cached, { memory });
147	+	}

1	+	import assert from "node:assert/strict";
2	+	import { test } from "node:test";
3	+
4	+	import type { FileDiff } from "@g1t/contracts";
5	+
6	+	import { diffContent, htmlOfFile, withHtml } from "./diff.ts";
7	+	import { highlightFile } from "./shiki.ts";
8	+
9	+	const FILE: FileDiff = {
10	+	path: "src/lib.rs",
11	+	status: "modified",
12	+	additions: 2,
13	+	deletions: 1,
14	+	binary: false,
15	+	hunks: [
16	+	{
17	+	lines: [
18	+	{ kind: "context", old: 1, new: 1, text: "fn main() {" },
19	+	{ kind: "delete", old: 2, new: null, text: ' println!("old");' },
20	+	{ kind: "add", old: null, new: 2, text: ' println!("new");' },
21	+	{ kind: "add", old: null, new: 3, text: "" },
22	+	{ kind: "context", old: 3, new: 4, text: "}" },
23	+	],
24	+	},
25	+	{ lines: [{ kind: "add", old: null, new: 10, text: "// end" }] },
26	+	],
27	+	};
28	+
29	+	test("a highlighted file's HTML, kept apart and given back, is the same highlighted file", async () => {
30	+	const highlighted = await highlightFile(FILE);
31	+	assert.ok(highlighted);
32	+	const html = htmlOfFile(highlighted);
33	+	assert.equal(html.length, 2);
34	+	assert.ok(html[0]!.every((row) => row !== null));
35	+	// Through the data centre's cache it travels as JSON.
36	+	const kept = JSON.parse(JSON.stringify(html));
37	+	assert.deepEqual(withHtml(FILE, kept), highlighted);
38	+	});
39	+
40	+	test("lines without HTML stay without it", () => {
41	+	const html = [[null, "<b>x</b>", null, "", null], [null]];
42	+	const file = withHtml(FILE, html);
43	+	assert.equal("html" in file.hunks[0]!.lines[0]!, false);
44	+	assert.equal(file.hunks[0]!.lines[1]!.html, "<b>x</b>");
45	+	// An empty line's HTML is kept: it is not the same as none.
46	+	assert.equal(file.hunks[0]!.lines[3]!.html, "");
47	+	assert.equal("html" in file.hunks[1]!.lines[0]!, false);
48	+	// The diff given in is not changed.
49	+	assert.equal("html" in FILE.hunks[0]!.lines[1]!, false);
50	+	});
51	+
52	+	test("what a file's colours depend on: each line's side and text, not its numbers or path", () => {
53	+	const moved: FileDiff = {
54	+	...FILE,
55	+	path: "other/lib.rs",
56	+	hunks: FILE.hunks.map((hunk) => ({ lines: hunk.lines.map((line) => ({ ...line, old: 99, new: 99 })) })),
57	+	};
58	+	assert.equal(diffContent(moved), diffContent(FILE));
59	+	const sideChanged: FileDiff = {
60	+	...FILE,
61	+	hunks: [{ lines: FILE.hunks[0]!.lines.map((line, n) => (n === 0 ? { ...line, kind: "add" as const } : line)) }, FILE.hunks[1]!],
62	+	};
63	+	assert.notEqual(diffContent(sideChanged), diffContent(FILE));
64	+	// A line split differently is different content.
65	+	const one: FileDiff = { ...FILE, hunks: [{ lines: [{ kind: "add", old: null, new: 1, text: "a\nb" }] }] };
66	+	const two: FileDiff = {
67	+	...FILE,
68	+	hunks: [{ lines: [{ kind: "add", old: null, new: 1, text: "a" }, { kind: "add", old: null, new: 2, text: "b" }] }],
69	+	};
70	+	assert.notEqual(diffContent(one), diffContent(two));
71	+	});

	1	+	import { waitUntil } from "cloudflare:workers";
	2	+
1	3		import type { Comparison } from "@g1t/contracts";
2	4
3		−	import type { HighlightedComparison, HighlightedFile } from "./diff";
	5	+	import { type SharedCache, contentCache, weightOfLines } from "./content-cache";
	6	+	import { type FileHtml, type HighlightedComparison, type HighlightedFile, diffContent, htmlOfFile, withHtml } from "./diff";
4	7
5	8		// Shiki and its grammars load on the first file highlighted, not when the
6		−	// Worker starts: most requests highlight nothing.
7		−	const shiki = () => import("./shiki");
	9	+	// Worker starts: most requests highlight nothing. The module is asked for
	10	+	// once per isolate.
	11	+	let shikiModule: Promise<typeof import("./shiki")> \| undefined;
	12	+	const shiki = () => (shikiModule ??= import("./shiki"));
8	13
9	14		const MAX_HIGHLIGHT_CHARS = 200_000;
10	15
11	16		/**
12		−	* Highlighted HTML for a file, or null when its language is unknown or it
13		−	* is too large, in which case the caller shows plain text.
	17	+	* Highlighting is most of the work of a file's page (about two thirds of a
	18	+	* small file's, nine tenths of a large one's), and the same text always
	19	+	* highlights the same way. So it is kept by a hash of the text and its
	20	+	* language: in the isolate, and in the data centre's cache for the other
	21	+	* isolates there (lib/content-cache.ts). A file read again on another
	22	+	* branch or commit, through blame, or by the next crawler, is highlighted
	23	+	* once. Bump `HIGHLIGHT_VERSION` when the theme, the grammars, Shiki or
	24	+	* `linesToHtml` change what they make.
14	25		*/
15		−	export async function highlight(path: string, text: string): Promise<string \| null> {
16		−	const { THEME, getHighlighter, languageOf } = await shiki();
17		−	const lang = languageOf(path);
18		−	if (!lang \|\| text.length > MAX_HIGHLIGHT_CHARS) return null;
	26	+	export const HIGHLIGHT_VERSION = "v1";
	27	+
	28	+	function dataCentre(): SharedCache \| null {
19	29		try {
20		−	return (await getHighlighter()).codeToHtml(text.replace(/\n$/, ""), { lang, theme: THEME });
	30	+	return (caches as unknown as { default: Cache }).default ?? null;
21	31		} catch {
	32	+	// No cache (local development).
22	33		return null;
23	34		}
24	35		}
25	36
	37	+	function defer(work: Promise<unknown>) {
	38	+	try {
	39	+	waitUntil(work);
	40	+	} catch {
	41	+	// Outside a request: the write finishes or not; nothing waits on it.
	42	+	}
	43	+	}
	44	+
26	45		/**
	46	+	* About 8 MB of highlighted lines per isolate (two bytes a character); one
	47	+	* file up to a quarter of that, which a 1,800-line file fits. A pull
	48	+	* request's first screens are at most 40,000 characters of text, about
	49	+	* ten times that highlighted.
	50	+	*/
	51	+	const linesCache = contentCache<string[]>({
	52	+	name: "highlight-lines",
	53	+	version: HIGHLIGHT_VERSION,
	54	+	maxWeight: 4_000_000,
	55	+	weigh: weightOfLines,
	56	+	shared: dataCentre,
	57	+	defer,
	58	+	});
	59	+
	60	+	const filesCache = contentCache<FileHtml>({
	61	+	name: "highlight-diff",
	62	+	version: HIGHLIGHT_VERSION,
	63	+	maxWeight: 2_000_000,
	64	+	weigh: (html) => html.reduce((sum, hunk) => sum + weightOfLines(hunk), 0),
	65	+	shared: dataCentre,
	66	+	defer,
	67	+	});
	68	+
	69	+	/**
27	70		* A file's lines as highlighted HTML, one string per line, or null when its
28	71		* language is unknown or it is too large.
29	72		*/
30	73		export async function highlightLines(path: string, text: string): Promise<string[] \| null> {
31		−	const { getHighlighter, languageOf, linesToHtml } = await shiki();
	74	+	const { languageOf } = await shiki();
32	75		const lang = languageOf(path);
33	76		if (!lang \|\| text.length > MAX_HIGHLIGHT_CHARS) return null;
34		−	try {
35		−	return linesToHtml(await getHighlighter(), text.replace(/\n$/, ""), lang);
36		−	} catch {
37		−	return null;
38		−	}
	77	+	const code = text.replace(/\n$/, "");
	78	+	return linesCache([lang, code], async () => {
	79	+	const { getHighlighter, linesToHtml } = await shiki();
	80	+	try {
	81	+	return linesToHtml(await getHighlighter(), code, lang);
	82	+	} catch {
	83	+	return null;
	84	+	}
	85	+	});
39	86		}
40	87
41	88		/**
⋯
45	92		*/
46	93		const FIRST_SCREENS_CHARS = 40_000;
47	94
	95	+	/** A file's diff highlighted, from the cache when the same lines were highlighted before. */
	96	+	async function highlightFileCached(file: Comparison["files"][number]): Promise<HighlightedFile \| null> {
	97	+	const { highlightFile, languageOf } = await shiki();
	98	+	const lang = languageOf(file.path);
	99	+	// Nothing to highlight: highlightFile says so without any work.
	100	+	if (!lang \|\| file.binary) return null;
	101	+	const html = await filesCache([lang, diffContent(file)], async () => {
	102	+	const highlighted = await highlightFile(file).catch(() => null);
	103	+	return highlighted ? htmlOfFile(highlighted) : null;
	104	+	});
	105	+	return html ? withHtml(file, html) : null;
	106	+	}
	107	+
48	108		/**
49	109		* A comparison with its first files' lines highlighted, in order, until
50	110		* the budget runs out, so what is on screen when the page arrives already
⋯
62	122		continue;
63	123		}
64	124		budget -= size;
65		−	const { highlightFile } = await shiki();
66		−	files.push((await highlightFile(file).catch(() => null)) ?? file);
	125	+	files.push((await highlightFileCached(file).catch(() => null)) ?? file);
67	126		}
68	127		return { ...comparison, files };
69	128		}

1	+	import assert from "node:assert/strict";
2	+	import { readFileSync } from "node:fs";
3	+	import { test } from "node:test";
4	+
5	+	import { createElement } from "react";
6	+	import { renderToStaticMarkup } from "react-dom/server";
7	+	import ReactMarkdown, { type Components } from "react-markdown";
8	+	import rehypeRaw from "rehype-raw";
9	+	import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
10	+	import remarkGfm from "remark-gfm";
11	+
12	+	import { rehypeAlerts, rehypeReferences } from "./markdown-plugins.ts";
13	+	import { markdownTree, renderMarkdownTree } from "./markdown-tree.ts";
14	+
15	+	// components/markdown.tsx's options: GitHub flavour, sanitized raw HTML,
16	+	// alerts and references.
17	+	const SCHEMA = {
18	+	...defaultSchema,
19	+	attributes: {
20	+	...defaultSchema.attributes,
21	+	code: [...(defaultSchema.attributes?.code ?? []), ["className", /^language-./]],
22	+	},
23	+	};
24	+	const repo = { namespace: "acme", name: "web" };
25	+	const options = {
26	+	remarkPlugins: [remarkGfm],
27	+	rehypePlugins: [rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]],
28	+	} as const;
29	+
30	+	// Components that read the node they are given, as the real ones do.
31	+	const components: Components = {
32	+	a: ({ href, children, node }) =>
33	+	createElement("a", { href, "data-ref": String((node?.properties as { dataRef?: string } \| undefined)?.dataRef ?? "") }, children),
34	+	blockquote: ({ children, node }) =>
35	+	createElement("blockquote", { "data-alert": String((node?.properties as { dataAlert?: string } \| undefined)?.dataAlert ?? "") }, children),
36	+	h2: ({ children }) => createElement("h2", { className: "heading" }, children),
37	+	img: ({ src, alt }) => createElement("img", { src: typeof src === "string" ? `/raw/${src}` : undefined, alt: alt ?? "" }),
38	+	};
39	+
40	+	const root = new URL("../../../../", import.meta.url);
41	+	const SAMPLES: [string, string][] = [
42	+	["README.md", readFileSync(new URL("README.md", root), "utf8")],
43	+	["docs/PERFORMANCE.md", readFileSync(new URL("docs/PERFORMANCE.md", root), "utf8")],
44	+	["CONTRIBUTING.md", readFileSync(new URL("CONTRIBUTING.md", root), "utf8")],
45	+	[
46	+	"edge cases",
47	+	[
48	+	"# Title",
49	+	"",
50	+	"> [!WARNING]",
51	+	"> Careful with #12, acme/api#3 and @Ana, and commit 0123456789abcdef0123456789abcdef01234567.",
52	+	"",
53	+	"- [x] done",
54	+	"- [ ] not done",
55	+	"",
56	+	"\| a \| b \|",
57	+	"\| - \| - \|",
58	+	"\| 1 \| 2 \|",
59	+	"",
60	+	"Footnote[^1] and ~~gone~~ and https://example.com.",
61	+	"",
62	+	"[^1]: The note.",
63	+	"",
64	+	'<div align="center"><img src="logo.png" alt="Logo" onerror="alert(1)"><script>alert(1)</script></div>',
65	+	"",
66	+	"[bad](javascript:alert(1)) [rel](./docs/x.md) [abs](/acme/web) ![pic](img/a.png)",
67	+	"",
68	+	"<details><summary>More</summary>",
69	+	"",
70	+	"Inside details.",
71	+	"",
72	+	"</details>",
73	+	"",
74	+	"```ts",
75	+	"const a = 1;",
76	+	"```",
77	+	"",
78	+	'<a href="vbscript:x" title="t">raw link</a> <iframe src="https://evil"></iframe>',
79	+	].join("\n"),
80	+	],
81	+	["empty", ""],
82	+	];
83	+
84	+	for (const [name, source] of SAMPLES) {
85	+	test(`${name}: the kept tree renders exactly what react-markdown renders`, () => {
86	+	const expected = renderToStaticMarkup(createElement(ReactMarkdown, { ...options, components, children: source }));
87	+	const tree = markdownTree(source, options as never);
88	+	assert.equal(renderToStaticMarkup(renderMarkdownTree(tree, components)), expected);
89	+	// Rendering does not change the tree: rendered again, it is the same.
90	+	assert.equal(renderToStaticMarkup(renderMarkdownTree(tree, components)), expected);
91	+	});
92	+	}

1	+	import type { Root } from "hast";
2	+	import { type Components as JsxComponents, toJsxRuntime } from "hast-util-to-jsx-runtime";
3	+	import { urlAttributes } from "html-url-attributes";
4	+	import type { ReactElement } from "react";
5	+	import { Fragment, jsx, jsxs } from "react/jsx-runtime";
6	+	import { type Components, defaultUrlTransform } from "react-markdown";
7	+	import remarkParse from "remark-parse";
8	+	import remarkRehype from "remark-rehype";
9	+	import { type PluggableList, unified } from "unified";
10	+	import { visit } from "unist-util-visit";
11	+	import { VFile } from "vfile";
12	+
13	+	/**
14	+	* Markdown in two steps, the same two `react-markdown`'s `<Markdown>` takes
15	+	* on every render, split so the first can be kept:
16	+	*
17	+	* 1. `markdownTree`: parse, run the plugins, make URLs safe. Most of the
18	+	* work (nine tenths of a large README's), and it depends only on the
19	+	* text and the plugins, so components/markdown.tsx keeps its trees.
20	+	* 2. `renderMarkdownTree`: the tree as React elements, with the page's
21	+	* components.
22	+	*
23	+	* Together they make exactly what `<Markdown>` makes for the same options
24	+	* (lib/markdown-tree.test.ts renders both and compares). Options this file
25	+	* does not take (`allowedElements`, `skipHtml`, `urlTransform`, …) are not
26	+	* used by g1t.
27	+	*/
28	+	export type MarkdownTreeOptions = {
29	+	remarkPlugins?: PluggableList;
30	+	rehypePlugins?: PluggableList;
31	+	};
32	+
33	+	/** The hast tree `<Markdown>` would render for `source`. Not changed by rendering, so it can be kept and rendered again. */
34	+	export function markdownTree(source: string, options: MarkdownTreeOptions = {}): Root {
35	+	const processor = unified()
36	+	.use(remarkParse)
37	+	.use(options.remarkPlugins ?? [])
38	+	.use(remarkRehype, { allowDangerousHtml: true })
39	+	.use(options.rehypePlugins ?? []);
40	+	const file = new VFile();
41	+	file.value = source;
42	+	const tree = processor.runSync(processor.parse(file), file) as Root;
43	+	// What react-markdown does before rendering: raw HTML left over becomes
44	+	// text, and every URL attribute goes through the default transform.
45	+	visit(tree, (node, index, parent) => {
46	+	if (node.type === "raw" && parent && typeof index === "number") {
47	+	parent.children[index] = { type: "text", value: node.value };
48	+	return index;
49	+	}
50	+	if (node.type === "element") {
51	+	for (const key in urlAttributes) {
52	+	if (Object.hasOwn(urlAttributes, key) && Object.hasOwn(node.properties, key)) {
53	+	const value = node.properties[key];
54	+	const test = urlAttributes[key];
55	+	if (test === null \|\| test.includes(node.tagName)) {
56	+	node.properties[key] = defaultUrlTransform(String(value \|\| ""));
57	+	}
58	+	}
59	+	}
60	+	}
61	+	return undefined;
62	+	});
63	+	return tree;
64	+	}
65	+
66	+	/** `tree` as React elements, as `<Markdown>` renders it with `components`. */
67	+	export function renderMarkdownTree(tree: Root, components?: Components): ReactElement {
68	+	return toJsxRuntime(tree, {
69	+	Fragment,
70	+	// react-markdown's components, which it hands to this same function.
71	+	components: components as JsxComponents \| undefined,
72	+	ignoreInvalidStyle: true,
73	+	jsx,
74	+	jsxs,
75	+	passKeys: true,
76	+	passNode: true,
77	+	});
78	+	}
79	+
80	+	/** How much a kept tree weighs: its text's length, a stand-in for the tree's size. */
81	+	export function markdownWeight(source: string): number {
82	+	return source.length + 64;
83	+	}

195	195		\| A branch's log, the branch list, a file by branch and path \| repos' data-centre cache \| until the repository's refs change (`refs_version`), 5 min at most \| only while no handed-out push credential is live; by commit hash for good (docs/ARTIFACTS.md R9) \|
196	196		\| A target branch's history, for mergeability \| the repos isolate \| 60 s, per target head \| 100 pull requests checked after a push walk it once (R10) \|
197	197		\| Git store credentials \| repos isolate and KV \| reused 50 min (1 h tokens); 3 min for ones handed out \| (R3) \|
	198	+	\| A file's highlighted lines (blob, blame) \| the site's isolate (about 8 MB), then the data centre's cache (`content.g1t.internal/highlight-lines/`) \| for good (30 days in the data centre) \| by SHA-256 of the language and the text, nothing else: the same text on any branch, commit or page is one entry. `HIGHLIGHT_VERSION` in `lib/highlight.server.ts` is in the key; bump it when the theme, grammars, Shiki or `linesToHtml` change. Nothing kept for a file without a language or over 200,000 characters \|
	199	+	\| A pull request's first highlighted files \| as above (`highlight-diff/`, about 4 MB per isolate) \| for good \| by the language and each line's side and text (`diffContent` in `lib/diff.ts`); line numbers and the path are not in it \|
	200	+	\| Parsed markdown \| the isolate, or the browser tab (400,000 characters of source, about 8 MB) \| until pushed out \| by the text and the repository its references point into (`components/markdown.tsx`); the tree is rendered with the page's components each time \|
	201	+	\| React Router's route tables \| the isolate \| its life \| the build is loaded once and the handler made once (`workers/app.ts`); development still reloads it per request \|
198	202
199	203		## Server-Timing
200	204
⋯
358	362		\| Crawler, nothing moved \| 460 to 570 ms \| not yet measured on production \|
359	363		\| Browser, first byte \| 115 to 160 ms (`total`), 200 to 245 ms measured from Colorado \| unchanged: the page does not wait for any of this \|
360	364
	365	+	## CPU per page (2026-10-09)
	366	+
	367	+	Workers bill CPU time past 30 million ms a cycle. From 1 to 9 October the
	368	+	site (`g1t`) used 83.3 million CPU-ms over 1.93 million requests, about 43
	369	+	ms a request and seven tenths of all g1t's Workers CPU; `g1t-repos` used
	370	+	28.3 million over 6.58 million (about 4 ms). Signed-out pages are kept
	371	+	for 30 s (above), but a crawler reads each file once, so most of its
	372	+	requests render.
	373	+
	374	+	### Measuring it
	375	+
	376	+	`Server-Timing` cannot show CPU: a Worker's clock does not move while it
	377	+	computes, so `total;dur=0` on a page that rendered for 20 ms is normal.
	378	+	Measure locally instead, with the built site and fake services:
	379	+
	380	+	1. `npm run build -w apps/web`.
	381	+	2. Load `build/server/index.js` in Node with `cloudflare:workers` pointed
	382	+	at a stub whose `env` has a service binding per service, answering
	383	+	`POST /rpc/<method>` from fixtures. A page's `.data` (fetched signed
	384	+	out from production) decoded with React Router's turbo-stream decoder
	385	+	gives realistic answers; files and READMEs can come from the working
	386	+	tree.
	387	+	3. Call the worker's `fetch` with a crawler's user agent (the whole page
	388	+	renders before the answer) and read `process.cpuUsage()` over 100
	389	+	requests after a few to warm up. Run Node with `--single-threaded` so
	390	+	garbage collection and compilation count on the one thread, as in a
	391	+	Worker; on Windows `cpuUsage` moves in 15.6 ms steps, so divide a long
	392	+	run, never time one request.
	393	+	4. `node --cpu-prof` on the same loop says where it goes.
	394	+
	395	+	### Where it went
	396	+
	397	+	Profiled with fixtures from flagon-io/g1t:
	398	+
	399	+	\| Page \| CPU a request \| Where \|
	400	+	\| --- \| --- \| --- \|
	401	+	\| A 130-line TypeScript file \| 30 ms \| 63% highlighting (Shiki's tokenizer), 17% rendering \|
	402	+	\| A 1,800-line TSX file (2.4 MB page) \| 250 ms \| 85% highlighting; the rest rendering and encoding the page \|
	403	+	\| The Files page with a README \| 32 ms \| 46% parsing the README (remark, rehype-raw, sanitize, the plugins) \|
	404	+	\| Any page \| 1 to 1.5 ms more \| React Router rebuilt its route tables for every request: given the build as a function, it wraps every route for the timings and flattens and ranks the route table again each time \|
	405	+	\| `package-lock.json` (3.8 MB page, too large to highlight) \| 170 ms \| rendering one row per line and the file again in the page's data \|
	406	+
	407	+	The CSP nonce, `isbot`, Server-Timing bookkeeping and the signed-out
	408	+	cache's own work were each under 1% of a page's CPU in the profiles.
	409	+
	410	+	### What changed
	411	+
	412	+	- Highlighting is kept by content (`lib/highlight.server.ts`,
	413	+	`lib/content-cache.ts`): the isolate first, then the data centre's
	414	+	cache, then Shiki. The key is a SHA-256 of the language and the text,
	415	+	with a version, so a file that is the same on another branch or commit,
	416	+	in blame, or for the next crawler is highlighted once per data centre.
	417	+	A pull request's first files are kept the same way.
	418	+	- Markdown is parsed once per text (`lib/markdown-tree.ts`,
	419	+	`components/markdown.tsx`): the steps `react-markdown` runs on every
	420	+	render are split, and the parsed tree is kept per isolate (and per
	421	+	browser tab). `lib/markdown-tree.test.ts` renders README.md, this file
	422	+	and a set of edge cases (raw HTML, scripts, `javascript:` links, alerts,
	423	+	references) both ways and checks the HTML is identical.
	424	+	- The request handler is made once per isolate (`workers/app.ts`).
	425	+	- Shiki's module is asked for once per isolate, not on every highlight.
	426	+
	427	+	### Measured
	428	+
	429	+	Locally, CPU a request, median of three runs of 100 requests, signed out
	430	+	as a crawler. "Seen" is a file or README this isolate (or data centre)
	431	+	has highlighted or parsed before; "new" is one it has not.
	432	+
	433	+	\| Page \| Before \| After, seen \| After, new \|
	434	+	\| --- \| --- \| --- \| --- \|
	435	+	\| `/` (landing) \| 12.7 ms \| 10.6 ms \| 10.6 ms \|
	436	+	\| `/pricing` \| 4.8 ms \| 3.3 ms \| 3.3 ms \|
	437	+	\| Files, root with README.md \| 30.8 ms \| 9.5 ms \| 23.7 ms \|
	438	+	\| Files, `docs/` (a 30,000-character README) \| 44.7 ms \| 8.9 ms \| 33.1 ms \|
	439	+	\| Files, no README \| 9.2 ms \| 7.2 ms \| 6.9 ms \|
	440	+	\| A 130-line TypeScript file \| 27.3 ms \| 7.8 ms \| 24.5 ms \|
	441	+	\| A 1,800-line TSX file \| 255.6 ms \| 41.3 ms \| 246.7 ms \|
	442	+	\| A 560-line Rust file \| 34.7 ms \| 14.8 ms \| 30.0 ms \|
	443	+	\| A markdown file's source \| 18.4 ms \| 11.6 ms \| 18.8 ms \|
	444	+	\| `package-lock.json` \| 200.2 ms \| 134.1 ms \| 127.8 ms \|
	445	+
	446	+	A pull request's first screens: a 288-line diff took 18.8 ms to
	447	+	highlight and 0.15 ms to read back from the data centre's cache. Hashing
	448	+	the text for the key is part of every "new" figure above. Small
	449	+	differences (a few ms) are within the noise of these runs; the large
	450	+	saving on `package-lock.json`, which nothing here caches, is partly that
	451	+	noise and partly less garbage per request.
	452	+
	453	+	What is left on large files is rendering: a row per line, and the same
	454	+	lines again in the page's data for hydration. Files over 200,000
	455	+	characters (lock files) are not highlighted and still cost about 130 ms
	456	+	for a crawler.
	457	+
361	458		## Client navigation
362	459
363	460		- `<Link prefetch="intent">` on the sidebar, project tabs, breadcrumbs,

4	4		export type HighlightedLine = DiffLine & { html?: string };
5	5		export type HighlightedFile = Omit<FileDiff, "hunks"> & { hunks: { lines: HighlightedLine[] }[] };
6	6		export type HighlightedComparison = Omit<Comparison, "files"> & { files: HighlightedFile[] };
	7	+
	8	+	/** Each hunk's lines' HTML, null for a line without any: what a highlighted file adds to its diff. */
	9	+	export type FileHtml = (string \| null)[][];
	10	+
	11	+	/** The HTML `highlighted` gives its lines, to keep apart from the diff it came from. */
	12	+	export function htmlOfFile(highlighted: HighlightedFile): FileHtml {
	13	+	return highlighted.hunks.map((hunk) => hunk.lines.map((line) => line.html ?? null));
	14	+	}
	15	+
	16	+	/** `file` with `html` given back to its lines: the same as the highlighted file it was taken from. */
	17	+	export function withHtml(file: FileDiff, html: FileHtml): HighlightedFile {
	18	+	return {
	19	+	...file,
	20	+	hunks: file.hunks.map((hunk, h) => ({
	21	+	lines: hunk.lines.map((line, n): HighlightedLine => {
	22	+	const row = html[h]?.[n];
	23	+	return row == null ? { ...line } : { ...line, html: row };
	24	+	}),
	25	+	})),
	26	+	};
	27	+	}
	28	+
	29	+	/**
	30	+	* What a file's highlighting depends on besides its language: each line's
	31	+	* side and text, hunk by hunk. Line numbers and the path do not change the
	32	+	* colours, so they are left out.
	33	+	*/
	34	+	export function diffContent(file: FileDiff): string {
	35	+	return JSON.stringify(file.hunks.map((hunk) => hunk.lines.map((line) => [line.kind, line.text])));
	36	+	}

19	19		"clsx": "^2.1.1",
20	20		"cmdk": "^1.1.1",
21	21		"isbot": "^5.1.36",
	22	+	"hast-util-to-jsx-runtime": "^2.3.6",
	23	+	"html-url-attributes": "^3.0.1",
22	24		"lucide-react": "^1.49.0",
23	25		"mdast-util-find-and-replace": "^3.0.2",
24	26		"radix-ui": "^1.6.7",
⋯
29	31		"rehype-raw": "^7.0.0",
30	32		"rehype-sanitize": "^6.0.0",
31	33		"remark-gfm": "^4.0.1",
	34	+	"remark-parse": "^11.0.0",
	35	+	"remark-rehype": "^11.1.2",
32	36		"shiki": "^4.5.0",
33	37		"simple-icons": "^16.34.0",
34	38		"tailwind-merge": "^3.7.0",
35	39		"unist-util-visit": "^5.1.0",
36		−	"uqr": "^0.1.3"
	40	+	"unified": "^11.0.5",
	41	+	"uqr": "^0.1.3",
	42	+	"vfile": "^6.0.3"
37	43		},
38	44		"devDependencies": {
39	45		"@cloudflare/vite-plugin": "^1.62.4",

13	13		import { usercontentPath } from "../app/lib/usercontent";
14	14		import { serveUsercontent } from "./usercontent";
15	15
16		−	const requestHandler = createRequestHandler(
17		−	() => import("virtual:react-router/server-build"),
18		−	import.meta.env.MODE,
19		−	);
	16	+	const loadBuild = () => import("virtual:react-router/server-build");
	17	+
	18	+	/**
	19	+	* Given a function for the build, React Router derives everything from it
	20	+	* again on every request: each route wrapped for the timings in
	21	+	* entry.server.tsx, then the route table flattened and ranked, about a
	22	+	* millisecond and a half of CPU a page. The build never changes while an
	23	+	* isolate lives, so outside development it is loaded once and the handler
	24	+	* made once. Development keeps the function, so a changed file is picked up.
	25	+	* Only the made handler is kept, never a promise of one, so no request
	26	+	* waits on another's (a build that fails to load is loaded again next time).
	27	+	*/
	28	+	let handler: ReturnType<typeof createRequestHandler> \| undefined;
	29	+	async function requestHandler(request: Request): Promise<Response> {
	30	+	if (import.meta.env.DEV) return createRequestHandler(loadBuild, import.meta.env.MODE)(request);
	31	+	handler ??= createRequestHandler(await loadBuild(), import.meta.env.MODE);
	32	+	return handler(request);
	33	+	}
20	34
21	35		const DOCS = "https://docs.g1t.sh";
22	36

78	78		"class-variance-authority": "^0.7.1",
79	79		"clsx": "^2.1.1",
80	80		"cmdk": "^1.1.1",
	81	+	"hast-util-to-jsx-runtime": "^2.3.6",
	82	+	"html-url-attributes": "^3.0.1",
81	83		"isbot": "^5.1.36",
82	84		"lucide-react": "^1.49.0",
83	85		"mdast-util-find-and-replace": "^3.0.2",
⋯
89	91		"rehype-raw": "^7.0.0",
90	92		"rehype-sanitize": "^6.0.0",
91	93		"remark-gfm": "^4.0.1",
	94	+	"remark-parse": "^11.0.0",
	95	+	"remark-rehype": "^11.1.2",
92	96		"shiki": "^4.5.0",
93	97		"simple-icons": "^16.34.0",
94	98		"tailwind-merge": "^3.7.0",
	99	+	"unified": "^11.0.5",
95	100		"unist-util-visit": "^5.1.0",
96		−	"uqr": "^0.1.3"
	101	+	"uqr": "^0.1.3",
	102	+	"vfile": "^6.0.3"
97	103		},
98	104		"devDependencies": {
99	105		"@cloudflare/vite-plugin": "^1.62.4",