flagon-io/g1t

public

Markdown is parsed once per text: the tree react-markdown makes on every render is kept, and only rendered with the page's components

On the Files page with a README, parsing it (remark, GFM, rehype-raw, sanitize, alerts and references) was 46% of the request's CPU, about 15 ms of 32, again on every view and on every revalidation in the browser. lib/markdown-tree.ts splits what react-markdown's <Markdown> does into markdownTree (parse, run the plugins, raw HTML left over to text, URLs through defaultUrlTransform) and renderMarkdownTree (toJsxRuntime with the same options). components/markdown.tsx keeps each tree by the text and the repository its references point into (400,000 characters of source per isolate or tab, about 8 MB: a tree measured about 20 bytes a character) and renders it with its components as before. lib/markdown-tree.test.ts renders README.md, docs/PERFORMANCE.md, CONTRIBUTING.md, an empty text and a set of edge cases (alerts, references, task lists, tables, footnotes, raw HTML with a script and an onerror, javascript: and vbscript: links, an iframe, details, images) with react-markdown and with the kept tree, twice, and checks the HTML is identical. unified, remark-parse, remark-rehype, vfile, hast-util-to-jsx-runtime and html-url-attributes, already installed for react-markdown at those versions, are now the site's own dependencies.

syntaqxcommitted 16h agoParent6db9485Browse files

8	8		MessageSquareWarning,
9	9		OctagonAlert,
10	10		} from "lucide-react";
	11	+	import type { Root } from "hast";
	12	+	import type { Components } from "react-markdown";
11	13		import { type ReactNode, isValidElement, useEffect, useRef, useState } from "react";
12		−	import ReactMarkdown from "react-markdown";
13	14		import { Link } from "react-router";
14	15		import rehypeRaw from "rehype-raw";
15	16		import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
16	17		import remarkGfm from "remark-gfm";
17	18
18	19		import { Checkbox } from "./ui/checkbox";
	20	+	import { WeightedLru } from "../lib/content-cache";
19	21		import { type AlertKind, G1T_MENTION_HREF, type MarkdownRepo, rehypeAlerts, rehypeReferences } from "../lib/markdown-plugins";
	22	+	import { markdownTree, markdownWeight, renderMarkdownTree } from "../lib/markdown-tree";
20	23		import { imageSource } from "../lib/usercontent";
21	24		import { UserCard } from "./user-card";
22	25
⋯
47	50		},
48	51		};
49	52
	53	+	/**
	54	+	* Parsed markdown, by repository and text: parsing and the plugins are
	55	+	* nine tenths of rendering a README, and a page's markdown is rendered
	56	+	* again on every view (and on every revalidation in the browser). The tree
	57	+	* depends only on the text and the repository its references point into
	58	+	* (lib/markdown-tree.ts). A tree takes about 20 bytes a character of its
	59	+	* text, so 400,000 characters of markdown, about 8 MB, per
	60	+	* isolate or tab.
	61	+	*/
	62	+	const trees = new WeightedLru<{ tree: Root; weight: number }>(400_000, (entry) => entry.weight);
	63	+
	64	+	function treeOf(source: string, repo: MarkdownRepo \| undefined): Root {
	65	+	const key = `${repo ? `${repo.namespace}/${repo.name}` : ""}\n${source}`;
	66	+	const kept = trees.get(key);
	67	+	if (kept) return kept.tree;
	68	+	const tree = markdownTree(source, {
	69	+	remarkPlugins: [remarkGfm],
	70	+	rehypePlugins: [rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]],
	71	+	});
	72	+	trees.set(key, { tree, weight: markdownWeight(source) });
	73	+	return tree;
	74	+	}
	75	+
50	76		const ALERT: Record<AlertKind, { title: string; icon: ReactNode; tone: string }> = {
51	77		note: { title: "Note", icon: <Info size={15} />, tone: "border-info/60 [&_.alert-title]:text-info" },
52	78		tip: { title: "Tip", icon: <Lightbulb size={15} />, tone: "border-success/60 [&_.alert-title]:text-accent" },
⋯
177	203		}) {
178	204		return (
179	205		<div className="prose">
180		−	<ReactMarkdown
181		−	remarkPlugins={[remarkGfm]}
182		−	rehypePlugins={[rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]]}
183		−	components={{
184		−	h1: ({ children }) => <Heading level={1}>{children}</Heading>,
185		−	h2: ({ children }) => <Heading level={2}>{children}</Heading>,
186		−	h3: ({ children }) => <Heading level={3}>{children}</Heading>,
187		−	h4: ({ children }) => <Heading level={4}>{children}</Heading>,
188		−	a({ href = "", children, node }) {
189		−	const ref = (node?.properties as { dataRef?: string } \| undefined)?.dataRef;
190		−	if (ref === "mention") {
191		−	// `@name` may be a person (with a card) or a workspace (none).
192		−	const name = href === G1T_MENTION_HREF ? "g1t" : href.replace(/^\//, "");
193		−	return (
194		−	<UserCard username={name}>
195		−	<Link to={href} prefetch="intent" className="font-medium">
196		−	{children}
197		−	</Link>
198		−	</UserCard>
199		−	);
200		−	}
201		−	if (ref) {
202		−	return (
203		−	<Link
204		−	to={href}
205		−	prefetch="intent"
206		−	className={ref === "commit" ? "font-mono text-[0.9em]" : ref === "mention" ? "font-medium" : ""}
207		−	>
	206	+	{renderMarkdownTree(treeOf(source, repo), {
	207	+	h1: ({ children }) => <Heading level={1}>{children}</Heading>,
	208	+	h2: ({ children }) => <Heading level={2}>{children}</Heading>,
	209	+	h3: ({ children }) => <Heading level={3}>{children}</Heading>,
	210	+	h4: ({ children }) => <Heading level={4}>{children}</Heading>,
	211	+	a({ href = "", children, node }) {
	212	+	const ref = (node?.properties as { dataRef?: string } \| undefined)?.dataRef;
	213	+	if (ref === "mention") {
	214	+	// `@name` may be a person (with a card) or a workspace (none).
	215	+	const name = href === G1T_MENTION_HREF ? "g1t" : href.replace(/^\//, "");
	216	+	return (
	217	+	<UserCard username={name}>
	218	+	<Link to={href} prefetch="intent" className="font-medium">
208	219		{children}
209	220		</Link>
210		−	);
211		−	}
212		−	if (href.startsWith("#")) return <a href={href}>{children}</a>;
213		−	if (isExternal(href)) {
214		−	return (
215		−	<a href={href} rel="noreferrer nofollow ugc" target="_blank">
216		−	{children}
217		−	</a>
218		−	);
219		−	}
220		−	// A link within the site, or relative to the document's folder.
221		−	const to = href.startsWith("/") \|\| !base ? href : `${base}/${href.replace(/^\.\//, "")}`;
222		−	return <Link to={to}>{children}</Link>;
223		−	},
224		−	blockquote({ children, node }) {
225		−	const kind = (node?.properties as { dataAlert?: AlertKind } \| undefined)?.dataAlert;
226		−	if (!kind \|\| !ALERT[kind]) return <blockquote>{children}</blockquote>;
227		−	const alert = ALERT[kind];
	221	+	</UserCard>
	222	+	);
	223	+	}
	224	+	if (ref) {
	225	+	return (
	226	+	<Link
	227	+	to={href}
	228	+	prefetch="intent"
	229	+	className={ref === "commit" ? "font-mono text-[0.9em]" : ref === "mention" ? "font-medium" : ""}
	230	+	>
	231	+	{children}
	232	+	</Link>
	233	+	);
	234	+	}
	235	+	if (href.startsWith("#")) return <a href={href}>{children}</a>;
	236	+	if (isExternal(href)) {
228	237		return (
229		−	<div className={`markdown-alert border-l-2 py-1 pl-4 ${alert.tone}`}>
230		−	<p className="alert-title flex items-center gap-2 text-sm font-medium">
231		−	{alert.icon}
232		−	{alert.title}
233		−	</p>
234		−	<div className="mt-1 [&>+]:mt-3">{children}</div>
235		−	</div>
	238	+	<a href={href} rel="noreferrer nofollow ugc" target="_blank">
	239	+	{children}
	240	+	</a>
236	241		);
237		−	},
238		−	pre({ children }) {
239		−	const code = Array.isArray(children) ? children[0] : children;
240		−	if (isValidElement<{ className?: string; children?: ReactNode }>(code)) {
241		−	const language = /language-([\w+-]+)/.exec(code.props.className ?? "")?.[1] ?? null;
242		−	return <CodeBlock language={language} code={textOf(code.props.children).replace(/\n$/, "")} />;
243		−	}
244		−	return <pre>{children}</pre>;
245		−	},
246		−	input({ type, checked, disabled }) {
247		−	// Task list boxes: shown, not editable.
248		−	return type === "checkbox" ? (
249		−	<Checkbox
250		−	checked={checked === true}
251		−	disabled={disabled !== false}
252		−	aria-label={checked ? "Done" : "Not done"}
253		−	className="mr-1.5 inline-flex translate-y-0.5 disabled:cursor-default disabled:opacity-100"
254		−	/>
255		−	) : null;
256		−	},
257		−	img({ src, alt }) {
258		−	const at = typeof src === "string" ? imageSource(src, rawBase) : undefined;
259		−	return <img src={at} alt={alt ?? ""} loading="lazy" className="inline max-w-full rounded" />;
260		−	},
261		−	}}
262		−	>
263		−	{source}
264		−	</ReactMarkdown>
	242	+	}
	243	+	// A link within the site, or relative to the document's folder.
	244	+	const to = href.startsWith("/") \|\| !base ? href : `${base}/${href.replace(/^\.\//, "")}`;
	245	+	return <Link to={to}>{children}</Link>;
	246	+	},
	247	+	blockquote({ children, node }) {
	248	+	const kind = (node?.properties as { dataAlert?: AlertKind } \| undefined)?.dataAlert;
	249	+	if (!kind \|\| !ALERT[kind]) return <blockquote>{children}</blockquote>;
	250	+	const alert = ALERT[kind];
	251	+	return (
	252	+	<div className={`markdown-alert border-l-2 py-1 pl-4 ${alert.tone}`}>
	253	+	<p className="alert-title flex items-center gap-2 text-sm font-medium">
	254	+	{alert.icon}
	255	+	{alert.title}
	256	+	</p>
	257	+	<div className="mt-1 [&>+]:mt-3">{children}</div>
	258	+	</div>
	259	+	);
	260	+	},
	261	+	pre({ children }) {
	262	+	const code = Array.isArray(children) ? children[0] : children;
	263	+	if (isValidElement<{ className?: string; children?: ReactNode }>(code)) {
	264	+	const language = /language-([\w+-]+)/.exec(code.props.className ?? "")?.[1] ?? null;
	265	+	return <CodeBlock language={language} code={textOf(code.props.children).replace(/\n$/, "")} />;
	266	+	}
	267	+	return <pre>{children}</pre>;
	268	+	},
	269	+	input({ type, checked, disabled }) {
	270	+	// Task list boxes: shown, not editable.
	271	+	return type === "checkbox" ? (
	272	+	<Checkbox
	273	+	checked={checked === true}
	274	+	disabled={disabled !== false}
	275	+	aria-label={checked ? "Done" : "Not done"}
	276	+	className="mr-1.5 inline-flex translate-y-0.5 disabled:cursor-default disabled:opacity-100"
	277	+	/>
	278	+	) : null;
	279	+	},
	280	+	img({ src, alt }) {
	281	+	const at = typeof src === "string" ? imageSource(src, rawBase) : undefined;
	282	+	return <img src={at} alt={alt ?? ""} loading="lazy" className="inline max-w-full rounded" />;
	283	+	},
	284	+	} satisfies Components)}
265	285		</div>
266	286		);
267	287		}

1	+	import assert from "node:assert/strict";
2	+	import { readFileSync } from "node:fs";
3	+	import { test } from "node:test";
4	+
5	+	import { createElement } from "react";
6	+	import { renderToStaticMarkup } from "react-dom/server";
7	+	import ReactMarkdown, { type Components } from "react-markdown";
8	+	import rehypeRaw from "rehype-raw";
9	+	import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
10	+	import remarkGfm from "remark-gfm";
11	+
12	+	import { rehypeAlerts, rehypeReferences } from "./markdown-plugins.ts";
13	+	import { markdownTree, renderMarkdownTree } from "./markdown-tree.ts";
14	+
15	+	// components/markdown.tsx's options: GitHub flavour, sanitized raw HTML,
16	+	// alerts and references.
17	+	const SCHEMA = {
18	+	...defaultSchema,
19	+	attributes: {
20	+	...defaultSchema.attributes,
21	+	code: [...(defaultSchema.attributes?.code ?? []), ["className", /^language-./]],
22	+	},
23	+	};
24	+	const repo = { namespace: "acme", name: "web" };
25	+	const options = {
26	+	remarkPlugins: [remarkGfm],
27	+	rehypePlugins: [rehypeRaw, [rehypeSanitize, SCHEMA], rehypeAlerts, [rehypeReferences, { repo }]],
28	+	} as const;
29	+
30	+	// Components that read the node they are given, as the real ones do.
31	+	const components: Components = {
32	+	a: ({ href, children, node }) =>
33	+	createElement("a", { href, "data-ref": String((node?.properties as { dataRef?: string } \| undefined)?.dataRef ?? "") }, children),
34	+	blockquote: ({ children, node }) =>
35	+	createElement("blockquote", { "data-alert": String((node?.properties as { dataAlert?: string } \| undefined)?.dataAlert ?? "") }, children),
36	+	h2: ({ children }) => createElement("h2", { className: "heading" }, children),
37	+	img: ({ src, alt }) => createElement("img", { src: typeof src === "string" ? `/raw/${src}` : undefined, alt: alt ?? "" }),
38	+	};
39	+
40	+	const root = new URL("../../../../", import.meta.url);
41	+	const SAMPLES: [string, string][] = [
42	+	["README.md", readFileSync(new URL("README.md", root), "utf8")],
43	+	["docs/PERFORMANCE.md", readFileSync(new URL("docs/PERFORMANCE.md", root), "utf8")],
44	+	["CONTRIBUTING.md", readFileSync(new URL("CONTRIBUTING.md", root), "utf8")],
45	+	[
46	+	"edge cases",
47	+	[
48	+	"# Title",
49	+	"",
50	+	"> [!WARNING]",
51	+	"> Careful with #12, acme/api#3 and @Ana, and commit 0123456789abcdef0123456789abcdef01234567.",
52	+	"",
53	+	"- [x] done",
54	+	"- [ ] not done",
55	+	"",
56	+	"\| a \| b \|",
57	+	"\| - \| - \|",
58	+	"\| 1 \| 2 \|",
59	+	"",
60	+	"Footnote[^1] and ~~gone~~ and https://example.com.",
61	+	"",
62	+	"[^1]: The note.",
63	+	"",
64	+	'<div align="center"><img src="logo.png" alt="Logo" onerror="alert(1)"><script>alert(1)</script></div>',
65	+	"",
66	+	"[bad](javascript:alert(1)) [rel](./docs/x.md) [abs](/acme/web) ![pic](img/a.png)",
67	+	"",
68	+	"<details><summary>More</summary>",
69	+	"",
70	+	"Inside details.",
71	+	"",
72	+	"</details>",
73	+	"",
74	+	"```ts",
75	+	"const a = 1;",
76	+	"```",
77	+	"",
78	+	'<a href="vbscript:x" title="t">raw link</a> <iframe src="https://evil"></iframe>',
79	+	].join("\n"),
80	+	],
81	+	["empty", ""],
82	+	];
83	+
84	+	for (const [name, source] of SAMPLES) {
85	+	test(`${name}: the kept tree renders exactly what react-markdown renders`, () => {
86	+	const expected = renderToStaticMarkup(createElement(ReactMarkdown, { ...options, components, children: source }));
87	+	const tree = markdownTree(source, options as never);
88	+	assert.equal(renderToStaticMarkup(renderMarkdownTree(tree, components)), expected);
89	+	// Rendering does not change the tree: rendered again, it is the same.
90	+	assert.equal(renderToStaticMarkup(renderMarkdownTree(tree, components)), expected);
91	+	});
92	+	}

1	+	import type { Root } from "hast";
2	+	import { type Components as JsxComponents, toJsxRuntime } from "hast-util-to-jsx-runtime";
3	+	import { urlAttributes } from "html-url-attributes";
4	+	import type { ReactElement } from "react";
5	+	import { Fragment, jsx, jsxs } from "react/jsx-runtime";
6	+	import { type Components, defaultUrlTransform } from "react-markdown";
7	+	import remarkParse from "remark-parse";
8	+	import remarkRehype from "remark-rehype";
9	+	import { type PluggableList, unified } from "unified";
10	+	import { visit } from "unist-util-visit";
11	+	import { VFile } from "vfile";
12	+
13	+	/**
14	+	* Markdown in two steps, the same two `react-markdown`'s `<Markdown>` takes
15	+	* on every render, split so the first can be kept:
16	+	*
17	+	* 1. `markdownTree`: parse, run the plugins, make URLs safe. Most of the
18	+	* work (nine tenths of a large README's), and it depends only on the
19	+	* text and the plugins, so components/markdown.tsx keeps its trees.
20	+	* 2. `renderMarkdownTree`: the tree as React elements, with the page's
21	+	* components.
22	+	*
23	+	* Together they make exactly what `<Markdown>` makes for the same options
24	+	* (lib/markdown-tree.test.ts renders both and compares). Options this file
25	+	* does not take (`allowedElements`, `skipHtml`, `urlTransform`, …) are not
26	+	* used by g1t.
27	+	*/
28	+	export type MarkdownTreeOptions = {
29	+	remarkPlugins?: PluggableList;
30	+	rehypePlugins?: PluggableList;
31	+	};
32	+
33	+	/** The hast tree `<Markdown>` would render for `source`. Not changed by rendering, so it can be kept and rendered again. */
34	+	export function markdownTree(source: string, options: MarkdownTreeOptions = {}): Root {
35	+	const processor = unified()
36	+	.use(remarkParse)
37	+	.use(options.remarkPlugins ?? [])
38	+	.use(remarkRehype, { allowDangerousHtml: true })
39	+	.use(options.rehypePlugins ?? []);
40	+	const file = new VFile();
41	+	file.value = source;
42	+	const tree = processor.runSync(processor.parse(file), file) as Root;
43	+	// What react-markdown does before rendering: raw HTML left over becomes
44	+	// text, and every URL attribute goes through the default transform.
45	+	visit(tree, (node, index, parent) => {
46	+	if (node.type === "raw" && parent && typeof index === "number") {
47	+	parent.children[index] = { type: "text", value: node.value };
48	+	return index;
49	+	}
50	+	if (node.type === "element") {
51	+	for (const key in urlAttributes) {
52	+	if (Object.hasOwn(urlAttributes, key) && Object.hasOwn(node.properties, key)) {
53	+	const value = node.properties[key];
54	+	const test = urlAttributes[key];
55	+	if (test === null \|\| test.includes(node.tagName)) {
56	+	node.properties[key] = defaultUrlTransform(String(value \|\| ""));
57	+	}
58	+	}
59	+	}
60	+	}
61	+	return undefined;
62	+	});
63	+	return tree;
64	+	}
65	+
66	+	/** `tree` as React elements, as `<Markdown>` renders it with `components`. */
67	+	export function renderMarkdownTree(tree: Root, components?: Components): ReactElement {
68	+	return toJsxRuntime(tree, {
69	+	Fragment,
70	+	// react-markdown's components, which it hands to this same function.
71	+	components: components as JsxComponents \| undefined,
72	+	ignoreInvalidStyle: true,
73	+	jsx,
74	+	jsxs,
75	+	passKeys: true,
76	+	passNode: true,
77	+	});
78	+	}
79	+
80	+	/** How much a kept tree weighs: its text's length, a stand-in for the tree's size. */
81	+	export function markdownWeight(source: string): number {
82	+	return source.length + 64;
83	+	}

19	19		"clsx": "^2.1.1",
20	20		"cmdk": "^1.1.1",
21	21		"isbot": "^5.1.36",
	22	+	"hast-util-to-jsx-runtime": "^2.3.6",
	23	+	"html-url-attributes": "^3.0.1",
22	24		"lucide-react": "^1.49.0",
23	25		"mdast-util-find-and-replace": "^3.0.2",
24	26		"radix-ui": "^1.6.7",
⋯
29	31		"rehype-raw": "^7.0.0",
30	32		"rehype-sanitize": "^6.0.0",
31	33		"remark-gfm": "^4.0.1",
	34	+	"remark-parse": "^11.0.0",
	35	+	"remark-rehype": "^11.1.2",
32	36		"shiki": "^4.5.0",
33	37		"simple-icons": "^16.34.0",
34	38		"tailwind-merge": "^3.7.0",
35	39		"unist-util-visit": "^5.1.0",
36		−	"uqr": "^0.1.3"
	40	+	"unified": "^11.0.5",
	41	+	"uqr": "^0.1.3",
	42	+	"vfile": "^6.0.3"
37	43		},
38	44		"devDependencies": {
39	45		"@cloudflare/vite-plugin": "^1.62.4",

78	78		"class-variance-authority": "^0.7.1",
79	79		"clsx": "^2.1.1",
80	80		"cmdk": "^1.1.1",
	81	+	"hast-util-to-jsx-runtime": "^2.3.6",
	82	+	"html-url-attributes": "^3.0.1",
81	83		"isbot": "^5.1.36",
82	84		"lucide-react": "^1.49.0",
83	85		"mdast-util-find-and-replace": "^3.0.2",
⋯
89	91		"rehype-raw": "^7.0.0",
90	92		"rehype-sanitize": "^6.0.0",
91	93		"remark-gfm": "^4.0.1",
	94	+	"remark-parse": "^11.0.0",
	95	+	"remark-rehype": "^11.1.2",
92	96		"shiki": "^4.5.0",
93	97		"simple-icons": "^16.34.0",
94	98		"tailwind-merge": "^3.7.0",
	99	+	"unified": "^11.0.5",
95	100		"unist-util-visit": "^5.1.0",
96		−	"uqr": "^0.1.3"
	101	+	"uqr": "^0.1.3",
	102	+	"vfile": "^6.0.3"
97	103		},
98	104		"devDependencies": {
99	105		"@cloudflare/vite-plugin": "^1.62.4",