| 1 | import { waitUntil } from "cloudflare:workers"; |
| 2 | |
| 3 | import type { Comparison } from "@g1t/contracts"; |
| 4 | |
| 5 | import { type SharedCache, contentCache, weightOfLines } from "./content-cache"; |
| 6 | import { type FileHtml, type HighlightedComparison, type HighlightedFile, diffContent, htmlOfFile, withHtml } from "./diff"; |
| 7 | |
| 8 | // Shiki and its grammars load on the first file highlighted, not when the |
| 9 | // Worker starts: most requests highlight nothing. The module is asked for |
| 10 | // once per isolate. |
| 11 | let shikiModule: Promise<typeof import("./shiki")> | undefined; |
| 12 | const shiki = () => (shikiModule ??= import("./shiki")); |
| 13 | |
| 14 | const MAX_HIGHLIGHT_CHARS = 200_000; |
| 15 | |
| 16 | /** |
| 17 | * Highlighting is most of the work of a file's page (about two thirds of a |
| 18 | * small file's, nine tenths of a large one's), and the same text always |
| 19 | * highlights the same way. So it is kept by a hash of the text and its |
| 20 | * language: in the isolate, and in the data centre's cache for the other |
| 21 | * isolates there (lib/content-cache.ts). A file read again on another |
| 22 | * branch or commit, through blame, or by the next crawler, is highlighted |
| 23 | * once. Bump `HIGHLIGHT_VERSION` when the theme, the grammars, Shiki or |
| 24 | * `linesToHtml` change what they make. |
| 25 | */ |
| 26 | export const HIGHLIGHT_VERSION = "v2"; |
| 27 | |
| 28 | function dataCentre(): SharedCache | null { |
| 29 | try { |
| 30 | return (caches as unknown as { default: Cache }).default ?? null; |
| 31 | } catch { |
| 32 | // No cache (local development). |
| 33 | return null; |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | function defer(work: Promise<unknown>) { |
| 38 | try { |
| 39 | waitUntil(work); |
| 40 | } catch { |
| 41 | // Outside a request: the write finishes or not; nothing waits on it. |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | /** |
| 46 | * About 8 MB of highlighted lines per isolate (two bytes a character); one |
| 47 | * file up to a quarter of that, which a 1,800-line file fits. A pull |
| 48 | * request's first screens are at most 16,000 characters of text, about |
| 49 | * four times that highlighted. |
| 50 | */ |
| 51 | const linesCache = contentCache<string[]>({ |
| 52 | name: "highlight-lines", |
| 53 | version: HIGHLIGHT_VERSION, |
| 54 | maxWeight: 4_000_000, |
| 55 | weigh: weightOfLines, |
| 56 | shared: dataCentre, |
| 57 | defer, |
| 58 | }); |
| 59 | |
| 60 | const filesCache = contentCache<FileHtml>({ |
| 61 | name: "highlight-diff", |
| 62 | version: HIGHLIGHT_VERSION, |
| 63 | maxWeight: 2_000_000, |
| 64 | weigh: (html) => html.reduce((sum, hunk) => sum + weightOfLines(hunk), 0), |
| 65 | shared: dataCentre, |
| 66 | defer, |
| 67 | }); |
| 68 | |
| 69 | /** |
| 70 | * A file's lines as highlighted HTML, one string per line, or null when its |
| 71 | * language is unknown or it is too large. |
| 72 | */ |
| 73 | export async function highlightLines(path: string, text: string): Promise<string[] | null> { |
| 74 | const { languageOf } = await shiki(); |
| 75 | const lang = languageOf(path); |
| 76 | if (!lang || text.length > MAX_HIGHLIGHT_CHARS) return null; |
| 77 | const code = text.replace(/\n$/, ""); |
| 78 | return linesCache([lang, code], async () => { |
| 79 | const { getHighlighter, linesToHtml } = await shiki(); |
| 80 | try { |
| 81 | return linesToHtml(await getHighlighter(), code, lang); |
| 82 | } catch { |
| 83 | return null; |
| 84 | } |
| 85 | }); |
| 86 | } |
| 87 | |
| 88 | /** |
| 89 | * How much of a change is highlighted before the page is sent: its first |
| 90 | * screens. The rest is highlighted in the browser as it comes near |
| 91 | * (components/diff-view.tsx), so a large change never holds the page up. |
| 92 | * Highlighted HTML is about four times its text, and is sent twice (drawn, |
| 93 | * and in the data the page hydrates from), so this is kept to a few |
| 94 | * screens: about 400 lines. |
| 95 | */ |
| 96 | const FIRST_SCREENS_CHARS = 16_000; |
| 97 | |
| 98 | /** A file's diff highlighted, from the cache when the same lines were highlighted before. */ |
| 99 | async function highlightFileCached(file: Comparison["files"][number]): Promise<HighlightedFile | null> { |
| 100 | const { highlightFile, languageOf } = await shiki(); |
| 101 | const lang = languageOf(file.path); |
| 102 | // Nothing to highlight: highlightFile says so without any work. |
| 103 | if (!lang || file.binary) return null; |
| 104 | const html = await filesCache([lang, diffContent(file)], async () => { |
| 105 | const highlighted = await highlightFile(file).catch(() => null); |
| 106 | return highlighted ? htmlOfFile(highlighted) : null; |
| 107 | }); |
| 108 | return html ? withHtml(file, html) : null; |
| 109 | } |
| 110 | |
| 111 | /** |
| 112 | * A comparison with its first files' lines highlighted, in order, until |
| 113 | * the budget runs out, so what is on screen when the page arrives already |
| 114 | * has its colours. |
| 115 | */ |
| 116 | export async function highlightFirstFiles(comparison: Comparison | HighlightedComparison): Promise<HighlightedComparison> { |
| 117 | let budget = FIRST_SCREENS_CHARS; |
| 118 | const files: HighlightedFile[] = []; |
| 119 | for (const file of comparison.files) { |
| 120 | // Sent without its lines (lib/diff.ts firstScreens): nothing to colour. |
| 121 | if (file.hunks.length === 0) { |
| 122 | files.push(file); |
| 123 | continue; |
| 124 | } |
| 125 | const size = file.hunks.reduce((sum, hunk) => sum + hunk.lines.reduce((n, line) => n + line.text.length + 1, 0), 0); |
| 126 | if (budget <= 0 || size > budget || file.binary) { |
| 127 | // Past the first screens: everything after it waits for the browser. |
| 128 | budget = 0; |
| 129 | files.push(file); |
| 130 | continue; |
| 131 | } |
| 132 | budget -= size; |
| 133 | files.push((await highlightFileCached(file).catch(() => null)) ?? file); |
| 134 | } |
| 135 | return { ...comparison, files }; |
| 136 | } |