Skip to content
Sign up

flagon-io/g1t

public
0
491 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Docs: a workspace knowledge base people and agents write together1/**
2 * Markdown into a page's Yjs document: how templates, agents' edits and
3 * accepted suggestions become blocks. Pure (yjs only).
4 *
5 * `parseMarkdown` reads the Markdown agents write (CommonMark plus GitHub's
6 * tables, task lists and alerts, `$$` math and Mermaid fences) into block
7 * specs; `writeBlocks` turns specs into BlockNote's `blockContainer`
8 * elements, with the attributes the editor gives its own blocks, so the
9 * editor shows them exactly as if typed. The inverse is markdown.ts.
10 */
11import * as Y from "yjs";
12
13export type Marks = Record<string, object>;
14export type Run = { text: string; marks?: Marks } | { node: "mention"; attrs: Record<string, string> };
15
16export type BlockSpec = {
17 type: string;
18 props?: Record<string, string | number | boolean>;
19 /** Inline content; a code block's text is one plain run. */
20 content?: Run[];
21 /** A table's cells: rows of cells of runs. */
22 rows?: Run[][][];
23 children?: BlockSpec[];
24};
25
26const ALERT_KINDS: Record<string, string> = { NOTE: "info", IMPORTANT: "info", TIP: "success", WARNING: "warning", CAUTION: "danger" };
27
28// ── Inline ────────────────────────────────────────────────────────────────
29
30/** Inline Markdown as runs: bold, italic, strike, underline, code, links. */
31export function parseInline(text: string, marks: Marks = {}): Run[] {
32 const runs: Run[] = [];
33 let plain = "";
34 const flush = () => {
35 if (plain) runs.push(Object.keys(marks).length ? { text: plain, marks: { ...marks } } : { text: plain });
36 plain = "";
37 };
38 let i = 0;
39 while (i < text.length) {
40 const rest = text.slice(i);
41 // Escapes.
42 if (rest[0] === "\\" && rest.length > 1 && /[\\`*_{}[\]()#+\-.!~|<>$]/.test(rest[1]!)) {
43 plain += rest[1];
44 i += 2;
45 continue;
46 }
47 // Hard break: a backslash at the end of a line.
48 if (rest.startsWith("\\\n")) {
49 plain += "\n";
50 i += 2;
51 continue;
52 }
53 // Code.
54 const code = /^(`+)([\s\S]*?[^`])\1(?!`)/.exec(rest);
55 if (code) {
56 flush();
57 runs.push({ text: code[2]!.replace(/^ (.*) $/, "$1"), marks: { ...marks, code: {} } });
58 i += code[0].length;
59 continue;
60 }
61 // Images inline: keep as a link to the image.
62 const link = /^!?\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/.exec(rest);
63 if (link) {
64 flush();
65 runs.push(...parseInline(link[1]! || link[2]!, { ...marks, link: { href: link[2]! } }));
66 i += link[0].length;
67 continue;
68 }
69 const auto = /^<(https?:\/\/[^>\s]+)>/.exec(rest);
70 if (auto) {
71 flush();
72 runs.push({ text: auto[1]!, marks: { ...marks, link: { href: auto[1]! } } });
73 i += auto[0].length;
74 continue;
75 }
76 const underline = /^<u>([\s\S]+?)<\/u>/.exec(rest);
77 if (underline) {
78 flush();
79 runs.push(...parseInline(underline[1]!, { ...marks, underline: {} }));
80 i += underline[0].length;
81 continue;
82 }
83 const strong = /^(\*\*|__)(?=\S)([\s\S]*?\S)\1/.exec(rest);
84 if (strong) {
85 flush();
86 runs.push(...parseInline(strong[2]!, { ...marks, bold: {} }));
87 i += strong[0].length;
88 continue;
89 }
90 const strike = /^~~(?=\S)([\s\S]*?\S)~~/.exec(rest);
91 if (strike) {
92 flush();
93 runs.push(...parseInline(strike[1]!, { ...marks, strike: {} }));
94 i += strike[0].length;
95 continue;
96 }
97 // `_` only at a word boundary, so snake_case stays as written.
98 const em = /^\*(?=\S)([\s\S]*?\S)\*(?!\*)/.exec(rest) ?? (/[A-Za-z0-9]$/.test(text.slice(0, i)) ? null : /^_(?=\S)([\s\S]*?\S)_(?![A-Za-z0-9])/.exec(rest));
99 if (em) {
100 flush();
101 runs.push(...parseInline(em[1]!, { ...marks, italic: {} }));
102 i += em[0].length;
103 continue;
104 }
105 plain += rest[0];
106 i += 1;
107 }
108 flush();
109 return merge(runs);
110}
111
112function sameMarks(a: Marks | undefined, b: Marks | undefined): boolean {
113 return JSON.stringify(a ?? {}) === JSON.stringify(b ?? {});
114}
115
116function merge(runs: Run[]): Run[] {
117 const out: Run[] = [];
118 for (const run of runs) {
119 const last = out[out.length - 1];
120 if (last && "text" in last && "text" in run && sameMarks(last.marks, run.marks)) last.text += run.text;
121 else out.push("text" in run ? { ...run } : run);
122 }
123 return out.filter((r) => !("text" in r) || r.text !== "");
124}
125
126// ── Blocks ────────────────────────────────────────────────────────────────
127
128type Line = { indent: number; text: string };
129
130function lineOf(raw: string): Line {
131 const expanded = raw.replace(/\t/g, " ");
132 const indent = /^ */.exec(expanded)![0].length;
133 return { indent, text: expanded.slice(indent) };
134}
135
136const LIST_ITEM = /^([-*+]|\d{1,9}[.)])\s+(.*)$/;
137const TABLE_RULE = /^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$/;
138
139function splitRow(line: string): string[] {
140 let s = line.trim();
141 if (s.startsWith("|")) s = s.slice(1);
142 if (s.endsWith("|") && !s.endsWith("\\|")) s = s.slice(0, -1);
143 const cells: string[] = [];
144 let cell = "";
145 for (let i = 0; i < s.length; i++) {
146 if (s[i] === "\\" && s[i + 1] === "|") {
147 cell += "|";
148 i++;
149 } else if (s[i] === "|") {
150 cells.push(cell.trim());
151 cell = "";
152 } else cell += s[i];
153 }
154 cells.push(cell.trim());
155 return cells;
156}
157
158function isBlockStart(text: string): boolean {
159 return /^(#{1,6}\s|>|```|~~~|\$\$|<details>|(-{3,}|\*{3,}|_{3,})\s*$)/.test(text) || LIST_ITEM.test(text) || /^!\[[^\]]*\]\([^)]+\)\s*$/.test(text);
160}
161
162/** Markdown as block specs. */
163export function parseMarkdown(markdown: string): BlockSpec[] {
164 const lines = String(markdown ?? "").replace(/\r\n?/g, "\n").split("\n");
165 return parseLines(lines.map(lineOf), 0, lines.length);
166}
167
168function parseLines(lines: Line[], from: number, to: number): BlockSpec[] {
169 const out: BlockSpec[] = [];
170 let i = from;
171 while (i < to) {
172 const line = lines[i]!;
173 const text = line.text;
174 if (!text.trim()) {
175 i++;
176 continue;
177 }
178 // Fenced code, Mermaid.
179 const fenceOpen = /^(`{3,}|~{3,})\s*([^\s`]*)/.exec(text);
180 if (fenceOpen) {
181 const marker = fenceOpen[1]!;
182 const lang = fenceOpen[2]!.toLowerCase();
183 const body: string[] = [];
184 i++;
185 while (i < to && !lines[i]!.text.startsWith(marker)) {
186 body.push(" ".repeat(Math.max(0, lines[i]!.indent - line.indent)) + lines[i]!.text);
187 i++;
188 }
189 i++;
190 const code = body.join("\n");
191 if (lang === "mermaid") out.push({ type: "mermaid", props: { code } });
192 else if (lang === "math" || lang === "latex" || lang === "katex") out.push({ type: "math", props: { expression: code } });
193 else out.push({ type: "codeBlock", props: { language: lang || "text" }, content: code ? [{ text: code }] : [] });
194 continue;
195 }
196 // Math.
197 if (text.startsWith("$$")) {
198 const same = /^\$\$(.+)\$\$\s*$/.exec(text);
199 if (same) {
200 out.push({ type: "math", props: { expression: same[1]!.trim() } });
201 i++;
202 continue;
203 }
204 const body: string[] = [];
205 i++;
206 while (i < to && !lines[i]!.text.startsWith("$$")) body.push(lines[i++]!.text);
207 i++;
208 out.push({ type: "math", props: { expression: body.join("\n").trim() } });
209 continue;
210 }
211 // Headings.
212 const heading = /^(#{1,6})\s+(.*?)\s*#*\s*$/.exec(text);
213 if (heading) {
214 out.push({ type: "heading", props: { level: Math.min(heading[1]!.length, 6) }, content: parseInline(heading[2]!) });
215 i++;
216 continue;
217 }
218 // Dividers.
219 if (/^(-{3,}|\*{3,}|_{3,})\s*$/.test(text)) {
220 out.push({ type: "divider" });
221 i++;
222 continue;
223 }
224 // Toggles.
225 if (text.startsWith("<details>")) {
226 let summary = "";
227 const inner: Line[] = [];
228 let depth = 0;
229 const first = text.slice("<details>".length);
230 const queue = first.trim() ? [{ indent: 0, text: first.trim() }] : [];
231 i++;
232 const take = (l: Line): boolean => {
233 const sm = /^<summary>([\s\S]*?)<\/summary>\s*(.*)$/.exec(l.text);
234 if (sm && !summary) {
235 summary = sm[1]!;
236 if (sm[2]) inner.push({ indent: 0, text: sm[2] });
237 return false;
238 }
239 if (l.text.startsWith("<details>")) depth++;
240 if (l.text.startsWith("</details>")) {
241 if (depth === 0) return true;
242 depth--;
243 }
244 inner.push(l);
245 return false;
246 };
247 let closed = false;
248 for (const l of queue) if (take(l)) closed = true;
249 while (!closed && i < to) {
250 closed = take(lines[i]!);
251 i++;
252 }
253 out.push({ type: "toggleListItem", content: parseInline(summary.trim()), children: parseLines(inner, 0, inner.length) });
254 continue;
255 }
256 // Quotes and callouts.
257 if (text.startsWith(">")) {
258 const body: string[] = [];
259 while (i < to && lines[i]!.text.startsWith(">")) {
260 body.push(lines[i]!.text.replace(/^>\s?/, ""));
261 i++;
262 }
263 const alert = /^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*(.*)$/i.exec(body[0] ?? "");
264 if (alert) {
265 const rest = [alert[2]!, ...body.slice(1)].filter((l, n) => n > 0 || l.trim()).join("\n").trim();
266 out.push({ type: "callout", props: { kind: ALERT_KINDS[alert[1]!.toUpperCase()] ?? "info" }, content: parseInline(joinParagraph(rest)) });
267 } else {
268 out.push({ type: "quote", content: parseInline(joinParagraph(body.join("\n").trim())) });
269 }
270 continue;
271 }
272 // Images on a line of their own.
273 const image = /^!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)\s*$/.exec(text);
274 if (image) {
275 out.push({ type: "image", props: { url: image[2]!, caption: image[1]!, name: image[1]! } });
276 i++;
277 continue;
278 }
279 // Tables.
280 if (text.includes("|") && i + 1 < to && TABLE_RULE.test(lines[i + 1]!.text)) {
281 const rows: Run[][][] = [splitRow(text).map((c) => parseInline(c))];
282 i += 2;
283 while (i < to && lines[i]!.text.includes("|") && lines[i]!.text.trim()) {
284 rows.push(splitRow(lines[i]!.text).map((c) => parseInline(c)));
285 i++;
286 }
287 out.push({ type: "table", rows });
288 continue;
289 }
290 // Lists.
291 const item = LIST_ITEM.exec(text);
292 if (item) {
293 const base = line.indent;
294 const marker = item[1]!;
295 const contentIndent = base + marker.length + 1;
296 let first = item[2]!;
297 let type = /^\d/.test(marker) ? "numberedListItem" : "bulletListItem";
298 const props: Record<string, string | number | boolean> = {};
299 const task = /^\[([ xX])\]\s+(.*)$/.exec(first);
300 if (task && type === "bulletListItem") {
301 type = "checkListItem";
302 props.checked = task[1] !== " ";
303 first = task[2]!;
304 }
305 if (type === "numberedListItem") {
306 const start = parseInt(marker, 10);
307 if (start !== 1 && !out.some((b) => b.type === "numberedListItem")) props.start = start;
308 }
309 i++;
310 // Continuation lines of the item's own paragraph.
311 const own = [first];
312 while (i < to && lines[i]!.text.trim() && lines[i]!.indent >= contentIndent - 1 && !isBlockStart(lines[i]!.text) && !LIST_ITEM.test(lines[i]!.text)) {
313 own.push(lines[i]!.text);
314 i++;
315 }
316 // Nested blocks: lines indented past the marker (blank lines allowed between).
317 const nested: Line[] = [];
318 while (i < to) {
319 const l = lines[i]!;
320 if (!l.text.trim()) {
321 // A blank line ends the item unless more indented content follows.
322 let j = i + 1;
323 while (j < to && !lines[j]!.text.trim()) j++;
324 if (j < to && lines[j]!.indent > base && lines[j]!.indent >= Math.min(contentIndent, base + 2)) {
325 for (; i < j; i++) nested.push({ indent: 0, text: "" });
326 continue;
327 }
328 break;
329 }
330 if (l.indent > base && l.indent >= Math.min(contentIndent, base + 2)) {
331 nested.push({ indent: l.indent - Math.min(contentIndent, l.indent), text: l.text });
332 i++;
333 continue;
334 }
335 break;
336 }
337 out.push({ type, props, content: parseInline(joinParagraph(own.join("\n"))), children: parseLines(nested, 0, nested.length) });
338 continue;
339 }
340 // A paragraph: lines up to a blank line or another block.
341 const para = [text];
342 i++;
343 while (i < to && lines[i]!.text.trim() && !isBlockStart(lines[i]!.text) && !(lines[i]!.text.includes("|") && i + 1 < to && TABLE_RULE.test(lines[i + 1]!.text))) {
344 para.push(lines[i]!.text);
345 i++;
346 }
347 out.push({ type: "paragraph", content: parseInline(joinParagraph(para.join("\n"))) });
348 }
349 return out;
350}
351
352/** Soft line breaks become spaces; a trailing backslash or two spaces stay a hard break. */
353function joinParagraph(text: string): string {
354 return text.replace(/( {2,}|\\)\n/g, "\\\n").replace(/([^\\])\n/g, "$1 ");
355}
356
357// ── Into Yjs ──────────────────────────────────────────────────────────────
358
359/** The attributes BlockNote gives each of its blocks when it makes them. */
360function defaults(type: string): Record<string, unknown> {
361 const colors = { backgroundColor: "default", textColor: "default" };
362 switch (type) {
363 case "paragraph":
364 case "bulletListItem":
365 case "numberedListItem":
366 case "toggleListItem":
367 return { ...colors, textAlignment: "left" };
368 case "checkListItem":
369 return { ...colors, textAlignment: "left", checked: false };
370 case "heading":
371 return { ...colors, textAlignment: "left", level: 1, isToggleable: false };
372 case "quote":
373 return { ...colors };
374 case "codeBlock":
375 return { language: "text" };
376 case "image":
377 return { textAlignment: "left", backgroundColor: "default", name: "", url: "", caption: "", showPreview: true };
378 case "table":
379 return { textColor: "default" };
380 case "callout":
381 return { ...colors, kind: "info" };
382 case "mermaid":
383 return { code: "" };
384 case "math":
385 return { expression: "" };
386 default:
387 return {};
388 }
389}
390
391function newBlockId(): string {
392 return crypto.randomUUID();
393}
394
395function textOf(runs: Run[] | undefined): (Y.XmlText | Y.XmlElement)[] {
396 const out: (Y.XmlText | Y.XmlElement)[] = [];
397 let text: Y.XmlText | null = null;
398 const pending: Run[] = [];
399 const flush = () => {
400 if (!pending.length) return;
401 text = new Y.XmlText();
402 let at = 0;
403 for (const run of pending) {
404 if (!("text" in run)) continue;
405 text.insert(at, run.text, run.marks && Object.keys(run.marks).length ? run.marks : {});
406 at += run.text.length;
407 }
408 out.push(text);
409 pending.length = 0;
410 };
411 for (const run of runs ?? []) {
412 if ("text" in run) pending.push(run);
413 else {
414 flush();
415 const el = new Y.XmlElement(run.node);
416 for (const [k, v] of Object.entries(run.attrs)) el.setAttribute(k, v);
417 out.push(el);
418 }
419 }
420 flush();
421 return out;
422}
423
424function tableNode(rows: Run[][][]): Y.XmlElement {
425 const table = new Y.XmlElement("table");
426 for (const [k, v] of Object.entries(defaults("table"))) table.setAttribute(k, v as string);
427 const width = Math.max(1, ...rows.map((r) => r.length));
428 table.insert(
429 0,
430 rows.map((row) => {
431 const tr = new Y.XmlElement("tableRow");
432 tr.insert(
433 0,
434 Array.from({ length: width }, (_, n) => {
435 const cell = new Y.XmlElement("tableCell");
436 for (const [k, v] of Object.entries({ textColor: "default", backgroundColor: "default", textAlignment: "left", colspan: 1, rowspan: 1 })) cell.setAttribute(k, v as unknown as string);
437 const p = new Y.XmlElement("tableParagraph");
438 p.insert(0, textOf(row[n] ?? []));
439 cell.insert(0, [p]);
440 return cell;
441 }),
442 );
443 return tr;
444 }),
445 );
446 return table;
447}
448
449/** One block spec as a `blockContainer` (with its children), ready to insert. */
450export function containerOf(spec: BlockSpec): Y.XmlElement {
451 const container = new Y.XmlElement("blockContainer");
452 container.setAttribute("id", newBlockId());
453 let content: Y.XmlElement;
454 if (spec.type === "table") content = tableNode(spec.rows ?? []);
455 else {
456 content = new Y.XmlElement(spec.type);
457 for (const [k, v] of Object.entries({ ...defaults(spec.type), ...(spec.props ?? {}) })) content.setAttribute(k, v as string);
458 if (spec.content?.length) content.insert(0, textOf(spec.content));
459 }
460 const parts: Y.XmlElement[] = [content];
461 if (spec.children?.length) {
462 const group = new Y.XmlElement("blockGroup");
463 group.insert(0, spec.children.map(containerOf));
464 parts.push(group);
465 }
466 container.insert(0, parts);
467 return container;
468}
469
470/** Block specs as containers. An empty list becomes one empty paragraph, as the editor keeps. */
471export function writeBlocks(specs: BlockSpec[]): Y.XmlElement[] {
472 return (specs.length ? specs : [{ type: "paragraph" }]).map(containerOf);
473}
474
475/** The fragment's block group, made when the document is new. */
476export function blockGroupOf(fragment: Y.XmlFragment): Y.XmlElement {
477 const found = fragment.toArray().find((n): n is Y.XmlElement => n instanceof Y.XmlElement && n.nodeName === "blockGroup");
478 if (found) return found;
479 const group = new Y.XmlElement("blockGroup");
480 fragment.insert(fragment.length, [group]);
481 return group;
482}
483
484/** Fills an empty document from Markdown (a template, an agent's new page). */
485export function seed(doc: Y.Doc, fragment: Y.XmlFragment, markdown: string): void {
486 doc.transact(() => {
487 const group = blockGroupOf(fragment);
488 if (group.length) group.delete(0, group.length);
489 group.insert(0, writeBlocks(parseMarkdown(markdown)));
490 });
491}