Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
232 linesCodeBlameRaw
1/**
2 * Passages: a page's Markdown (or a repository's docs file) split into
3 * sections an agent can be handed whole, for the semantic index
4 * (src/indexer.ts) and recall. Pure.
5 *
6 * A passage is one section under its heading path ("Runbook › Rollback"),
7 * between MIN_CHARS and MAX_CHARS where it can be: long sections split on
8 * paragraph boundaries (a fenced code block stays one paragraph), tiny ones
9 * join the next. What is embedded is the passage with its document's title
10 * and heading in front (`embedText`); what an agent reads is the passage.
11 */
12
13/** A passage shorter than this joins the next one. */
14export const MIN_CHARS = 300;
15/** A passage longer than this splits on paragraphs. */
16export const MAX_CHARS = 1500;
17/** The most passages one document keeps; past it the rest goes unindexed (words still find the page). */
18export const MAX_CHUNKS = 150;
19/** The most text embedded for one passage: the model reads 512 tokens. */
20export const EMBED_CHARS = 2000;
21/** Between heading levels in a passage's heading path. */
22export const HEADING_SEPARATOR = " › ";
23
24export type Chunk = {
25 seq: number;
26 /** The heading path the passage sits under, or null before the first heading. */
27 heading: string | null;
28 /** The passage as Markdown. */
29 text: string;
30};
31
32type Section = { heading: string | null; lines: string[] };
33
34const FENCE = /^\s{0,3}(`{3,}|~{3,})/;
35const HEADING = /^\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$/;
36
37/** A heading's text as plain words: no emphasis, code ticks or links. */
38function headingText(raw: string): string {
39 return raw
40 .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
41 .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
42 .replace(/[*_`~]+/g, "")
43 .replace(/\s+/g, " ")
44 .trim();
45}
46
47/** Drops YAML front matter, as repository docs files often have. */
48function withoutFrontMatter(markdown: string): string {
49 const front = /^---\r?\n[\s\S]*?\r?\n---[ \t]*(\r?\n|$)/.exec(markdown);
50 return front ? markdown.slice(front[0].length) : markdown;
51}
52
53/** The document split at its headings, each section with its heading path. */
54function sections(markdown: string, title: string): Section[] {
55 const lines = withoutFrontMatter(markdown).replace(/\r\n?/g, "\n").split("\n");
56 const out: Section[] = [{ heading: null, lines: [] }];
57 const path: { level: number; text: string }[] = [];
58 let fence: string | null = null;
59 let first = true;
60 const wantTitle = headingText(title).toLowerCase();
61 for (const line of lines) {
62 const f = FENCE.exec(line);
63 if (fence) {
64 if (f && f[1]![0] === fence[0] && f[1]!.length >= fence.length && line.trim() === f[1]) fence = null;
65 out[out.length - 1]!.lines.push(line);
66 continue;
67 }
68 if (f) {
69 fence = f[1]!;
70 out[out.length - 1]!.lines.push(line);
71 continue;
72 }
73 const h = HEADING.exec(line);
74 if (!h) {
75 if (line.trim()) first = false;
76 out[out.length - 1]!.lines.push(line);
77 continue;
78 }
79 const level = h[1]!.length;
80 const text = headingText(h[2]!);
81 // A file's leading `# Title` is its title, not a section of it.
82 if (first && level === 1 && text.toLowerCase() === wantTitle) {
83 first = false;
84 continue;
85 }
86 first = false;
87 while (path.length && path[path.length - 1]!.level >= level) path.pop();
88 path.push({ level, text });
89 out.push({
90 heading:
91 path
92 .map((p) => p.text)
93 .filter(Boolean)
94 .join(HEADING_SEPARATOR) || null,
95 lines: [],
96 });
97 }
98 return out;
99}
100
101/** Paragraphs: runs of lines between blank lines, with a fenced block kept whole. */
102function paragraphs(lines: string[]): string[] {
103 const out: string[] = [];
104 let current: string[] = [];
105 let fence: string | null = null;
106 const flush = () => {
107 const text = current.join("\n").trim();
108 if (text) out.push(text);
109 current = [];
110 };
111 for (const line of lines) {
112 const f = FENCE.exec(line);
113 if (fence) {
114 current.push(line);
115 if (f && f[1]![0] === fence[0] && f[1]!.length >= fence.length && line.trim() === f[1]) fence = null;
116 continue;
117 }
118 if (f) {
119 fence = f[1]!;
120 current.push(line);
121 continue;
122 }
123 if (!line.trim()) flush();
124 else current.push(line);
125 }
126 flush();
127 return out;
128}
129
130/** A paragraph longer than MAX_CHARS, cut at sentence ends or spaces. */
131function cut(text: string): string[] {
132 const out: string[] = [];
133 let rest = text;
134 while (rest.length > MAX_CHARS) {
135 const window = rest.slice(0, MAX_CHARS);
136 let at = Math.max(window.lastIndexOf(". "), window.lastIndexOf(".\n"), window.lastIndexOf("\n"));
137 if (at < MAX_CHARS / 2) at = window.lastIndexOf(" ");
138 if (at < MAX_CHARS / 2) at = MAX_CHARS - 1;
139 out.push(rest.slice(0, at + 1).trim());
140 rest = rest.slice(at + 1).trim();
141 }
142 if (rest) out.push(rest);
143 return out;
144}
145
146/** One section's passages: its paragraphs packed up to MAX_CHARS. */
147function pack(paras: string[]): string[] {
148 const out: string[] = [];
149 let current = "";
150 for (const para of paras.flatMap(cut)) {
151 if (!current) current = para;
152 else if (current.length + 2 + para.length <= MAX_CHARS) current += `\n\n${para}`;
153 else {
154 out.push(current);
155 current = para;
156 }
157 }
158 if (current) out.push(current);
159 return out;
160}
161
162/**
163 * A document's passages, in order. `title` is the page's or file's title:
164 * not part of any passage, but a file's leading `# Title` is dropped as it.
165 */
166export function chunkMarkdown(markdown: string, title = ""): Chunk[] {
167 const pieces: { heading: string | null; text: string }[] = [];
168 for (const section of sections(String(markdown ?? ""), title)) {
169 for (const text of pack(paragraphs(section.lines))) pieces.push({ heading: section.heading, text });
170 // A heading with nothing under it but subsections still names them: nothing to keep alone.
171 }
172 // Tiny passages join the next one (its heading written in), while that fits.
173 const merged: { heading: string | null; text: string }[] = [];
174 for (const piece of pieces) {
175 const last = merged[merged.length - 1];
176 if (last && last.text.length < MIN_CHARS) {
177 const joined =
178 piece.heading && piece.heading !== last.heading
179 ? `${last.text}\n\n**${piece.heading.split(HEADING_SEPARATOR).pop()}**\n\n${piece.text}`
180 : `${last.text}\n\n${piece.text}`;
181 if (joined.length <= MAX_CHARS) {
182 last.text = joined;
183 if (!last.heading) last.heading = piece.heading;
184 continue;
185 }
186 }
187 merged.push({ ...piece });
188 }
189 // A tiny last passage joins the one before it, when that fits.
190 if (merged.length > 1) {
191 const last = merged[merged.length - 1]!;
192 const before = merged[merged.length - 2]!;
193 if (last.text.length < MIN_CHARS && before.text.length + last.text.length + 40 <= MAX_CHARS) {
194 before.text =
195 last.heading && last.heading !== before.heading
196 ? `${before.text}\n\n**${last.heading.split(HEADING_SEPARATOR).pop()}**\n\n${last.text}`
197 : `${before.text}\n\n${last.text}`;
198 merged.pop();
199 }
200 }
201 return merged.slice(0, MAX_CHUNKS).map((piece, seq) => ({ seq, heading: piece.heading, text: piece.text }));
202}
203
204/** What is embedded for a passage: its document's title and heading in front, so a passage alone still says what it is about. */
205export function embedText(title: string, chunk: Pick<Chunk, "heading" | "text">): string {
206 const head = [title.trim(), chunk.heading ?? ""].filter(Boolean).join(HEADING_SEPARATOR);
207 return (head ? `${head}\n\n${chunk.text}` : chunk.text).slice(0, EMBED_CHARS);
208}
209
210/** A short, stable hash (hex) of a string: whether a passage changed since it was embedded. Not for security. */
211export function textHash(text: string): string {
212 let h1 = 0xdeadbeef;
213 let h2 = 0x41c6ce57;
214 for (let i = 0; i < text.length; i++) {
215 const c = text.charCodeAt(i);
216 h1 = Math.imul(h1 ^ c, 2654435761);
217 h2 = Math.imul(h2 ^ c, 1597334677);
218 }
219 h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
220 h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
221 return (h2 >>> 0).toString(16).padStart(8, "0") + (h1 >>> 0).toString(16).padStart(8, "0");
222}
223
224/** A repository docs file's id in the index (`rf_` and 32 hex): short enough for a vector id, the same each time. */
225export function repoFileId(spaceId: string, path: string): string {
226 return `rf_${textHash(`${spaceId}\n${path}`)}${textHash(`${path}\n${spaceId}`)}`;
227}
228
229/** A passage's id, in D1 and in the vector index: `<page or file id>:<seq>`. */
230export function chunkId(docId: string, seq: number): string {
231 return `${docId}:${seq}`;
232}