Skip to content
Sign up

flagon-io/g1t

public
0
255 linesCodeBlameRaw
1/**
2 * Which docs, and which of their lines, are worth suggesting as memory.
3 *
4 * Memory is for how to work in a project: how to build, test and run it,
5 * its conventions and its traps. A project's docs also hold plans,
6 * roadmaps, feedback to vendors and release notes, which say what someone
7 * wants, not how things are; none of that is suggested. A line is taken
8 * whole (a bullet with its continuation lines), cut only between sentences
9 * to fit, and left out when even its first sentence does not fit. Pure.
10 */
11
12/** README, AGENTS.md (or CLAUDE.md), CONTRIBUTING, or another doc. */
13export type DocRole = "readme" | "agents" | "contributing" | "doc";
14
15/** What a project's `.g1t/project.yml` says about memory from its docs. */
16export type MemoryDocsConfig = {
17 /** Docs to read for memory besides the ones g1t picks, by path or `dir/*`. */
18 docs: string[];
19 /** Docs never read for memory, by path or `dir/*`. Wins over `docs`. */
20 skip: string[];
21};
22
23/** The longest memory suggested from a doc, in characters. */
24export const MAX_DOC_HINT_CHARS = 300;
25
26/** Doc names that say how to work on a project. */
27const WORKING_NAME =
28 /(build|test|setup|set-up|develop|contribut|hacking|architecture|convention|style|guideline|standard|deploy|install|getting[-_ ]?started|local|onboard|troubleshoot|debug|runbook|self[-_ ]?host|coding|structure|layout|workflow|agents|claude)/i;
29/** Doc names that hold plans, feedback, history or news. */
30const PLAN_NAME =
31 /(plan|roadmap|feedback|changelog|changes|history|news|release[-_ ]?notes|rfc|proposal|idea|todo|backlog|vision|strategy|research|demo|incident|postmortem|post-mortem|retro|meeting|minutes|notes|draft|wish|faq|announce|blog|pitch)/i;
32
33/** Headings of sections that hold plans, asks or history. */
34const PLAN_HEADING =
35 /\b(roadmap|plans?|planned|planning|milestones?|phase \d+|future|later|next steps|what'?s next|what we'?d love|wish ?list|asks?|feedback|changelog|release notes|what'?s new|open questions|non-goals|goals|ideas|proposals?|backlog|todo|executive summary|what we (hit|tried|built|want))\b/i;
36/** Headings of sections that say how things are done here. */
37const CONVENTION_HEADING =
38 /\b(conventions?|guidelines?|code style|coding style|style guide|standards|gotchas?|pitfalls?|caveats?|known issues|troubleshooting|before you (push|commit|open)|house rules|how we work|working (here|on|in)|dos and don'?ts|rules for (contributors|agents|code))\b/i;
39/** Headings of sections that say how to set up, build or test. */
40export const SETUP_HEADING =
41 /\b(develop|development|getting started|setup|set up|install|installing|build|building|test|testing|contribut|running|run locally|local|before you push|deploying)\b/i;
42
43/** A bullet's bold label that marks it as a plan, an ask or a report. */
44const PLAN_LABEL =
45 /^(ask|asks|what we (tried|hit|built|want|need|'d love)|what it means|proposal|idea|plan|next|later|future|todo|status|why it matters)\b/i;
46/** Words that make a line a plan or a wish rather than how things are. */
47const ASPIRATION =
48 /\b(will|won'?t|would|should|could|might|shall|plans? to|planned|we'?d|we'?ll|eventually|someday|one day|in the future|not yet|coming soon|todo|tbd|wip|roadmap)\b|^ask\b/i;
49/** The modal words an AGENTS.md uses to state a rule; aspirational elsewhere. */
50const RULE_MODAL = /\b(will|won'?t|would|should|could|might|shall)\b/gi;
51
52/** Words that mark a line as a trap. */
53const GOTCHA =
54 /^(never|don'?t|do not|avoid|beware|careful|warning|watch out)\b|\b(must not|must never|gotcha|beware|careful|fails? (unless|if|when|until|without)|breaks? (if|when|unless)|otherwise)\b/i;
55/** Words that mark a line as a rule of how things are done. */
56const CONVENTION =
57 /^(use|run|prefer|keep|put|name|write|import|add|look|always|never|don'?t|do not|avoid|make sure|remember|check|update|commit|open|follow|place|call|read|test|document)\b|\b(we use|must|prefer|instead of|rather than|always|never|convention|is required|are required)\b/i;
58
59/** Whether `path` matches one of `patterns`: an exact path, or `dir/*`. */
60function matches(path: string, patterns: string[]): boolean {
61 const lower = path.toLowerCase();
62 return patterns.some((raw) => {
63 const pattern = raw.trim().replace(/^\.?\//, "").toLowerCase();
64 if (!pattern) return false;
65 if (pattern.endsWith("/*")) return lower.startsWith(pattern.slice(0, -1)) && !lower.slice(pattern.length - 1).includes("/");
66 if (pattern.endsWith("/")) return lower.startsWith(pattern);
67 return lower === pattern;
68 });
69}
70
71/**
72 * Whether memory is suggested from the doc at `path`: a README, AGENTS.md,
73 * CLAUDE.md or CONTRIBUTING always, a runbook, and another doc when its
74 * name says how to work on the project (`docs/DEPLOYING.md`, not
75 * `docs/PLAN.md` or `docs/CHANGELOG.md`). `.g1t/project.yml` can add docs
76 * or leave them out.
77 */
78export function harvestable(path: string, role: DocRole, config?: MemoryDocsConfig | null): boolean {
79 if (config && matches(path, config.skip)) return false;
80 if (config && matches(path, config.docs)) return true;
81 if (role !== "doc") return true;
82 const name = path.slice(path.lastIndexOf("/") + 1).replace(/\.(md|markdown|txt)$/i, "");
83 if (PLAN_NAME.test(name)) return false;
84 if (/(^|\/)runbooks\//i.test(path)) return true;
85 return WORKING_NAME.test(name);
86}
87
88/**
89 * Whether a doc reads as a plan, a report or feedback rather than how to
90 * work: many of its headings are plans or asks, or many of its bullets are
91 * labelled as asks and findings.
92 */
93export function planLike(text: string): boolean {
94 let headings = 0;
95 let planHeadings = 0;
96 let labels = 0;
97 let aspirations = 0;
98 let lines = 0;
99 for (const line of text.split(/\r?\n/)) {
100 const heading = /^#{1,6}\s+(.*)$/.exec(line)?.[1];
101 if (heading) {
102 headings++;
103 if (PLAN_HEADING.test(heading)) planHeadings++;
104 continue;
105 }
106 const label = /^\s*[-*+]\s+\*\*([^*]+)\*\*/.exec(line)?.[1];
107 if (label && PLAN_LABEL.test(label.trim())) labels++;
108 if (line.trim()) {
109 lines++;
110 if (/\b(we will|we'?ll|we plan|we'?d love|will be|is planned|are planned|not yet built|coming soon)\b/i.test(line)) aspirations++;
111 }
112 }
113 return labels >= 3 || (headings >= 3 && planHeadings * 3 >= headings) || (lines >= 20 && aspirations * 8 >= lines);
114}
115
116/** Markdown inline marks taken out: bold, italics, links to their text, images. */
117export function plain(text: string): string {
118 return text
119 .replace(/!\[[^\]]*\]\([^)]*\)/g, "")
120 .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
121 .replace(/\*\*([^*]+)\*\*/g, "$1")
122 .replace(/__([^_]+)__/g, "$1")
123 .replace(/(^|\s)\*([^*\s][^*]*)\*(?=\s|[.,;:!?)]|$)/g, "$1$2")
124 .replace(/\s+/g, " ")
125 .trim();
126}
127
128const ABBREVIATION = /(\b(e\.g|i\.e|etc|vs|cf|approx|no|fig)\.|\b[A-Z]\.)$/i;
129
130/** A line's sentences, never split inside `code` or after an abbreviation. */
131export function sentencesOf(text: string): string[] {
132 const out: string[] = [];
133 let current = "";
134 let code = false;
135 for (let i = 0; i < text.length; i++) {
136 const c = text[i];
137 current += c;
138 if (c === "`") code = !code;
139 if (code || !/[.!?]/.test(c)) continue;
140 const next = text[i + 1];
141 // A full stop inside a word (README.md, v1.2) or before more punctuation.
142 if (next !== undefined && !/\s/.test(next)) continue;
143 if (ABBREVIATION.test(current.trimEnd())) continue;
144 out.push(current.trim());
145 current = "";
146 }
147 if (current.trim()) out.push(current.trim());
148 return out;
149}
150
151/**
152 * `text` whole if it fits in `max` characters; otherwise as many of its
153 * first sentences as fit; null when not even the first one does. Never cut
154 * inside a sentence.
155 */
156export function wholeSentences(text: string, max = MAX_DOC_HINT_CHARS): string | null {
157 const line = text.replace(/\s+/g, " ").trim();
158 if (line.length <= max) return line;
159 let out = "";
160 for (const sentence of sentencesOf(line)) {
161 const next = out ? `${out} ${sentence}` : sentence;
162 if (next.length > max) break;
163 out = next;
164 }
165 return out && /[.!?]$/.test(out) ? out : null;
166}
167
168/** Whether a line says what someone wants or plans rather than how things are. */
169export function aspirational(text: string, role: DocRole): boolean {
170 const label = /^\*\*([^*]+)\*\*/.exec(text.trim())?.[1]?.trim();
171 if (label && PLAN_LABEL.test(label)) return true;
172 const words = plain(text);
173 if (/^(ask|what we (tried|hit|built))\b/i.test(words)) return true;
174 // AGENTS.md states its rules with "should" and "will": those are rules.
175 const checked = role === "agents" ? words.replace(RULE_MODAL, "") : words;
176 return ASPIRATION.test(checked);
177}
178
179/** The kind of memory a line from a doc is, and how sure g1t is of it. */
180export function classify(text: string, heading: string, role: DocRole): { kind: "fact" | "convention" | "gotcha"; confidence: number } {
181 const agents = role === "agents";
182 if (GOTCHA.test(text)) return { kind: "gotcha", confidence: agents ? 0.9 : 0.6 };
183 if (CONVENTION.test(text) || (agents && CONVENTION_HEADING.test(heading))) return { kind: "convention", confidence: agents ? 0.9 : 0.6 };
184 return { kind: "fact", confidence: agents ? 0.85 : 0.5 };
185}
186
187/** Headings of sections that report or explain rather than say how. */
188const REPORT_HEADING = /\b(speed|performance|benchmarks?|timings?|costs?|numbers|results|findings|lessons|why|background|motivation|history|incidents?|features|status)\b/i;
189
190/** Whether a section's bullets are worth reading for memory. */
191export function bulletSection(heading: string, role: DocRole): boolean {
192 if (PLAN_HEADING.test(heading) || (role !== "agents" && REPORT_HEADING.test(heading))) return false;
193 return role === "agents" || CONVENTION_HEADING.test(heading) || SETUP_HEADING.test(heading);
194}
195
196/** Whether a section is a plan, an ask or history, with everything under it. */
197export function planSection(heading: string): boolean {
198 return PLAN_HEADING.test(heading);
199}
200
201export type Bullet = { text: string; heading: string };
202
203/**
204 * A doc's bullets, each whole with its continuation lines, with the heading
205 * it is under. Bullets in code blocks and in plan sections (and their
206 * subsections) are left out.
207 */
208export function bulletsOf(markdown: string): Bullet[] {
209 const out: Bullet[] = [];
210 let heading = "";
211 // The level of the plan section being skipped, if any.
212 let skipping = 0;
213 let fenced = false;
214 let current: { text: string; indent: number } | null = null;
215 const flush = () => {
216 if (current && !skipping) out.push({ text: current.text.replace(/\s+/g, " ").trim(), heading });
217 current = null;
218 };
219 for (const raw of markdown.split(/\r?\n/)) {
220 if (/^\s*(```|~~~)/.test(raw)) {
221 flush();
222 fenced = !fenced;
223 continue;
224 }
225 if (fenced) continue;
226 const h = /^(#{1,6})\s+(.*)$/.exec(raw);
227 if (h) {
228 flush();
229 const level = h[1].length;
230 if (skipping && level <= skipping) skipping = 0;
231 heading = h[2].trim();
232 if (!skipping && planSection(heading)) skipping = level;
233 continue;
234 }
235 const bullet = /^(\s*)(?:[-*+]|\d+[.)])\s+(.*)$/.exec(raw);
236 if (bullet) {
237 flush();
238 current = { text: bullet[2], indent: bullet[1].length };
239 continue;
240 }
241 if (current && raw.trim() && /^\s+/.test(raw) && raw.search(/\S/) > current.indent) {
242 current.text += ` ${raw.trim()}`;
243 continue;
244 }
245 // A blank line or a paragraph ends the bullet. A bullet's lazy
246 // continuation (not indented) is still part of it.
247 if (current && raw.trim() && !/^\s*[|<>]/.test(raw)) {
248 current.text += ` ${raw.trim()}`;
249 continue;
250 }
251 flush();
252 }
253 flush();
254 return out;
255}