Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

gitagentscloudflarerustforgemcp
177 linesCodeBlameRaw
1import assert from "node:assert/strict";
2import { readFileSync, readdirSync } from "node:fs";
3import { DatabaseSync } from "node:sqlite";
4import { test } from "node:test";
5
6import { EMBED_PER_HOUR, indexDoc, type IndexDoc } from "./indexer.ts";
7import type { Embedder, VectorMetadata, VectorStore } from "./vectors.ts";
8
9/** D1, as far as the indexer uses it, over node's SQLite with the service's migrations. */
10function fakeD1(): D1Database {
11 const db = new DatabaseSync(":memory:");
12 const dir = new URL("../migrations/", import.meta.url);
13 for (const file of readdirSync(dir).sort()) db.exec(readFileSync(new URL(file, dir), "utf8"));
14 const statement = (sql: string, params: unknown[] = []) => ({
15 sql,
16 params,
17 bind: (...values: unknown[]) => statement(sql, values),
18 first: async () => (db.prepare(sql).get(...(params as never[])) as unknown) ?? null,
19 all: async () => ({ results: db.prepare(sql).all(...(params as never[])) }),
20 run: async () => db.prepare(sql).run(...(params as never[])),
21 });
22 return {
23 prepare: (sql: string) => statement(sql),
24 batch: async (list: ReturnType<typeof statement>[]) => list.map((s) => db.prepare(s.sql).run(...(s.params as never[]))),
25 } as unknown as D1Database;
26}
27
28function fakes() {
29 const vectors = new Map<string, { values: number[]; metadata: VectorMetadata }>();
30 let embedded = 0;
31 const embedder: Embedder = {
32 async embed(texts) {
33 embedded += texts.length;
34 return texts.map((t) => [t.length, 1]);
35 },
36 };
37 const store: VectorStore = {
38 async upsert(list) {
39 for (const v of list) vectors.set(v.id, { values: v.values, metadata: v.metadata });
40 },
41 async get(ids) {
42 return ids.filter((id) => vectors.has(id)).map((id) => ({ id, values: vectors.get(id)!.values }));
43 },
44 async delete(ids) {
45 for (const id of ids) vectors.delete(id);
46 },
47 async query() {
48 return [];
49 },
50 };
51 return { vectors, embedder, store, embeddedCount: () => embedded };
52}
53
54const section = (name: string) => `## ${name}\n\n${Array.from({ length: 60 }, (_, i) => `${name.toLowerCase()}${i}`).join(" ")}.`;
55const page = (markdown: string, over: Partial<IndexDoc> = {}): IndexDoc => ({
56 kind: "page",
57 doc_id: "pag_1",
58 workspace_id: "w1",
59 space_id: "s1",
60 title: "Ops",
61 markdown,
62 ...over,
63});
64
65test("a page's passages are stored, embedded once, and only changed ones again", async () => {
66 const db = fakeD1();
67 const f = fakes();
68 const first = await indexDoc(db, f.embedder, f.store, page([section("Deploy"), section("Rollback")].join("\n\n")));
69 assert.deepEqual(first, { chunks: 2, embedded: 2, capped: false });
70 assert.deepEqual([...f.vectors.keys()].sort(), ["pag_1:0", "pag_1:1"]);
71 assert.deepEqual(f.vectors.get("pag_1:0")!.metadata, { workspace_id: "w1", space_id: "s1", kind: "page", page_id: "pag_1" });
72 // Saved again unchanged: nothing embedded.
73 assert.equal((await indexDoc(db, f.embedder, f.store, page([section("Deploy"), section("Rollback")].join("\n\n")))).embedded, 0);
74 // One section changed.
75 assert.equal((await indexDoc(db, f.embedder, f.store, page([section("Deploy"), section("Restore")].join("\n\n")))).embedded, 1);
76 assert.equal(f.embeddedCount(), 3);
77 // Word search finds passages by heading and text.
78 const hit = await db.prepare("SELECT chunk_id FROM doc_chunks_fts WHERE doc_chunks_fts MATCH ?").bind('"restore"').all<{ chunk_id: string }>();
79 assert.deepEqual(
80 hit.results.map((r) => r.chunk_id),
81 ["pag_1:1"],
82 );
83});
84
85test("a passage that only moved keeps its vector; gone passages leave the index", async () => {
86 const db = fakeD1();
87 const f = fakes();
88 await indexDoc(db, f.embedder, f.store, page([section("Alpha"), section("Beta"), section("Gamma")].join("\n\n")));
89 const before = f.embeddedCount();
90 // Alpha removed: Beta and Gamma move up a place, nothing new to embed.
91 const r = await indexDoc(db, f.embedder, f.store, page([section("Beta"), section("Gamma")].join("\n\n")));
92 assert.equal(r.embedded, 0);
93 assert.equal(f.embeddedCount(), before);
94 assert.deepEqual([...f.vectors.keys()].sort(), ["pag_1:0", "pag_1:1"]);
95 const rows = await db
96 .prepare("SELECT id, heading, hash = vector_hash AS current FROM doc_chunks ORDER BY seq")
97 .all<{ id: string; heading: string; current: number }>();
98 assert.deepEqual(
99 rows.results.map((x) => [x.id, x.heading, x.current]),
100 [
101 ["pag_1:0", "Beta", 1],
102 ["pag_1:1", "Gamma", 1],
103 ],
104 );
105});
106
107test("moving a page to another space files its vectors there without embedding", async () => {
108 const db = fakeD1();
109 const f = fakes();
110 await indexDoc(db, f.embedder, f.store, page(section("Deploy")));
111 const r = await indexDoc(db, f.embedder, f.store, page(section("Deploy"), { space_id: "s2" }));
112 assert.equal(r.embedded, 0);
113 assert.equal(f.vectors.get("pag_1:0")!.metadata.space_id, "s2");
114 assert.equal((await db.prepare("SELECT space_id FROM doc_chunks WHERE id = 'pag_1:0'").first<{ space_id: string }>())!.space_id, "s2");
115});
116
117test("embedding stops at the hourly cap and the rest waits, kept for words", async () => {
118 const db = fakeD1();
119 const f = fakes();
120 const at = new Date("2026-10-09T10:15:00Z");
121 await db
122 .prepare("INSERT INTO doc_embed_usage (workspace_id, hour, chunks, tokens) VALUES ('w1', '2026-10-09T10', ?, 0)")
123 .bind(EMBED_PER_HOUR - 1)
124 .run();
125 const r = await indexDoc(db, f.embedder, f.store, page([section("One"), section("Two"), section("Three")].join("\n\n")), at);
126 assert.deepEqual(r, { chunks: 3, embedded: 1, capped: true });
127 const waiting = await db.prepare("SELECT COUNT(*) AS n FROM doc_chunks WHERE vector_hash IS NULL OR vector_hash <> hash").first<{ n: number }>();
128 assert.equal(waiting!.n, 2);
129 // The next hour, a save picks up the rest.
130 const later = await indexDoc(
131 db,
132 f.embedder,
133 f.store,
134 page([section("One"), section("Two"), section("Three")].join("\n\n")),
135 new Date("2026-10-09T11:01:00Z"),
136 );
137 assert.deepEqual(later, { chunks: 3, embedded: 2, capped: false });
138});
139
140test("an embedding failure leaves passages for the next save, never throws", async () => {
141 const db = fakeD1();
142 const f = fakes();
143 const broken: Embedder = {
144 async embed() {
145 throw new Error("model down");
146 },
147 };
148 const r = await indexDoc(db, broken, f.store, page(section("Deploy")));
149 assert.equal(r.chunks, 1);
150 assert.equal(f.vectors.size, 0);
151 assert.equal((await indexDoc(db, f.embedder, f.store, page(section("Deploy")))).embedded, 1);
152});
153
154test("without an embedder, passages are kept for words only", async () => {
155 const db = fakeD1();
156 const r = await indexDoc(db, null, null, page(section("Deploy")));
157 assert.deepEqual(r, { chunks: 1, embedded: 0, capped: false });
158 assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM doc_chunks_fts").first<{ n: number }>())!.n, 1);
159});
160
161test("a project's docs file is indexed with its repository", async () => {
162 const db = fakeD1();
163 const f = fakes();
164 await indexDoc(db, f.embedder, f.store, {
165 kind: "repo_file",
166 doc_id: "rf_x",
167 workspace_id: "w1",
168 space_id: "rds_1",
169 title: "Setup",
170 markdown: `# Setup\n\n${section("Install")}`,
171 repo_id: "r1",
172 path: "docs/setup.md",
173 });
174 assert.deepEqual(f.vectors.get("rf_x:0")!.metadata, { workspace_id: "w1", space_id: "rds_1", kind: "repo_file", repo_file_id: "rf_x", repo_id: "r1" });
175 const row = await db.prepare("SELECT page_id, repo_file_id, path, heading FROM doc_chunks").first<Record<string, unknown>>();
176 assert.deepEqual({ ...row }, { page_id: null, repo_file_id: "rf_x", path: "docs/setup.md", heading: "Install" });
177});