| 1 | /** |
| 2 | * A project's docs in Docs: a repository's `docs/` folder and its |
| 3 | * README.md, read from the default branch into D1 so the sidebar lists |
| 4 | * them and search finds them beside the workspace's pages. Read when the |
| 5 | * folder is added, and again on every push to the default branch |
| 6 | * (src/staleness.ts `onEvent`); only files whose blob changed are read |
| 7 | * again. Read-only here: changes go through the repository. |
| 8 | * |
| 9 | * Who sees one is decided when it is read: the viewer must be able to read |
| 10 | * the repository (`ReposApi.readable`), whoever added it. |
| 11 | */ |
| 12 | import { reposClient, type ServiceBinding } from "@g1t/contracts"; |
| 13 | |
| 14 | import { searchText } from "./markdown.ts"; |
| 15 | import { pickRepoDocs, repoDocTitle } from "./repo-docs.ts"; |
| 16 | |
| 17 | export { isRepoDoc, pickRepoDocs, repoDocTitle } from "./repo-docs.ts"; |
| 18 | |
| 19 | /** A file larger than this is listed but not read. */ |
| 20 | const MAX_FILE_BYTES = 512 * 1024; |
| 21 | |
| 22 | export type RepoSpaceRow = { |
| 23 | id: string; |
| 24 | workspace_id: string; |
| 25 | repo_id: string; |
| 26 | repo: string; |
| 27 | default_branch: string; |
| 28 | commit_sha: string | null; |
| 29 | indexed_at: string | null; |
| 30 | added_by: string; |
| 31 | added_at: string; |
| 32 | }; |
| 33 | |
| 34 | function decodeBase64(data: string): string { |
| 35 | const binary = atob(data); |
| 36 | const bytes = new Uint8Array(binary.length); |
| 37 | for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); |
| 38 | return new TextDecoder().decode(bytes); |
| 39 | } |
| 40 | |
| 41 | /** |
| 42 | * Reads a repository's docs again into one space: lists the default |
| 43 | * branch, reads the files whose blob changed, drops the ones gone. Asks |
| 44 | * repos without a viewer (`listFiles`, `rawBlobs`), so callers check the |
| 45 | * repository can be read first, as adding one does. |
| 46 | */ |
| 47 | export async function indexRepoSpace( |
| 48 | env: { DB: D1Database; REPOS: ServiceBinding }, |
| 49 | space: RepoSpaceRow, |
| 50 | now = new Date(), |
| 51 | ): Promise<{ files: number; read: number; changed: string[]; gone: string[] }> { |
| 52 | const db = env.DB; |
| 53 | const repos = reposClient(env.REPOS); |
| 54 | const listing = await repos.listFiles(space.repo_id, null, 10_000); |
| 55 | const wanted = pickRepoDocs(listing.files.filter((f): f is { path: string; hash: string } => !!f.hash)); |
| 56 | const kept = new Map( |
| 57 | (await db.prepare("SELECT path, hash FROM repo_files WHERE space_id = ?").bind(space.id).all<{ path: string; hash: string }>()).results.map((r) => [r.path, r.hash]), |
| 58 | ); |
| 59 | const changed = wanted.filter((f) => kept.get(f.path) !== f.hash); |
| 60 | const blobs = new Map<string, string | null>(); |
| 61 | for (let i = 0; i < changed.length; i += 100) { |
| 62 | const batch = changed.slice(i, i + 100); |
| 63 | for (const blob of await repos.rawBlobs(space.repo_id, [...new Set(batch.map((f) => f.hash))], MAX_FILE_BYTES)) blobs.set(blob.hash, blob.data); |
| 64 | } |
| 65 | const statements: D1PreparedStatement[] = []; |
| 66 | const gone = [...kept.keys()].filter((path) => !wanted.some((f) => f.path === path)); |
| 67 | for (const path of gone) { |
| 68 | statements.push(db.prepare("DELETE FROM repo_files WHERE space_id = ? AND path = ?").bind(space.id, path)); |
| 69 | statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, path)); |
| 70 | } |
| 71 | for (const file of changed) { |
| 72 | const data = blobs.get(file.hash); |
| 73 | const markdown = data ? decodeBase64(data) : `_This file is too large to show here._ Open it in Code.\n`; |
| 74 | const title = repoDocTitle(file.path, markdown); |
| 75 | statements.push( |
| 76 | db |
| 77 | .prepare("INSERT INTO repo_files (space_id, path, hash, title, markdown) VALUES (?, ?, ?, ?, ?) ON CONFLICT (space_id, path) DO UPDATE SET hash = excluded.hash, title = excluded.title, markdown = excluded.markdown") |
| 78 | .bind(space.id, file.path, file.hash, title, markdown), |
| 79 | ); |
| 80 | statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, file.path)); |
| 81 | statements.push(db.prepare("INSERT INTO repo_files_fts (space_id, path, title, body) VALUES (?, ?, ?, ?)").bind(space.id, file.path, title, searchText(markdown))); |
| 82 | } |
| 83 | statements.push(db.prepare("UPDATE repo_spaces SET commit_sha = ?, indexed_at = ? WHERE id = ?").bind(listing.commit, now.toISOString(), space.id)); |
| 84 | for (let i = 0; i < statements.length; i += 50) await db.batch(statements.slice(i, i + 50)); |
| 85 | return { files: wanted.length, read: changed.length, changed: changed.map((f) => f.path), gone }; |
| 86 | } |
| 87 | |
| 88 | /** |
| 89 | * Every space showing a repository's docs, read again after a push; then |
| 90 | * `indexed` with what changed in each (the semantic index, src/indexer.ts). |
| 91 | * Never throws for one space's sake. |
| 92 | */ |
| 93 | export async function reindexRepo( |
| 94 | env: { DB: D1Database; REPOS: ServiceBinding }, |
| 95 | repoId: string, |
| 96 | indexed: (spaceId: string, changed: string[], gone: string[]) => Promise<void> = async () => {}, |
| 97 | ): Promise<void> { |
| 98 | const spaces = (await env.DB.prepare("SELECT * FROM repo_spaces WHERE repo_id = ?").bind(repoId).all<RepoSpaceRow>()).results; |
| 99 | for (const space of spaces) { |
| 100 | try { |
| 101 | const read = await indexRepoSpace(env, space); |
| 102 | await indexed(space.id, read.changed, read.gone); |
| 103 | } catch (error) { |
| 104 | console.error("docs could not read a project's docs", space.repo, String(error)); |
| 105 | } |
| 106 | } |
| 107 | } |