Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Docs know what code they describe; a project's docs folder in Docs; Docs events; files on any S3 store | 1 | /** |
| 2 | * A project's docs in Docs: a repository's `docs/` folder and its | |
| 3 | * README.md, read from the default branch into D1 so the sidebar lists | |
| 4 | * them and search finds them beside the workspace's pages. Read when the | |
| 5 | * folder is added, and again on every push to the default branch | |
| 6 | * (src/staleness.ts `onEvent`); only files whose blob changed are read | |
| 7 | * again. Read-only here: changes go through the repository. | |
| 8 | * | |
| 9 | * Who sees one is decided when it is read: the viewer must be able to read | |
| 10 | * the repository (`ReposApi.readable`), whoever added it. | |
| 11 | */ | |
| 12 | import { reposClient, type ServiceBinding } from "@g1t/contracts"; | |
| 13 | ||
| 14 | import { searchText } from "./markdown.ts"; | |
| 15 | import { pickRepoDocs, repoDocTitle } from "./repo-docs.ts"; | |
| 16 | ||
| 17 | export { isRepoDoc, pickRepoDocs, repoDocTitle } from "./repo-docs.ts"; | |
| 18 | ||
| 19 | /** A file larger than this is listed but not read. */ | |
| 20 | const MAX_FILE_BYTES = 512 * 1024; | |
| 21 | ||
| 22 | export type RepoSpaceRow = { | |
| 23 | id: string; | |
| 24 | workspace_id: string; | |
| 25 | repo_id: string; | |
| 26 | repo: string; | |
| 27 | default_branch: string; | |
| 28 | commit_sha: string | null; | |
| 29 | indexed_at: string | null; | |
| 30 | added_by: string; | |
| 31 | added_at: string; | |
| 32 | }; | |
| 33 | ||
| 34 | function decodeBase64(data: string): string { | |
| 35 | const binary = atob(data); | |
| 36 | const bytes = new Uint8Array(binary.length); | |
| 37 | for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); | |
| 38 | return new TextDecoder().decode(bytes); | |
| 39 | } | |
| 40 | ||
| 41 | /** | |
| 42 | * Reads a repository's docs again into one space: lists the default | |
| 43 | * branch, reads the files whose blob changed, drops the ones gone. Asks | |
| 44 | * repos without a viewer (`listFiles`, `rawBlobs`), so callers check the | |
| 45 | * repository can be read first, as adding one does. | |
| 46 | */ | |
| 47 | export async function indexRepoSpace(env: { DB: D1Database; REPOS: ServiceBinding }, space: RepoSpaceRow, now = new Date()): Promise<{ files: number; read: number }> { | |
| 48 | const db = env.DB; | |
| 49 | const repos = reposClient(env.REPOS); | |
| 50 | const listing = await repos.listFiles(space.repo_id, null, 10_000); | |
| 51 | const wanted = pickRepoDocs(listing.files.filter((f): f is { path: string; hash: string } => !!f.hash)); | |
| 52 | const kept = new Map( | |
| 53 | (await db.prepare("SELECT path, hash FROM repo_files WHERE space_id = ?").bind(space.id).all<{ path: string; hash: string }>()).results.map((r) => [r.path, r.hash]), | |
| 54 | ); | |
| 55 | const changed = wanted.filter((f) => kept.get(f.path) !== f.hash); | |
| 56 | const blobs = new Map<string, string | null>(); | |
| 57 | for (let i = 0; i < changed.length; i += 100) { | |
| 58 | const batch = changed.slice(i, i + 100); | |
| 59 | for (const blob of await repos.rawBlobs(space.repo_id, [...new Set(batch.map((f) => f.hash))], MAX_FILE_BYTES)) blobs.set(blob.hash, blob.data); | |
| 60 | } | |
| 61 | const statements: D1PreparedStatement[] = []; | |
| 62 | const gone = [...kept.keys()].filter((path) => !wanted.some((f) => f.path === path)); | |
| 63 | for (const path of gone) { | |
| 64 | statements.push(db.prepare("DELETE FROM repo_files WHERE space_id = ? AND path = ?").bind(space.id, path)); | |
| 65 | statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, path)); | |
| 66 | } | |
| 67 | for (const file of changed) { | |
| 68 | const data = blobs.get(file.hash); | |
| 69 | const markdown = data ? decodeBase64(data) : `_This file is too large to show here._ Open it in Code.\n`; | |
| 70 | const title = repoDocTitle(file.path, markdown); | |
| 71 | statements.push( | |
| 72 | db | |
| 73 | .prepare("INSERT INTO repo_files (space_id, path, hash, title, markdown) VALUES (?, ?, ?, ?, ?) ON CONFLICT (space_id, path) DO UPDATE SET hash = excluded.hash, title = excluded.title, markdown = excluded.markdown") | |
| 74 | .bind(space.id, file.path, file.hash, title, markdown), | |
| 75 | ); | |
| 76 | statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, file.path)); | |
| 77 | statements.push(db.prepare("INSERT INTO repo_files_fts (space_id, path, title, body) VALUES (?, ?, ?, ?)").bind(space.id, file.path, title, searchText(markdown))); | |
| 78 | } | |
| 79 | statements.push(db.prepare("UPDATE repo_spaces SET commit_sha = ?, indexed_at = ? WHERE id = ?").bind(listing.commit, now.toISOString(), space.id)); | |
| 80 | for (let i = 0; i < statements.length; i += 50) await db.batch(statements.slice(i, i + 50)); | |
| 81 | return { files: wanted.length, read: changed.length }; | |
| 82 | } | |
| 83 | ||
| 84 | /** Every space showing a repository's docs, read again after a push. Never throws for one space's sake. */ | |
| 85 | export async function reindexRepo(env: { DB: D1Database; REPOS: ServiceBinding }, repoId: string): Promise<void> { | |
| 86 | const spaces = (await env.DB.prepare("SELECT * FROM repo_spaces WHERE repo_id = ?").bind(repoId).all<RepoSpaceRow>()).results; | |
| 87 | for (const space of spaces) { | |
| 88 | try { | |
| 89 | await indexRepoSpace(env, space); | |
| 90 | } catch (error) { | |
| 91 | console.error("docs could not read a project's docs", space.repo, String(error)); | |
| 92 | } | |
| 93 | } | |
| 94 | } |
