Skip to content
Sign up

flagon-io/g1t

public
0
107 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

The artifacts service is services/artifacts, the Worker g1t-artifacts, bound as ARTIFACTS by the API, the site and the agents; its live rooms move to it with a Durable Object transfer from g1t-docs-service, and its database, bucket, indexes and queue keep their names. The git store's binding and settings are GITSTORE, its ops scripts gitstore-*, and workflow run artifacts keep their compatible API under run_artifacts modules. The deploy tool puts a Worker that has never deployed before the Workers in its stage that bind to it, and the deploy guide gives the cutover runbook.1/**
2 * A project's docs in Docs: a repository's `docs/` folder and its
3 * README.md, read from the default branch into D1 so the sidebar lists
4 * them and search finds them beside the workspace's pages. Read when the
5 * folder is added, and again on every push to the default branch
6 * (src/staleness.ts `onEvent`); only files whose blob changed are read
7 * again. Read-only here: changes go through the repository.
8 *
9 * Who sees one is decided when it is read: the viewer must be able to read
10 * the repository (`ReposApi.readable`), whoever added it.
11 */
12import { reposClient, type ServiceBinding } from "@g1t/contracts";
13
14import { searchText } from "./markdown.ts";
15import { pickRepoDocs, repoDocTitle } from "./repo-docs.ts";
16
17export { isRepoDoc, pickRepoDocs, repoDocTitle } from "./repo-docs.ts";
18
19/** A file larger than this is listed but not read. */
20const MAX_FILE_BYTES = 512 * 1024;
21
22export type RepoSpaceRow = {
23 id: string;
24 workspace_id: string;
25 repo_id: string;
26 repo: string;
27 default_branch: string;
28 commit_sha: string | null;
29 indexed_at: string | null;
30 added_by: string;
31 added_at: string;
32};
33
34function decodeBase64(data: string): string {
35 const binary = atob(data);
36 const bytes = new Uint8Array(binary.length);
37 for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
38 return new TextDecoder().decode(bytes);
39}
40
41/**
42 * Reads a repository's docs again into one space: lists the default
43 * branch, reads the files whose blob changed, drops the ones gone. Asks
44 * repos without a viewer (`listFiles`, `rawBlobs`), so callers check the
45 * repository can be read first, as adding one does.
46 */
47export async function indexRepoSpace(
48 env: { DB: D1Database; REPOS: ServiceBinding },
49 space: RepoSpaceRow,
50 now = new Date(),
51): Promise<{ files: number; read: number; changed: string[]; gone: string[] }> {
52 const db = env.DB;
53 const repos = reposClient(env.REPOS);
54 const listing = await repos.listFiles(space.repo_id, null, 10_000);
55 const wanted = pickRepoDocs(listing.files.filter((f): f is { path: string; hash: string } => !!f.hash));
56 const kept = new Map(
57 (await db.prepare("SELECT path, hash FROM repo_files WHERE space_id = ?").bind(space.id).all<{ path: string; hash: string }>()).results.map((r) => [r.path, r.hash]),
58 );
59 const changed = wanted.filter((f) => kept.get(f.path) !== f.hash);
60 const blobs = new Map<string, string | null>();
61 for (let i = 0; i < changed.length; i += 100) {
62 const batch = changed.slice(i, i + 100);
63 for (const blob of await repos.rawBlobs(space.repo_id, [...new Set(batch.map((f) => f.hash))], MAX_FILE_BYTES)) blobs.set(blob.hash, blob.data);
64 }
65 const statements: D1PreparedStatement[] = [];
66 const gone = [...kept.keys()].filter((path) => !wanted.some((f) => f.path === path));
67 for (const path of gone) {
68 statements.push(db.prepare("DELETE FROM repo_files WHERE space_id = ? AND path = ?").bind(space.id, path));
69 statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, path));
70 }
71 for (const file of changed) {
72 const data = blobs.get(file.hash);
73 const markdown = data ? decodeBase64(data) : `_This file is too large to show here._ Open it in Code.\n`;
74 const title = repoDocTitle(file.path, markdown);
75 statements.push(
76 db
77 .prepare("INSERT INTO repo_files (space_id, path, hash, title, markdown) VALUES (?, ?, ?, ?, ?) ON CONFLICT (space_id, path) DO UPDATE SET hash = excluded.hash, title = excluded.title, markdown = excluded.markdown")
78 .bind(space.id, file.path, file.hash, title, markdown),
79 );
80 statements.push(db.prepare("DELETE FROM repo_files_fts WHERE space_id = ? AND path = ?").bind(space.id, file.path));
81 statements.push(db.prepare("INSERT INTO repo_files_fts (space_id, path, title, body) VALUES (?, ?, ?, ?)").bind(space.id, file.path, title, searchText(markdown)));
82 }
83 statements.push(db.prepare("UPDATE repo_spaces SET commit_sha = ?, indexed_at = ? WHERE id = ?").bind(listing.commit, now.toISOString(), space.id));
84 for (let i = 0; i < statements.length; i += 50) await db.batch(statements.slice(i, i + 50));
85 return { files: wanted.length, read: changed.length, changed: changed.map((f) => f.path), gone };
86}
87
88/**
89 * Every space showing a repository's docs, read again after a push; then
90 * `indexed` with what changed in each (the semantic index, src/indexer.ts).
91 * Never throws for one space's sake.
92 */
93export async function reindexRepo(
94 env: { DB: D1Database; REPOS: ServiceBinding },
95 repoId: string,
96 indexed: (spaceId: string, changed: string[], gone: string[]) => Promise<void> = async () => {},
97): Promise<void> {
98 const spaces = (await env.DB.prepare("SELECT * FROM repo_spaces WHERE repo_id = ?").bind(repoId).all<RepoSpaceRow>()).results;
99 for (const space of spaces) {
100 try {
101 const read = await indexRepoSpace(env, space);
102 await indexed(space.id, read.changed, read.gone);
103 } catch (error) {
104 console.error("docs could not read a project's docs", space.repo, String(error));
105 }
106 }
107}