Sign in

pulls/pr_01m47d24b0e6n91zwymwxg0vpx

public

pr_01m47d24b0e6n91zwymwxg0vpx/services/projects/src/index.ts

541 lines22,585 bytesCodeBlame
1/**
2 * The projects service: what a workspace builds and runs.
3 *
4 * A project has one source, where its code lives: today a repository hosted
5 * on g1t and a root directory in it. Everything about running it
6 * (deployments, environments, domains, secrets and variables) hangs off the
7 * project; other services key their data by its id. Every repository gets
8 * a project of its own name: when it is created (from `repo.created`), and
9 * for repositories made before projects existed, the first time their
10 * workspace's projects are asked for.
11 *
12 * Reached through service bindings: `POST /rpc/<method>`.
13 */
14
15import { parse as parseYaml } from "yaml";
16
17import {
18 fail,
19 identityClient,
20 newId,
21 ok,
22 reposClient,
23 type Dependencies,
24 type DependencyLink,
25 type G1tEvent,
26 type NewProject,
27 type Project,
28 type ProjectGraph,
29 type Repo,
30 type Result,
31 type ServiceBinding,
32 type User,
33 type Viewer,
34} from "@g1t/contracts";
35
36type Env = {
37 DB: D1Database;
38 REPOS: ServiceBinding;
39 IDENTITY: ServiceBinding;
40};
41
42type Row = {
43 id: string;
44 workspace: string;
45 slug: string;
46 name: string;
47 description: string | null;
48 source_kind: string;
49 repo_id: string;
50 repo_namespace: string;
51 repo_name: string;
52 repo_private: number;
53 default_branch: string;
54 root_dir: string;
55 is_primary: number;
56 created_by: string;
57 created_at: string;
58 updated_at: string;
59};
60
61const now = () => new Date().toISOString();
62
63/** A variable's name for a dependency's address, spelled as secrets' names are. */
64const ALIAS = /^[A-Z_][A-Z0-9_]{0,99}$/;
65/** How many dependencies a project may declare. */
66const MAX_DEPENDENCIES = 50;
67
68type LinkRow = { slug: string; name: string; alias: string | null; source: "ui" | "file" };
69type NodeRow = { id: string; slug: string; workspace: string; alias: string | null };
70
71function toProject(row: Row): Project {
72 return {
73 id: row.id,
74 workspace: row.workspace,
75 slug: row.slug,
76 name: row.name,
77 description: row.description,
78 source: {
79 kind: "hosted",
80 repoId: row.repo_id,
81 repo: { namespace: row.repo_namespace, name: row.repo_name },
82 rootDir: row.root_dir,
83 defaultBranch: row.default_branch,
84 },
85 private: !!row.repo_private,
86 primary: !!row.is_primary,
87 createdBy: row.created_by,
88 createdAt: row.created_at,
89 updatedAt: row.updated_at,
90 };
91}
92
93/** A name as an address: lowercase letters, digits, `-`, `_` and `.`. */
94function slugOf(name: string): string {
95 return name
96 .trim()
97 .toLowerCase()
98 .replace(/[^a-z0-9._-]+/g, "-")
99 .replace(/^[-.]+|[-.]+$/g, "")
100 .slice(0, 100);
101}
102
103function isMember(viewer: Viewer, workspace: string): boolean {
104 return !!viewer?.workspaces?.some((m) => m.slug === workspace.toLowerCase());
105}
106
107class Projects {
108 constructor(private readonly env: Env) {}
109
110 private get db() {
111 return this.env.DB;
112 }
113
114 private async workspaceActor(slug: string): Promise<User | null> {
115 const workspace = await identityClient(this.env.IDENTITY).getWorkspace(slug);
116 if (!workspace) return null;
117 return {
118 id: workspace.id,
119 username: workspace.slug,
120 kind: "workspace",
121 verified: true,
122 workspaces: [{ slug: workspace.slug, role: "member" }],
123 };
124 }
125
126 /** A free slug for `wanted` in the workspace. */
127 private async freeSlug(workspace: string, wanted: string): Promise<string> {
128 const base = slugOf(wanted) || "project";
129 for (let n = 1; n < 100; n++) {
130 const slug = n === 1 ? base : `${base}-${n}`;
131 const taken = await this.db
132 .prepare("SELECT 1 FROM projects WHERE workspace = ? AND slug = ?")
133 .bind(workspace, slug)
134 .first();
135 if (!taken) return slug;
136 }
137 return `${base}-${crypto.randomUUID().slice(0, 6)}`;
138 }
139
140 /** Gives a repository its own project, unless it has one. */
141 private async ensureFor(repo: Repo, createdBy: string): Promise<void> {
142 if (repo.forkOf) return;
143 const existing = await this.db.prepare("SELECT id FROM projects WHERE repo_id = ?").bind(repo.id).first();
144 if (existing) {
145 await this.db
146 .prepare("UPDATE projects SET repo_private = ?, default_branch = ?, repo_name = ? WHERE repo_id = ?")
147 .bind(repo.isPrivate ? 1 : 0, repo.defaultBranch, repo.name, repo.id)
148 .run();
149 return;
150 }
151 const at = now();
152 await this.db
153 .prepare(
154 `INSERT INTO projects (id, workspace, slug, name, description, repo_id, repo_namespace, repo_name, repo_private,
155 default_branch, root_dir, is_primary, created_by, created_at, updated_at)
156 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', 1, ?, ?, ?)
157 ON CONFLICT (workspace, slug) DO NOTHING`,
158 )
159 .bind(
160 newId("prj"),
161 repo.namespace.toLowerCase(),
162 await this.freeSlug(repo.namespace.toLowerCase(), repo.name),
163 repo.name,
164 repo.description,
165 repo.id,
166 repo.namespace,
167 repo.name,
168 repo.isPrivate ? 1 : 0,
169 repo.defaultBranch,
170 createdBy,
171 at,
172 at,
173 )
174 .run();
175 }
176
177 /** Projects for a workspace's repositories made before projects existed. Once. */
178 private async backfill(workspace: string): Promise<void> {
179 const done = await this.db.prepare("SELECT 1 FROM backfilled WHERE workspace = ?").bind(workspace).first();
180 if (done) return;
181 const actor = await this.workspaceActor(workspace);
182 if (!actor) return;
183 const list = await reposClient(this.env.REPOS).list(actor, { namespace: workspace });
184 for (const repo of list) {
185 if (repo.namespace.toLowerCase() === workspace) await this.ensureFor(repo, "g1t");
186 }
187 await this.db.prepare("INSERT OR REPLACE INTO backfilled (workspace, at) VALUES (?, ?)").bind(workspace, now()).run();
188 }
189
190 private visible(row: Row, viewer: Viewer): boolean {
191 return !row.repo_private || isMember(viewer, row.workspace);
192 }
193
194 async list(a: { workspace: string; viewer: Viewer }): Promise<Result<Project[]>> {
195 const workspace = a.workspace.toLowerCase();
196 await this.backfill(workspace);
197 const rows = await this.db
198 .prepare("SELECT * FROM projects WHERE workspace = ? ORDER BY name COLLATE NOCASE")
199 .bind(workspace)
200 .all<Row>();
201 return ok(rows.results.filter((row) => this.visible(row, a.viewer)).map(toProject));
202 }
203
204 async get(a: { workspace: string; slug: string; viewer: Viewer }): Promise<Result<Project>> {
205 const workspace = a.workspace.toLowerCase();
206 await this.backfill(workspace);
207 const row = await this.db
208 .prepare("SELECT * FROM projects WHERE workspace = ? AND slug = ?")
209 .bind(workspace, a.slug.toLowerCase())
210 .first<Row>();
211 if (!row || !this.visible(row, a.viewer)) return fail("not_found", "There is no such project.");
212 return ok(toProject(row));
213 }
214
215 async byRepo(a: { repoId: string }): Promise<Project[]> {
216 const rows = await this.db
217 .prepare("SELECT * FROM projects WHERE repo_id = ? ORDER BY is_primary DESC, created_at")
218 .bind(a.repoId)
219 .all<Row>();
220 if (rows.results.length > 0) return rows.results.map(toProject);
221 // A repository from before projects: give it its own now.
222 const path = await this.env.REPOS.fetch("https://repos/rpc/path_by_id", {
223 method: "POST",
224 headers: { "content-type": "application/json" },
225 body: JSON.stringify({ id: a.repoId }),
226 });
227 const repoPath = path.ok ? ((await path.json()) as { namespace: string; name: string } | null) : null;
228 if (!repoPath) return [];
229 const actor = await this.workspaceActor(repoPath.namespace);
230 const repo = actor ? await reposClient(this.env.REPOS).get(repoPath, actor) : null;
231 if (!repo?.ok) return [];
232 await this.ensureFor(repo.value, "g1t");
233 const again = await this.db.prepare("SELECT * FROM projects WHERE repo_id = ?").bind(a.repoId).all<Row>();
234 return again.results.map(toProject);
235 }
236
237 async create(a: { actor: User; workspace: string; input: NewProject }): Promise<Result<Project>> {
238 const workspace = a.workspace.toLowerCase();
239 if (!isMember(a.actor, workspace)) return fail("forbidden", "Only members can add projects to a workspace.");
240 if (a.input.repo.namespace.toLowerCase() !== workspace) {
241 return fail("invalid", "A project builds from one of its own workspace's repositories.");
242 }
243 const repo = await reposClient(this.env.REPOS).get(a.input.repo, a.actor);
244 if (!repo.ok) return repo;
245 if (repo.value.forkOf) return fail("invalid", "A pull request's working copy cannot be a project's source.");
246 const name = a.input.name.trim();
247 if (!name || name.length > 100) return fail("invalid", "A project's name is 1 to 100 characters.");
248 const rootDir = (a.input.rootDir ?? "").trim().replace(/^\/+|\/+$/g, "");
249 if (rootDir.split("/").some((part) => part === "..")) return fail("invalid", "The root directory is inside the repository.");
250 const slug = slugOf(name);
251 if (!slug) return fail("invalid", "Give the project a name with letters or digits.");
252 const taken = await this.db.prepare("SELECT 1 FROM projects WHERE workspace = ? AND slug = ?").bind(workspace, slug).first();
253 if (taken) return fail("conflict", `${workspace} already has a project called ${slug}.`);
254 const primary = !(await this.db.prepare("SELECT 1 FROM projects WHERE repo_id = ?").bind(repo.value.id).first());
255 const at = now();
256 const id = newId("prj");
257 await this.db
258 .prepare(
259 `INSERT INTO projects (id, workspace, slug, name, description, repo_id, repo_namespace, repo_name, repo_private,
260 default_branch, root_dir, is_primary, created_by, created_at, updated_at)
261 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
262 )
263 .bind(
264 id,
265 workspace,
266 slug,
267 name,
268 a.input.description?.trim() || null,
269 repo.value.id,
270 repo.value.namespace,
271 repo.value.name,
272 repo.value.isPrivate ? 1 : 0,
273 repo.value.defaultBranch,
274 rootDir,
275 primary ? 1 : 0,
276 a.actor.username,
277 at,
278 at,
279 )
280 .run();
281 return ok(toProject((await this.db.prepare("SELECT * FROM projects WHERE id = ?").bind(id).first<Row>())!));
282 }
283
284 async update(a: {
285 actor: User;
286 workspace: string;
287 slug: string;
288 changes: { name?: string; description?: string | null; rootDir?: string };
289 }): Promise<Result<Project>> {
290 const workspace = a.workspace.toLowerCase();
291 if (!isMember(a.actor, workspace)) return fail("forbidden", "Only members can change a workspace's projects.");
292 const row = await this.db
293 .prepare("SELECT * FROM projects WHERE workspace = ? AND slug = ?")
294 .bind(workspace, a.slug.toLowerCase())
295 .first<Row>();
296 if (!row) return fail("not_found", "There is no such project.");
297 const name = a.changes.name?.trim() || row.name;
298 const description = a.changes.description === undefined ? row.description : a.changes.description?.trim() || null;
299 const rootDir = a.changes.rootDir === undefined ? row.root_dir : a.changes.rootDir.trim().replace(/^\/+|\/+$/g, "");
300 if (rootDir.split("/").some((part) => part === "..")) return fail("invalid", "The root directory is inside the repository.");
301 await this.db
302 .prepare("UPDATE projects SET name = ?, description = ?, root_dir = ?, updated_at = ? WHERE id = ?")
303 .bind(name.slice(0, 100), description, rootDir, now(), row.id)
304 .run();
305 return ok(toProject((await this.db.prepare("SELECT * FROM projects WHERE id = ?").bind(row.id).first<Row>())!));
306 }
307
308 // ---- Dependencies ------------------------------------------------------
309
310 private async row(workspace: string, slug: string): Promise<Row | null> {
311 return this.db
312 .prepare("SELECT * FROM projects WHERE workspace = ? AND slug = ?")
313 .bind(workspace.toLowerCase(), slug.toLowerCase())
314 .first<Row>();
315 }
316
317 private async links(projectId: string): Promise<Dependencies> {
318 const [out, into] = await Promise.all([
319 this.db
320 .prepare(
321 `SELECT p.slug, p.name, d.alias, d.source FROM dependencies d JOIN projects p ON p.id = d.depends_on_id
322 WHERE d.project_id = ? ORDER BY p.name COLLATE NOCASE`,
323 )
324 .bind(projectId)
325 .all<LinkRow>(),
326 this.db
327 .prepare(
328 `SELECT p.slug, p.name, d.alias, d.source FROM dependencies d JOIN projects p ON p.id = d.project_id
329 WHERE d.depends_on_id = ? ORDER BY p.name COLLATE NOCASE`,
330 )
331 .bind(projectId)
332 .all<LinkRow>(),
333 ]);
334 const link = (r: LinkRow): DependencyLink => ({ slug: r.slug, name: r.name, as: r.alias, source: r.source });
335 return { dependsOn: out.results.map(link), usedBy: into.results.map(link) };
336 }
337
338 /** Whether `from` already reaches `to` through dependencies. */
339 private async reaches(from: string, to: string): Promise<boolean> {
340 const seen = new Set<string>([from]);
341 let frontier = [from];
342 while (frontier.length > 0) {
343 const marks = frontier.map(() => "?").join(", ");
344 const next = await this.db
345 .prepare(`SELECT depends_on_id AS id FROM dependencies WHERE project_id IN (${marks})`)
346 .bind(...frontier)
347 .all<{ id: string }>();
348 frontier = [];
349 for (const { id } of next.results) {
350 if (id === to) return true;
351 if (!seen.has(id)) {
352 seen.add(id);
353 frontier.push(id);
354 }
355 }
356 }
357 return false;
358 }
359
360 async dependencies(a: { workspace: string; slug: string; viewer: Viewer }): Promise<Result<Dependencies>> {
361 const row = await this.row(a.workspace, a.slug);
362 if (!row || !this.visible(row, a.viewer)) return fail("not_found", "There is no such project.");
363 return ok(await this.links(row.id));
364 }
365
366 /** Records `row` using `target`, after the checks every way of declaring one shares. */
367 private async declare(row: Row, target: Row, alias: string | null, source: "ui" | "file", by: string): Promise<Result<true>> {
368 if (target.id === row.id) return fail("invalid", "A project cannot depend on itself.");
369 if (alias != null && !ALIAS.test(alias)) {
370 return fail("invalid", "The variable's name is capital letters, digits and underscores, such as API_URL.");
371 }
372 if (await this.reaches(target.id, row.id)) {
373 return fail("conflict", `${target.slug} already depends on ${row.slug}, directly or through others; that would be a cycle.`);
374 }
375 const count = await this.db
376 .prepare("SELECT COUNT(*) AS n FROM dependencies WHERE project_id = ?")
377 .bind(row.id)
378 .first<{ n: number }>();
379 if ((count?.n ?? 0) >= MAX_DEPENDENCIES) return fail("invalid", `A project can depend on at most ${MAX_DEPENDENCIES} others.`);
380 await this.db
381 .prepare(
382 `INSERT INTO dependencies (project_id, depends_on_id, alias, source, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?)
383 ON CONFLICT (project_id, depends_on_id) DO UPDATE SET alias = excluded.alias, source = excluded.source`,
384 )
385 .bind(row.id, target.id, alias, source, by, now())
386 .run();
387 return ok(true);
388 }
389
390 async addDependency(a: { actor: User; workspace: string; slug: string; on: string; as: string | null }): Promise<Result<Dependencies>> {
391 if (!isMember(a.actor, a.workspace)) return fail("forbidden", "Only members can change a workspace's projects.");
392 const [row, target] = await Promise.all([this.row(a.workspace, a.slug), this.row(a.workspace, a.on)]);
393 if (!row) return fail("not_found", "There is no such project.");
394 if (!target) return fail("not_found", `${a.workspace} has no project called ${a.on}.`);
395 const existing = await this.db
396 .prepare("SELECT source FROM dependencies WHERE project_id = ? AND depends_on_id = ?")
397 .bind(row.id, target.id)
398 .first<{ source: string }>();
399 if (existing?.source === "file") return fail("conflict", "This dependency is declared in .g1t/project.yml; change it there.");
400 const alias = a.as?.trim() ? a.as.trim().toUpperCase() : null;
401 const done = await this.declare(row, target, alias, "ui", a.actor.username);
402 if (!done.ok) return done;
403 return ok(await this.links(row.id));
404 }
405
406 async removeDependency(a: { actor: User; workspace: string; slug: string; on: string }): Promise<Result<Dependencies>> {
407 if (!isMember(a.actor, a.workspace)) return fail("forbidden", "Only members can change a workspace's projects.");
408 const [row, target] = await Promise.all([this.row(a.workspace, a.slug), this.row(a.workspace, a.on)]);
409 if (!row || !target) return fail("not_found", "There is no such dependency.");
410 const removed = await this.db
411 .prepare("DELETE FROM dependencies WHERE project_id = ? AND depends_on_id = ? AND source = 'ui' RETURNING project_id")
412 .bind(row.id, target.id)
413 .first();
414 if (!removed) return fail("conflict", "This dependency is declared in .g1t/project.yml, or does not exist; change the file.");
415 return ok(await this.links(row.id));
416 }
417
418 async graph(a: { projectId: string }): Promise<ProjectGraph> {
419 const [out, into] = await Promise.all([
420 this.db
421 .prepare(
422 `SELECT p.id, p.slug, p.workspace, d.alias FROM dependencies d JOIN projects p ON p.id = d.depends_on_id WHERE d.project_id = ?`,
423 )
424 .bind(a.projectId)
425 .all<NodeRow>(),
426 this.db
427 .prepare(
428 `SELECT p.id, p.slug, p.workspace, d.alias FROM dependencies d JOIN projects p ON p.id = d.project_id WHERE d.depends_on_id = ?`,
429 )
430 .bind(a.projectId)
431 .all<NodeRow>(),
432 ]);
433 const node = (r: NodeRow) => ({ id: r.id, slug: r.slug, workspace: r.workspace, as: r.alias });
434 return { dependsOn: out.results.map(node), usedBy: into.results.map(node) };
435 }
436
437 /** For the runner: each project on a repository, with what it uses and what uses it. */
438 async contextForRepo(a: { repoId: string }): Promise<{ slug: string; name: string; dependencies: Dependencies }[]> {
439 const rows = await this.db.prepare("SELECT * FROM projects WHERE repo_id = ?").bind(a.repoId).all<Row>();
440 return Promise.all(rows.results.map(async (row) => ({ slug: row.slug, name: row.name, dependencies: await this.links(row.id) })));
441 }
442
443 /**
444 * A project's `.g1t/project.yml` at a commit of its default branch:
445 *
446 * dependsOn:
447 * - project: api
448 * as: API_URL
449 *
450 * Its dependencies replace the ones the file declared before. Ones that
451 * cannot be kept (an unknown project, a cycle) are left out.
452 */
453 private async syncFile(row: Row, commit: string): Promise<void> {
454 const actor = await this.workspaceActor(row.workspace);
455 if (!actor) return;
456 const path = row.root_dir ? `${row.root_dir}/.g1t/project.yml` : ".g1t/project.yml";
457 const blob = await reposClient(this.env.REPOS).blob({ namespace: row.repo_namespace, name: row.repo_name }, actor, commit, path);
458 if (!blob.ok || blob.value.text == null) {
459 // No file (any more): what it declared goes with it.
460 await this.db.prepare("DELETE FROM dependencies WHERE project_id = ? AND source = 'file'").bind(row.id).run();
461 return;
462 }
463 let declared: unknown[] = [];
464 try {
465 const parsed = parseYaml(blob.value.text) as { dependsOn?: unknown } | null;
466 if (Array.isArray(parsed?.dependsOn)) declared = parsed.dependsOn;
467 } catch (error) {
468 console.error("could not read", path, "of", row.slug, error);
469 return;
470 }
471 await this.db.prepare("DELETE FROM dependencies WHERE project_id = ? AND source = 'file'").bind(row.id).run();
472 for (const entry of declared.slice(0, MAX_DEPENDENCIES)) {
473 const item = entry as { project?: unknown; as?: unknown } | string;
474 const on = typeof item === "string" ? item : typeof item?.project === "string" ? item.project : null;
475 if (!on) continue;
476 const target = await this.row(row.workspace, on);
477 if (!target) continue;
478 const alias = typeof item === "object" && typeof item.as === "string" ? item.as.trim().toUpperCase() : null;
479 await this.declare(row, target, alias, "file", "g1t");
480 }
481 }
482
483 async onEvent(event: G1tEvent): Promise<void> {
484 if (event.type === "git.push" && event.data.defaultBranch) {
485 const rows = await this.db.prepare("SELECT * FROM projects WHERE repo_id = ?").bind(event.data.repoId).all<Row>();
486 for (const row of rows.results) await this.syncFile(row, event.data.after);
487 return;
488 }
489 if (event.type !== "repo.created") return;
490 const actor = await this.workspaceActor(event.data.namespace);
491 if (!actor) return;
492 const repo = await reposClient(this.env.REPOS).get({ namespace: event.data.namespace, name: event.data.name }, actor);
493 if (repo.ok) await this.ensureFor(repo.value, event.actor ?? "g1t");
494 }
495}
496
497export default {
498 async fetch(request: Request, env: Env): Promise<Response> {
499 const match = new URL(request.url).pathname.match(/^\/rpc\/([a-z_]+)$/);
500 if (request.method !== "POST" || !match) return new Response("Not found\n", { status: 404 });
501 const service = new Projects(env);
502 const args = (await request.json().catch(() => ({}))) as any;
503 switch (match[1]) {
504 case "list":
505 return Response.json(await service.list(args));
506 case "get":
507 return Response.json(await service.get(args));
508 case "by_repo":
509 return Response.json(await service.byRepo(args));
510 case "create":
511 return Response.json(await service.create(args));
512 case "update":
513 return Response.json(await service.update(args));
514 case "dependencies":
515 return Response.json(await service.dependencies(args));
516 case "add_dependency":
517 return Response.json(await service.addDependency(args));
518 case "remove_dependency":
519 return Response.json(await service.removeDependency(args));
520 case "graph":
521 return Response.json(await service.graph(args));
522 case "context_for_repo":
523 return Response.json(await service.contextForRepo(args));
524 default:
525 return new Response("Unknown method\n", { status: 404 });
526 }
527 },
528
529 async queue(batch: MessageBatch<G1tEvent>, env: Env): Promise<void> {
530 const service = new Projects(env);
531 for (const message of batch.messages) {
532 try {
533 await service.onEvent(message.body);
534 message.ack();
535 } catch (error) {
536 console.error("projects could not handle", message.body.type, error);
537 message.retry();
538 }
539 }
540 },
541} satisfies ExportedHandler<Env, G1tEvent>;