Sign in

flagon-io/g1t

public

g1t/deploy/self-host/configs.mjs

328 lines13,256 bytesCodeBlame
1#!/usr/bin/env node
2// Writes the Wrangler configs a self-hosted g1t runs with, derived from the
3// hosted ones, so the two never drift apart.
4//
5// Each hosted service's wrangler.jsonc is read and changed only where
6// Cloudflare-only things live:
7//
8// - account, routes, placement, observability and builds are dropped;
9// - ARTIFACTS (git storage) becomes a service binding to workers/artifacts,
10// which keeps repositories in the git store (gitstore/server.mjs);
11// - EMAIL (Email Sending) becomes a service binding to workers/mail;
12// - the packages service keeps files in S3-compatible storage (MinIO)
13// instead of R2;
14// - services that are off in this phase (agents, the context hub, the
15// g1t.page dispatcher, model proxy) are bound to workers/off instead, and
16// events stop queueing work for them;
17// - URLs that name g1t.sh name PUBLIC_URL instead, and billing is free.
18//
19// Usage: node configs.mjs [outDir]
20// Environment: PUBLIC_URL, GITSTORE_URL, GITSTORE_SECRET, MAIL_URL,
21// ACTIONS_KEY, INTEGRATIONS_KEY, WEBHOOKS_KEY, IDENTITY_KEY,
22// PACKAGES_TOKEN_SECRET, S3_ENDPOINT, S3_BUCKET, S3_REGION,
23// S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_PUBLIC_ENDPOINT, and optionally
24// your own GitHub App: GITHUB_APP_ID, GITHUB_APP_SLUG, GITHUB_APP_CLIENT_ID,
25// GITHUB_APP_CLIENT_SECRET, GITHUB_APP_PRIVATE_KEY, GITHUB_APP_WEBHOOK_SECRET.
26//
27// The output is for `wrangler dev` (see start.sh): every Worker in one
28// workerd, the site first, with D1, KV and Queues kept on disk.
29
30import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
31import { dirname, join, relative, resolve } from "node:path";
32import { fileURLToPath } from "node:url";
33
34const here = dirname(fileURLToPath(import.meta.url));
35const root = resolve(here, "../..");
36const out = resolve(process.argv[2] ?? join(here, ".generated"));
37mkdirSync(out, { recursive: true });
38
39const PUBLIC_URL = (process.env.PUBLIC_URL ?? "http://localhost:8787").replace(/\/$/, "");
40
41// What runs, and what is off, is each unit's `self_host` in
42// deploy/stack.jsonc: the list hosted g1t deploys from.
43const STACK = Object.values(parseJsonc(readFileSync(join(root, "deploy/stack.jsonc"), "utf8")).units);
44
45/** Services that run, in the order Wrangler is given them (the site first). */
46export const RUNNING = STACK.filter((unit) => unit.self_host === "run")
47 .map((unit) => ({ name: unit.worker, dir: unit.path, web: unit.kind === "react-router" }))
48 .sort((a, b) => Number(b.web) - Number(a.web));
49
50/** What the off Worker calls each service that is off in phase 1. */
51const OFF_NAMES = {
52 "g1t-runner": "Agents",
53 "g1t-context": "Context search and memory",
54};
55const OFF = Object.fromEntries(
56 STACK.filter((unit) => unit.self_host === "off").map((unit) => [unit.worker, OFF_NAMES[unit.worker] ?? unit.worker]),
57);
58
59/** Sealing keys, by the service that holds each (hosted: Wrangler secrets). */
60const SECRETS = {
61 "g1t-actions": "ACTIONS_KEY",
62 "g1t-integrations": "INTEGRATIONS_KEY",
63 "g1t-webhooks": "WEBHOOKS_KEY",
64};
65
66/**
67 * g1t.sh's GitHub App is its own: an installation registers one of its
68 * own, or has none, and then no GitHub buttons appear. Its public settings
69 * replace the hosted vars; its secrets go only to the service that uses each.
70 */
71const GITHUB_VARS = ["GITHUB_APP_ID", "GITHUB_APP_SLUG", "GITHUB_APP_CLIENT_ID"];
72const GITHUB_SECRETS = {
73 "g1t-identity": ["GITHUB_APP_CLIENT_SECRET", "IDENTITY_KEY", "REGISTRATION_MODE"],
74 "g1t-integrations": ["GITHUB_APP_PRIVATE_KEY", "GITHUB_APP_WEBHOOK_SECRET"],
75};
76
77/**
78 * Services whose cron triggers scheduler.mjs runs here: sweeps and
79 * reminders that need nothing self-hosting lacks. Not run: actions (its
80 * minute would start scheduled workflows with no runner to take them),
81 * billing (reconciles against Cloudflare and Stripe), deployments (calls
82 * Cloudflare's API) and the services that are off.
83 */
84const SELF_HOST_CRONS = new Set(["g1t-repos", "g1t-events", "g1t-identity", "g1t-security", "g1t-webhooks", "g1t-packages"]);
85
86/** Queues whose consumers are off: events stops sending to them. */
87const OFF_QUEUES = new Set(["g1t-events-runner", "g1t-events-context"]);
88
89/** Strips comments and trailing commas from JSONC. Strings are respected. */
90function parseJsonc(text) {
91 let result = "";
92 let inString = false;
93 for (let i = 0; i < text.length; i++) {
94 const char = text[i];
95 if (inString) {
96 result += char;
97 if (char === "\\") result += text[++i];
98 else if (char === '"') inString = false;
99 } else if (char === '"') {
100 inString = true;
101 result += char;
102 } else if (char === "/" && text[i + 1] === "/") {
103 while (i < text.length && text[i] !== "\n") i++;
104 result += "\n";
105 } else if (char === "/" && text[i + 1] === "*") {
106 i = text.indexOf("*/", i + 2) + 1;
107 } else {
108 result += char;
109 }
110 }
111 return JSON.parse(result.replace(/,(\s*[}\]])/g, "$1"));
112}
113
114const rel = (path) => relative(out, resolve(root, path)).replaceAll("\\", "/");
115
116function hostedUrl(value) {
117 return typeof value === "string" ? value.replace(/https:\/\/(api\.)?g1t\.sh/g, PUBLIC_URL) : value;
118}
119
120function selfHosted(service) {
121 const hosted = parseJsonc(readFileSync(join(root, service.dir, "wrangler.jsonc"), "utf8"));
122 const config = {
123 name: hosted.name,
124 compatibility_date: hosted.compatibility_date,
125 compatibility_flags: hosted.compatibility_flags,
126 rules: hosted.rules,
127 vars: {},
128 };
129
130 if (service.web) {
131 // The site as React Router built it (apps/web/build), not its sources.
132 config.main = rel(`${service.dir}/build/server/index.js`);
133 config.no_bundle = true;
134 config.rules = [{ type: "ESModule", globs: ["**/*.js", "**/*.mjs"] }];
135 config.assets = { directory: rel(`${service.dir}/build/client`) };
136 } else {
137 config.main = rel(join(service.dir, hosted.main));
138 }
139
140 for (const [key, value] of Object.entries(hosted.vars ?? {})) config.vars[key] = hostedUrl(value);
141
142 if (hosted.d1_databases) {
143 config.d1_databases = hosted.d1_databases.map((db) => ({
144 binding: db.binding,
145 database_name: db.database_name,
146 database_id: db.database_id,
147 migrations_dir: rel(join(service.dir, db.migrations_dir ?? "migrations")),
148 }));
149 }
150 if (hosted.kv_namespaces) config.kv_namespaces = hosted.kv_namespaces.map(({ binding, id }) => ({ binding, id }));
151 if (hosted.triggers) config.triggers = hosted.triggers;
152
153 if (hosted.queues) {
154 config.queues = {};
155 if (hosted.queues.producers) {
156 config.queues.producers = hosted.queues.producers.filter((producer) => !OFF_QUEUES.has(producer.queue));
157 }
158 if (hosted.queues.consumers) config.queues.consumers = hosted.queues.consumers;
159 }
160
161 config.services = (hosted.services ?? []).map((binding) =>
162 OFF[binding.service] ? { binding: binding.binding, service: offName(binding.service) } : binding,
163 );
164
165 // Cloudflare-only bindings, and what stands in for them.
166 if (hosted.artifacts) {
167 for (const artifacts of hosted.artifacts) {
168 config.services.push({ binding: artifacts.binding, service: "g1t-artifacts" });
169 }
170 }
171 if (hosted.send_email) {
172 for (const email of hosted.send_email) config.services.push({ binding: email.name, service: "g1t-mail" });
173 }
174
175 // Secrets the hosted services hold, given here from the environment, each
176 // only to the service that uses it.
177 const secret = SECRETS[hosted.name];
178 if (secret && process.env[secret]) config.vars[secret] = process.env[secret];
179 for (const name of GITHUB_VARS) {
180 if (name in config.vars) config.vars[name] = process.env[name] ?? "";
181 }
182 for (const name of GITHUB_SECRETS[hosted.name] ?? []) {
183 if (process.env[name]) config.vars[name] = process.env[name];
184 }
185
186 // Self-hosted g1t charges nothing: billing records usage at cost and never
187 // stops work for it.
188 if (hosted.name === "g1t-billing") config.vars.FREE_WHILE_BUILDING = "true";
189 // Anyone may register on an installation of your own unless you set
190 // REGISTRATION_MODE=invite; invites then work as on g1t.sh, and the owners
191 // of INVITE_STAFF_WORKSPACES (yours, not g1t.sh's) invite without limit.
192 if (hosted.name === "g1t-identity") {
193 config.vars.REGISTRATION_MODE = process.env.REGISTRATION_MODE || "open";
194 config.vars.INVITE_STAFF_WORKSPACES = process.env.INVITE_STAFF_WORKSPACES ?? "";
195 if (process.env.INVITES_PER_USER) config.vars.INVITES_PER_USER = process.env.INVITES_PER_USER;
196 // Access requests are summarised to your own address, not g1t.sh's.
197 config.vars.WAITLIST_NOTIFY_EMAIL = process.env.WAITLIST_NOTIFY_EMAIL ?? "";
198 }
199 // Packages' files go to the compose file's MinIO (or any S3-compatible
200 // store) instead of R2, with no request size limit, and package
201 // addresses start with this installation's host.
202 if (hosted.name === "g1t-packages") {
203 Object.assign(config.vars, {
204 BLOB_STORE: "s3",
205 S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000",
206 S3_BUCKET: process.env.S3_BUCKET ?? "g1t-packages",
207 S3_REGION: process.env.S3_REGION ?? "us-east-1",
208 S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "",
209 S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY ?? "",
210 S3_PUBLIC_ENDPOINT: process.env.S3_PUBLIC_ENDPOINT ?? "",
211 MAX_REQUEST_BYTES: "0",
212 STORAGE_LIMITS: "off",
213 REGISTRY_HOST: new URL(PUBLIC_URL).host,
214 PACKAGES_TOKEN_SECRET: process.env.PACKAGES_TOKEN_SECRET ?? "",
215 });
216 delete config.vars.R2_ACCOUNT_ID;
217 delete config.vars.R2_BUCKET;
218 }
219 // Nothing to deploy to: deployments are off (no Cloudflare API token).
220 if (hosted.name === "g1t-deployments") delete config.vars.CUSTOM_HOSTNAMES_ZONE_ID;
221
222 return config;
223}
224
225function offName(service) {
226 return `${service}-off`;
227}
228
229function write(name, config) {
230 const path = join(out, `${name}.json`);
231 writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`);
232 return path;
233}
234
235const files = [];
236for (const service of RUNNING) files.push(write(service.name, selfHosted(service)));
237
238const compatibility_date = "2026-09-26";
239files.push(
240 write("g1t-artifacts", {
241 name: "g1t-artifacts",
242 main: rel("deploy/self-host/workers/artifacts/index.js"),
243 compatibility_date,
244 vars: {
245 GITSTORE_URL: process.env.GITSTORE_URL ?? "http://gitstore:8080",
246 GITSTORE_SECRET: process.env.GITSTORE_SECRET ?? "",
247 },
248 }),
249);
250files.push(
251 write("g1t-mail", {
252 name: "g1t-mail",
253 main: rel("deploy/self-host/workers/mail/index.js"),
254 compatibility_date,
255 vars: {
256 PUBLIC_URL,
257 MAIL_URL: process.env.MAIL_URL ?? "",
258 MAIL_FROM: process.env.MAIL_FROM ?? "",
259 },
260 }),
261);
262for (const [service, feature] of Object.entries(OFF)) {
263 files.push(
264 write(offName(service), {
265 name: offName(service),
266 main: rel("deploy/self-host/workers/off/index.js"),
267 compatibility_date,
268 vars: { OFF_NAME: feature },
269 }),
270 );
271}
272
273// The order Wrangler takes them in: the site first, as the one that serves.
274writeFileSync(join(out, "workers.txt"), `${files.map((file) => relative(out, file)).join("\n")}\n`);
275
276// What scheduler.mjs runs: each service's own crons, as hosted g1t's Cron
277// Triggers run them, for the services in SELF_HOST_CRONS.
278const schedules = RUNNING.filter((service) => SELF_HOST_CRONS.has(service.name))
279 .map((service) => ({
280 worker: service.name,
281 crons: parseJsonc(readFileSync(join(root, service.dir, "wrangler.jsonc"), "utf8")).triggers?.crons ?? [],
282 }))
283 .filter((schedule) => schedule.crons.length > 0);
284writeFileSync(join(out, "schedules.json"), `${JSON.stringify(schedules, null, 2)}\n`);
285
286// The status page runs in a workerd of its own (status.sh, the `status`
287// service in docker-compose.yml), so it stays up when the site does not:
288// not in workers.txt. It checks the site from inside Compose
289// (STATUS_CHECK_URL) and links to it at PUBLIC_URL. Parts this
290// installation does not run (the API, MCP, docs, g1t.page, the model
291// proxy, billing) are left off its page; a public repository of yours in
292// STATUS_PROBE_REPO adds the git check.
293{
294 const hosted = parseJsonc(readFileSync(join(root, "apps/status/wrangler.jsonc"), "utf8"));
295 const db = hosted.d1_databases[0];
296 write("g1t-status", {
297 name: hosted.name,
298 main: rel(join("apps/status", hosted.main)),
299 compatibility_date: hosted.compatibility_date,
300 rules: hosted.rules,
301 triggers: hosted.triggers,
302 d1_databases: [
303 {
304 binding: db.binding,
305 database_name: db.database_name,
306 database_id: db.database_id,
307 migrations_dir: rel(join("apps/status", db.migrations_dir)),
308 },
309 ],
310 vars: {
311 SITE_URL: (process.env.STATUS_CHECK_URL ?? "http://g1t:8787").replace(/\/$/, ""),
312 PUBLIC_SITE_URL: PUBLIC_URL,
313 API_URL: "",
314 MCP_URL: "",
315 DOCS_URL: "",
316 PAGES_URL: "",
317 MODELS_URL: "",
318 PROBE_REPO: process.env.STATUS_PROBE_REPO ?? "",
319 SUPPORT_URL: `${PUBLIC_URL}/support`,
320 OG_IMAGE: "",
321 // Its own address, for links made outside a request. No email
322 // binding here: subscribing by email is off, the feeds work.
323 STATUS_URL: `http://localhost:${process.env.STATUS_PORT ?? "8788"}`,
324 STATUS_ALERT_EMAIL: "",
325 },
326 });
327}
328console.log(`Wrote ${files.length} configs to ${out}`);