Sign in

flagon-io/g1t

public

g1t/deploy/self-host/configs.mjs

341 lines13,923 bytesCodeBlame
1#!/usr/bin/env node
2// Writes the Wrangler configs a self-hosted g1t runs with, derived from the
3// hosted ones, so the two never drift apart.
4//
5// Each hosted service's wrangler.jsonc is read and changed only where
6// Cloudflare-only things live:
7//
8// - account, routes, placement, observability and builds are dropped;
9// - ARTIFACTS (git storage) becomes a service binding to workers/artifacts,
10// which keeps repositories in the git store (gitstore/server.mjs);
11// - EMAIL (Email Sending) becomes a service binding to workers/mail;
12// - the packages service keeps files in S3-compatible storage (MinIO)
13// instead of R2, and the repos service its nightly backups (a bucket of
14// their own, BACKUP_S3_BUCKET);
15// - services that are off in this phase (agents, the context hub, the
16// g1t.page dispatcher, model proxy) are bound to workers/off instead, and
17// events stop queueing work for them;
18// - URLs that name g1t.sh name PUBLIC_URL instead, and billing is free.
19//
20// Usage: node configs.mjs [outDir]
21// Environment: PUBLIC_URL, GITSTORE_URL, GITSTORE_SECRET, MAIL_URL,
22// ACTIONS_KEY, INTEGRATIONS_KEY, WEBHOOKS_KEY, IDENTITY_KEY,
23// PACKAGES_TOKEN_SECRET, S3_ENDPOINT, S3_BUCKET, BACKUP_S3_BUCKET, S3_REGION,
24// S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_PUBLIC_ENDPOINT, and optionally
25// your own GitHub App: GITHUB_APP_ID, GITHUB_APP_SLUG, GITHUB_APP_CLIENT_ID,
26// GITHUB_APP_CLIENT_SECRET, GITHUB_APP_PRIVATE_KEY, GITHUB_APP_WEBHOOK_SECRET.
27//
28// The output is for `wrangler dev` (see start.sh): every Worker in one
29// workerd, the site first, with D1, KV and Queues kept on disk.
30
31import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
32import { dirname, join, relative, resolve } from "node:path";
33import { fileURLToPath } from "node:url";
34
35const here = dirname(fileURLToPath(import.meta.url));
36const root = resolve(here, "../..");
37const out = resolve(process.argv[2] ?? join(here, ".generated"));
38mkdirSync(out, { recursive: true });
39
40const PUBLIC_URL = (process.env.PUBLIC_URL ?? "http://localhost:8787").replace(/\/$/, "");
41
42// What runs, and what is off, is each unit's `self_host` in
43// deploy/stack.jsonc: the list hosted g1t deploys from.
44const STACK = Object.values(parseJsonc(readFileSync(join(root, "deploy/stack.jsonc"), "utf8")).units);
45
46/** Services that run, in the order Wrangler is given them (the site first). */
47export const RUNNING = STACK.filter((unit) => unit.self_host === "run")
48 .map((unit) => ({ name: unit.worker, dir: unit.path, web: unit.kind === "react-router" }))
49 .sort((a, b) => Number(b.web) - Number(a.web));
50
51/** What the off Worker calls each service that is off in phase 1. */
52const OFF_NAMES = {
53 "g1t-runner": "Agents",
54 "g1t-context": "Context search and memory",
55};
56const OFF = Object.fromEntries(
57 STACK.filter((unit) => unit.self_host === "off").map((unit) => [unit.worker, OFF_NAMES[unit.worker] ?? unit.worker]),
58);
59
60/** Sealing keys, by the service that holds each (hosted: Wrangler secrets). */
61const SECRETS = {
62 "g1t-actions": "ACTIONS_KEY",
63 "g1t-integrations": "INTEGRATIONS_KEY",
64 "g1t-webhooks": "WEBHOOKS_KEY",
65};
66
67/**
68 * g1t.sh's GitHub App is its own: an installation registers one of its
69 * own, or has none, and then no GitHub buttons appear. Its public settings
70 * replace the hosted vars; its secrets go only to the service that uses each.
71 */
72const GITHUB_VARS = ["GITHUB_APP_ID", "GITHUB_APP_SLUG", "GITHUB_APP_CLIENT_ID"];
73const GITHUB_SECRETS = {
74 "g1t-identity": ["GITHUB_APP_CLIENT_SECRET", "IDENTITY_KEY", "REGISTRATION_MODE"],
75 "g1t-integrations": ["GITHUB_APP_PRIVATE_KEY", "GITHUB_APP_WEBHOOK_SECRET"],
76};
77
78/**
79 * Services whose cron triggers scheduler.mjs runs here: sweeps and
80 * reminders that need nothing self-hosting lacks. Not run: actions (its
81 * minute would start scheduled workflows with no runner to take them),
82 * billing (reconciles against Cloudflare and Stripe), deployments (calls
83 * Cloudflare's API) and the services that are off.
84 */
85const SELF_HOST_CRONS = new Set(["g1t-repos", "g1t-events", "g1t-identity", "g1t-security", "g1t-webhooks", "g1t-packages"]);
86
87/** Queues whose consumers are off: events stops sending to them. */
88const OFF_QUEUES = new Set(["g1t-events-runner", "g1t-events-context"]);
89
90/** Strips comments and trailing commas from JSONC. Strings are respected. */
91function parseJsonc(text) {
92 let result = "";
93 let inString = false;
94 for (let i = 0; i < text.length; i++) {
95 const char = text[i];
96 if (inString) {
97 result += char;
98 if (char === "\\") result += text[++i];
99 else if (char === '"') inString = false;
100 } else if (char === '"') {
101 inString = true;
102 result += char;
103 } else if (char === "/" && text[i + 1] === "/") {
104 while (i < text.length && text[i] !== "\n") i++;
105 result += "\n";
106 } else if (char === "/" && text[i + 1] === "*") {
107 i = text.indexOf("*/", i + 2) + 1;
108 } else {
109 result += char;
110 }
111 }
112 return JSON.parse(result.replace(/,(\s*[}\]])/g, "$1"));
113}
114
115const rel = (path) => relative(out, resolve(root, path)).replaceAll("\\", "/");
116
117function hostedUrl(value) {
118 return typeof value === "string" ? value.replace(/https:\/\/(api\.)?g1t\.sh/g, PUBLIC_URL) : value;
119}
120
121function selfHosted(service) {
122 const hosted = parseJsonc(readFileSync(join(root, service.dir, "wrangler.jsonc"), "utf8"));
123 const config = {
124 name: hosted.name,
125 compatibility_date: hosted.compatibility_date,
126 compatibility_flags: hosted.compatibility_flags,
127 rules: hosted.rules,
128 vars: {},
129 };
130
131 if (service.web) {
132 // The site as React Router built it (apps/web/build), not its sources.
133 config.main = rel(`${service.dir}/build/server/index.js`);
134 config.no_bundle = true;
135 config.rules = [{ type: "ESModule", globs: ["**/*.js", "**/*.mjs"] }];
136 config.assets = { directory: rel(`${service.dir}/build/client`) };
137 } else {
138 config.main = rel(join(service.dir, hosted.main));
139 }
140
141 for (const [key, value] of Object.entries(hosted.vars ?? {})) config.vars[key] = hostedUrl(value);
142
143 if (hosted.d1_databases) {
144 config.d1_databases = hosted.d1_databases.map((db) => ({
145 binding: db.binding,
146 database_name: db.database_name,
147 database_id: db.database_id,
148 migrations_dir: rel(join(service.dir, db.migrations_dir ?? "migrations")),
149 }));
150 }
151 if (hosted.kv_namespaces) config.kv_namespaces = hosted.kv_namespaces.map(({ binding, id }) => ({ binding, id }));
152 if (hosted.triggers) config.triggers = hosted.triggers;
153
154 if (hosted.queues) {
155 config.queues = {};
156 if (hosted.queues.producers) {
157 config.queues.producers = hosted.queues.producers.filter((producer) => !OFF_QUEUES.has(producer.queue));
158 }
159 if (hosted.queues.consumers) config.queues.consumers = hosted.queues.consumers;
160 }
161
162 config.services = (hosted.services ?? []).map((binding) =>
163 OFF[binding.service] ? { binding: binding.binding, service: offName(binding.service) } : binding,
164 );
165
166 // Cloudflare-only bindings, and what stands in for them.
167 if (hosted.artifacts) {
168 for (const artifacts of hosted.artifacts) {
169 config.services.push({ binding: artifacts.binding, service: "g1t-artifacts" });
170 }
171 }
172 if (hosted.send_email) {
173 for (const email of hosted.send_email) config.services.push({ binding: email.name, service: "g1t-mail" });
174 }
175
176 // Secrets the hosted services hold, given here from the environment, each
177 // only to the service that uses it.
178 const secret = SECRETS[hosted.name];
179 if (secret && process.env[secret]) config.vars[secret] = process.env[secret];
180 for (const name of GITHUB_VARS) {
181 if (name in config.vars) config.vars[name] = process.env[name] ?? "";
182 }
183 for (const name of GITHUB_SECRETS[hosted.name] ?? []) {
184 if (process.env[name]) config.vars[name] = process.env[name];
185 }
186
187 // Self-hosted g1t charges nothing: billing records usage at cost and never
188 // stops work for it.
189 if (hosted.name === "g1t-billing") config.vars.FREE_WHILE_BUILDING = "true";
190 // Anyone may register on an installation of your own unless you set
191 // REGISTRATION_MODE=invite; invites then work as on g1t.sh, and the owners
192 // of INVITE_STAFF_WORKSPACES (yours, not g1t.sh's) invite without limit.
193 if (hosted.name === "g1t-identity") {
194 config.vars.REGISTRATION_MODE = process.env.REGISTRATION_MODE || "open";
195 config.vars.INVITE_STAFF_WORKSPACES = process.env.INVITE_STAFF_WORKSPACES ?? "";
196 if (process.env.INVITES_PER_USER) config.vars.INVITES_PER_USER = process.env.INVITES_PER_USER;
197 // Access requests are summarised to your own address, not g1t.sh's.
198 config.vars.WAITLIST_NOTIFY_EMAIL = process.env.WAITLIST_NOTIFY_EMAIL ?? "";
199 }
200 // Packages' files go to the compose file's MinIO (or any S3-compatible
201 // store) instead of R2, with no request size limit, and package
202 // addresses start with this installation's host.
203 if (hosted.name === "g1t-packages") {
204 Object.assign(config.vars, {
205 BLOB_STORE: "s3",
206 S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000",
207 S3_BUCKET: process.env.S3_BUCKET ?? "g1t-packages",
208 S3_REGION: process.env.S3_REGION ?? "us-east-1",
209 S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "",
210 S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY ?? "",
211 S3_PUBLIC_ENDPOINT: process.env.S3_PUBLIC_ENDPOINT ?? "",
212 MAX_REQUEST_BYTES: "0",
213 STORAGE_LIMITS: "off",
214 REGISTRY_HOST: new URL(PUBLIC_URL).host,
215 PACKAGES_TOKEN_SECRET: process.env.PACKAGES_TOKEN_SECRET ?? "",
216 });
217 delete config.vars.R2_ACCOUNT_ID;
218 delete config.vars.R2_BUCKET;
219 }
220 // Nightly backups' bundles go to a bucket of their own on the same
221 // S3-compatible store, instead of the BACKUPS R2 bucket.
222 if (hosted.name === "g1t-repos") {
223 Object.assign(config.vars, {
224 BACKUP_STORE: "s3",
225 BACKUP_S3_BUCKET: process.env.BACKUP_S3_BUCKET ?? "g1t-backups",
226 S3_ENDPOINT: process.env.S3_ENDPOINT ?? "http://minio:9000",
227 S3_REGION: process.env.S3_REGION ?? "us-east-1",
228 S3_ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID ?? "",
229 S3_SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY ?? "",
230 });
231 }
232 // Nothing to deploy to: deployments are off (no Cloudflare API token).
233 if (hosted.name === "g1t-deployments") delete config.vars.CUSTOM_HOSTNAMES_ZONE_ID;
234
235 return config;
236}
237
238function offName(service) {
239 return `${service}-off`;
240}
241
242function write(name, config) {
243 const path = join(out, `${name}.json`);
244 writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`);
245 return path;
246}
247
248const files = [];
249for (const service of RUNNING) files.push(write(service.name, selfHosted(service)));
250
251const compatibility_date = "2026-09-26";
252files.push(
253 write("g1t-artifacts", {
254 name: "g1t-artifacts",
255 main: rel("deploy/self-host/workers/artifacts/index.js"),
256 compatibility_date,
257 vars: {
258 GITSTORE_URL: process.env.GITSTORE_URL ?? "http://gitstore:8080",
259 GITSTORE_SECRET: process.env.GITSTORE_SECRET ?? "",
260 },
261 }),
262);
263files.push(
264 write("g1t-mail", {
265 name: "g1t-mail",
266 main: rel("deploy/self-host/workers/mail/index.js"),
267 compatibility_date,
268 vars: {
269 PUBLIC_URL,
270 MAIL_URL: process.env.MAIL_URL ?? "",
271 MAIL_FROM: process.env.MAIL_FROM ?? "",
272 },
273 }),
274);
275for (const [service, feature] of Object.entries(OFF)) {
276 files.push(
277 write(offName(service), {
278 name: offName(service),
279 main: rel("deploy/self-host/workers/off/index.js"),
280 compatibility_date,
281 vars: { OFF_NAME: feature },
282 }),
283 );
284}
285
286// The order Wrangler takes them in: the site first, as the one that serves.
287writeFileSync(join(out, "workers.txt"), `${files.map((file) => relative(out, file)).join("\n")}\n`);
288
289// What scheduler.mjs runs: each service's own crons, as hosted g1t's Cron
290// Triggers run them, for the services in SELF_HOST_CRONS.
291const schedules = RUNNING.filter((service) => SELF_HOST_CRONS.has(service.name))
292 .map((service) => ({
293 worker: service.name,
294 crons: parseJsonc(readFileSync(join(root, service.dir, "wrangler.jsonc"), "utf8")).triggers?.crons ?? [],
295 }))
296 .filter((schedule) => schedule.crons.length > 0);
297writeFileSync(join(out, "schedules.json"), `${JSON.stringify(schedules, null, 2)}\n`);
298
299// The status page runs in a workerd of its own (status.sh, the `status`
300// service in docker-compose.yml), so it stays up when the site does not:
301// not in workers.txt. It checks the site from inside Compose
302// (STATUS_CHECK_URL) and links to it at PUBLIC_URL. Parts this
303// installation does not run (the API, MCP, docs, g1t.page, the model
304// proxy, billing) are left off its page; a public repository of yours in
305// STATUS_PROBE_REPO adds the git check.
306{
307 const hosted = parseJsonc(readFileSync(join(root, "apps/status/wrangler.jsonc"), "utf8"));
308 const db = hosted.d1_databases[0];
309 write("g1t-status", {
310 name: hosted.name,
311 main: rel(join("apps/status", hosted.main)),
312 compatibility_date: hosted.compatibility_date,
313 rules: hosted.rules,
314 triggers: hosted.triggers,
315 d1_databases: [
316 {
317 binding: db.binding,
318 database_name: db.database_name,
319 database_id: db.database_id,
320 migrations_dir: rel(join("apps/status", db.migrations_dir)),
321 },
322 ],
323 vars: {
324 SITE_URL: (process.env.STATUS_CHECK_URL ?? "http://g1t:8787").replace(/\/$/, ""),
325 PUBLIC_SITE_URL: PUBLIC_URL,
326 API_URL: "",
327 MCP_URL: "",
328 DOCS_URL: "",
329 PAGES_URL: "",
330 MODELS_URL: "",
331 PROBE_REPO: process.env.STATUS_PROBE_REPO ?? "",
332 SUPPORT_URL: `${PUBLIC_URL}/support`,
333 OG_IMAGE: "",
334 // Its own address, for links made outside a request. No email
335 // binding here: subscribing by email is off, the feeds work.
336 STATUS_URL: `http://localhost:${process.env.STATUS_PORT ?? "8788"}`,
337 STATUS_ALERT_EMAIL: "",
338 },
339 });
340}
341console.log(`Wrote ${files.length} configs to ${out}`);