| 1 | import { createRequestHandler } from "react-router"; |
| 2 | |
| 3 | import { identityClient, isNamespaceShaped } from "@g1t/contracts"; |
| 4 | |
| 5 | import { addressesFor } from "../app/lib/addresses"; |
| 6 | import { hardenRegistryHeaders } from "../app/lib/content-safety"; |
| 7 | import { withSiteHeaders } from "../app/lib/page-headers"; |
| 8 | import { finishResponse, withRequestPerf } from "../app/lib/perf.server"; |
| 9 | import { READ_DEADLINE_MS, withDeadline } from "../app/lib/deadline"; |
| 10 | import { wantsServerErrors } from "../app/lib/developer-mode"; |
| 11 | import { crawlerLimited, gitLimited, pageLimited, usercontentLimited } from "../app/lib/front-door-limits"; |
| 12 | import { alsoKeptForCrawlers, costlyForCrawlers, crawlerCacheKey, crawlerKeepSeconds, crawlerOf, keptForCrawlers, signedOut } from "../app/lib/crawlers"; |
| 13 | import { goImport } from "../app/lib/go-get"; |
| 14 | import { repositoryOfPage, stillPublic } from "../app/lib/public-cache"; |
| 15 | import { registryWorkspace, servicePath } from "../app/lib/registry-paths"; |
| 16 | import { usercontentPath } from "../app/lib/usercontent"; |
| 17 | import { TOKEN_CHALLENGE } from "../app/lib/website-token"; |
| 18 | import { serveUsercontent } from "./usercontent"; |
| 19 | import { receiveSaml } from "./sso-saml"; |
| 20 | import { samlPath } from "../app/lib/sso"; |
| 21 | import { serveArchivedAsset } from "../app/lib/asset-archive"; |
| 22 | import { needsBuildRefresh } from "../app/lib/build-version"; |
| 23 | import { developerResource, pageRepresentation } from "../app/lib/agent-pages"; |
| 24 | |
| 25 | const loadBuild = () => import("virtual:react-router/server-build"); |
| 26 | |
| 27 | /** |
| 28 | * Given a function for the build, React Router derives everything from it |
| 29 | * again on every request: each route wrapped for the timings in |
| 30 | * entry.server.tsx, then the route table flattened and ranked, about a |
| 31 | * millisecond and a half of CPU a page. The build never changes while an |
| 32 | * isolate lives, so outside development it is loaded once and the handler |
| 33 | * made once. Development keeps the function, so a changed file is picked up. |
| 34 | * Only the made handler is kept, never a promise of one, so no request |
| 35 | * waits on another's (a build that fails to load is loaded again next time). |
| 36 | */ |
| 37 | let handler: ReturnType<typeof createRequestHandler> | undefined; |
| 38 | /** |
| 39 | * The same, in React Router's development mode: for a signed-in request |
| 40 | * with developer mode on (app/lib/developer-mode.ts), so a page that fails |
| 41 | * shows the server's own error rather than "Unexpected Server Error". Made |
| 42 | * the first time someone asks for it. |
| 43 | */ |
| 44 | let developerHandler: ReturnType<typeof createRequestHandler> | undefined; |
| 45 | async function requestHandler(request: Request): Promise<Response> { |
| 46 | if (import.meta.env.DEV) return createRequestHandler(loadBuild, import.meta.env.MODE)(request); |
| 47 | if (wantsServerErrors(request)) { |
| 48 | developerHandler ??= createRequestHandler(await loadBuild(), "development"); |
| 49 | return developerHandler(request); |
| 50 | } |
| 51 | handler ??= createRequestHandler(await loadBuild(), import.meta.env.MODE); |
| 52 | return handler(request); |
| 53 | } |
| 54 | |
| 55 | const DOCS = "https://docs.g1t.sh"; |
| 56 | |
| 57 | /** Where the documentation pages that used to live under /docs are now. */ |
| 58 | const MOVED_DOCS: Record<string, string> = { |
| 59 | "/docs": "/quickstart/", |
| 60 | "/docs/concepts": "/concepts/overview/", |
| 61 | "/docs/authentication": "/guides/authentication/", |
| 62 | "/docs/git": "/guides/git/", |
| 63 | "/docs/agents": "/guides/bring-your-own-agent/", |
| 64 | "/docs/api": "/reference/api/", |
| 65 | "/docs/api/reference": "/reference/api/", |
| 66 | }; |
| 67 | |
| 68 | export default { |
| 69 | async fetch(request, env, ctx) { |
| 70 | // Repository files and avatars, on their own origin |
| 71 | // (g1tusercontent.com): answered before anything of the site's runs, |
| 72 | // so nothing there reads or sets the session cookie. |
| 73 | const usercontent = usercontentPath(new URL(request.url), addressesFor(env).usercontent); |
| 74 | if (usercontent !== null) { |
| 75 | return (await usercontentLimited(env, request, usercontent)) ?? serveUsercontent(env, ctx, request, usercontent); |
| 76 | } |
| 77 | // Every answer: no sniffing, a referrer of the origin alone, and no |
| 78 | // framing of pages (app/lib/page-headers.ts). |
| 79 | return withSiteHeaders(await site(request, env, ctx)); |
| 80 | }, |
| 81 | } satisfies ExportedHandler<Env>; |
| 82 | |
| 83 | async function site(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> { |
| 84 | const { pathname } = new URL(request.url); |
| 85 | // An old tab's chunk or stylesheet must not turn into an HTML route/404. |
| 86 | if (pathname.startsWith("/assets/")) return serveArchivedAsset(request, env.DOWNLOADS); |
| 87 | // Git over HTTPS shares this hostname but belongs to the repos service. |
| 88 | // Its answer goes back to the git client as it is: a repository under a |
| 89 | // renamed workspace's old name answers with a 301, which git follows and |
| 90 | // must see, so the redirect is never followed here. |
| 91 | // The container registry (`docker login g1t.sh`) and the npm registry |
| 92 | // (`g1t.sh/-/npm/`) are the packages |
| 93 | // service's, handed over the same way. |
| 94 | // `go get g1t.sh/<workspace>/<repo>`: where its code is, from the |
| 95 | // address alone, so it costs nothing and caches. |
| 96 | const go = request.method === "GET" ? goImport(new URL(request.url)) : null; |
| 97 | if (go) { |
| 98 | return new Response(go, { |
| 99 | headers: { "content-type": "text/html; charset=utf-8", "cache-control": "public, max-age=3600" }, |
| 100 | }); |
| 101 | } |
| 102 | const service = servicePath(pathname); |
| 103 | if (service === "git") { |
| 104 | // Per address without credentials, per credential with them |
| 105 | // (app/lib/front-door-limits.ts): anonymous clones are not free to |
| 106 | // the repository's owner. |
| 107 | return (await gitLimited(env, request)) ?? proxyGit(env, request); |
| 108 | } |
| 109 | if (service === "packages") { |
| 110 | return proxyPackages(env, request); |
| 111 | } |
| 112 | // The documentation is its own site. |
| 113 | const developer = ["GET", "HEAD"].includes(request.method) ? developerResource(pathname, addressesFor(env)) : null; |
| 114 | if (developer) return Response.redirect(developer, 308); |
| 115 | if (pathname === "/docs" || pathname.startsWith("/docs/")) { |
| 116 | const page = pathname.endsWith("/") ? pathname.slice(0, -1) : pathname; |
| 117 | const target = MOVED_DOCS[page] ?? "/"; |
| 118 | return Response.redirect(DOCS + target, 301); |
| 119 | } |
| 120 | // Pages and data requests, limited per address signed out and per |
| 121 | // session signed in (app/lib/front-door-limits.ts). |
| 122 | const limited = await pageLimited(env, request, pathname); |
| 123 | if (limited) return limited; |
| 124 | // A workspace's identity provider posting a SAML response, from its own |
| 125 | // site: React Router refuses cross-site posts (workers/sso-saml.ts). |
| 126 | const saml = request.method === "POST" ? samlPath(pathname) : null; |
| 127 | if (saml) return receiveSaml(env, request, saml); |
| 128 | // Crawlers see what anyone signed out sees, from the cache for hours |
| 129 | // (app/lib/crawlers.ts). |
| 130 | const crawler = crawlerOf(request); |
| 131 | if (crawler) return serveCrawler(env, request, ctx, crawler); |
| 132 | // Every page and data request says where its time went (Server-Timing) |
| 133 | // and keeps the reader's D1 bookmarks (app/lib/perf.server.ts). |
| 134 | const render = () => withRequestPerf(request, async () => finishResponse(request, await requestHandler(request))); |
| 135 | if (anonymousPage(request, pathname)) return servePublic(env, request, ctx, render); |
| 136 | const answer = await render(); |
| 137 | // A token the site refused (app/lib/website-token.ts) is told so as the |
| 138 | // API tells it: React Router drops the headers of what middleware throws. |
| 139 | if (answer.status === 401 && request.headers.has("authorization")) answer.headers.set("www-authenticate", TOKEN_CHALLENGE); |
| 140 | return answer; |
| 141 | } |
| 142 | |
| 143 | /** |
| 144 | * Public pages as someone signed out sees them: the same for every such |
| 145 | * visitor, so kept in this data centre's cache. Reserved first segments |
| 146 | * (settings, sign-in, invitations and the like) and workspace pages (`-`) |
| 147 | * are never kept. A repository's kept page is served only while repos |
| 148 | * says the repository is still public: one made private or deleted is |
| 149 | * never served from any data centre's copy. |
| 150 | */ |
| 151 | const PUBLIC_TOP = /^\/(?:|_root\.data|pricing|explore|security|support|policies(?:\/[a-z-]+)?)(?:\.data)?$/; |
| 152 | const PUBLIC_PROJECT = |
| 153 | /^\/(?!(?:settings|u|auth|oauth|integrations|new|invite|workspaces|device|verify|confirm-email|login|register|logout|forgot|reset|search|status|avatars|docs)\/)[^/]+\/(?!-\/|-$)[^/]+(?:\/(?:code|commits|issues|pulls|pull\/\d+|issues\/\d+|commit\/[0-9a-f]+|tree\/.+|blob\/.+))?(?:\.data)?$/; |
| 154 | /** Fresh for this long; then served once more while a new copy is made. */ |
| 155 | const PUBLIC_FRESH_SECONDS = 30; |
| 156 | const PUBLIC_STALE_SECONDS = 300; |
| 157 | /** The longest a cache lookup holds up a public page. */ |
| 158 | const CACHE_DEADLINE_MS = 2_000; |
| 159 | |
| 160 | function anonymousPage(request: Request, pathname: string): boolean { |
| 161 | if (request.method !== "GET") return false; |
| 162 | // A cached data response must not bypass the build-switch middleware. |
| 163 | if (needsBuildRefresh(request, import.meta.env.G1T_BUILD_ID)) return false; |
| 164 | if (/(?:^|;\s*)g1t_session=/.test(request.headers.get("cookie") ?? "")) return false; |
| 165 | // A token signs the request in (app/lib/website-token.ts): never kept, never served a kept page. |
| 166 | if (request.headers.has("authorization")) return false; |
| 167 | return PUBLIC_TOP.test(pathname) || PUBLIC_PROJECT.test(pathname); |
| 168 | } |
| 169 | |
| 170 | async function servePublic(env: Env, request: Request, ctx: ExecutionContext, render: () => Promise<Response>): Promise<Response> { |
| 171 | const cache = (caches as unknown as { default: Cache }).default; |
| 172 | const keyUrl = new URL(request.url); |
| 173 | // Cached HTML and route data belong to the build that rendered them. |
| 174 | keyUrl.searchParams.set("_g1t_build", import.meta.env.G1T_BUILD_ID); |
| 175 | // Vary alone does not partition the Workers Cache API's explicit key. |
| 176 | keyUrl.searchParams.set("_g1t_representation", pageRepresentation(request.headers.get("accept"))); |
| 177 | const key = new Request(keyUrl, { method: "GET" }); |
| 178 | const repository = PUBLIC_PROJECT.test(new URL(request.url).pathname) ? repositoryOfPage(new URL(request.url).pathname) : null; |
| 179 | // Asked alongside the cache, so a hit waits for one indexed read at most. |
| 180 | // A cache that does not answer is a miss: the page is drawn afresh. |
| 181 | const [cached, visible] = await Promise.all([ |
| 182 | withDeadline("cache", CACHE_DEADLINE_MS, () => cache.match(key)).catch(() => undefined), |
| 183 | repository ? isStillPublic(env, repository) : Promise.resolve(true), |
| 184 | ]); |
| 185 | if (cached && !visible) { |
| 186 | ctx.waitUntil( |
| 187 | cache.delete(key).then( |
| 188 | () => undefined, |
| 189 | () => undefined, |
| 190 | ), |
| 191 | ); |
| 192 | return render(); |
| 193 | } |
| 194 | const keptAt = Number(cached?.headers.get("x-g1t-kept-at") ?? 0); |
| 195 | const age = Math.round((Date.now() - keptAt) / 1000); |
| 196 | const refresh = async () => { |
| 197 | const fresh = await render(); |
| 198 | // Only a plain answer for everyone: nothing that sets a cookie or says |
| 199 | // it is private. |
| 200 | const cacheable = |
| 201 | (fresh.status === 200 || fresh.status === 404) && !fresh.headers.has("set-cookie") && !/private|no-store/.test(fresh.headers.get("cache-control") ?? ""); |
| 202 | if (cacheable) { |
| 203 | const copy = new Response(fresh.clone().body, fresh); |
| 204 | copy.headers.set("x-g1t-kept-at", String(Date.now())); |
| 205 | copy.headers.set("cache-control", `public, max-age=${PUBLIC_STALE_SECONDS}`); |
| 206 | ctx.waitUntil(cache.put(key, copy)); |
| 207 | } |
| 208 | return fresh; |
| 209 | }; |
| 210 | if (cached && keptAt > 0 && age < PUBLIC_STALE_SECONDS) { |
| 211 | if (age >= PUBLIC_FRESH_SECONDS) |
| 212 | ctx.waitUntil( |
| 213 | refresh().then( |
| 214 | () => undefined, |
| 215 | () => undefined, |
| 216 | ), |
| 217 | ); |
| 218 | const answer = new Response(cached.body, cached); |
| 219 | answer.headers.delete("x-g1t-kept-at"); |
| 220 | answer.headers.delete("cache-control"); |
| 221 | answer.headers.set("server-timing", `cache;desc="hit, ${age}s old"`); |
| 222 | return answer; |
| 223 | } |
| 224 | return refresh(); |
| 225 | } |
| 226 | |
| 227 | /** |
| 228 | * A crawler's request (app/lib/crawlers.ts), without its cookies or token. |
| 229 | * A public page is served from this data centre's cache, kept for six hours |
| 230 | * (a page of one commit for a day) under its address alone, and drawn only |
| 231 | * when it is not there; a costly page drawn counts against the crawler's |
| 232 | * limit. A repository's kept page is served only while it is public, as for |
| 233 | * people, but whether it is public is kept for a minute: repos asked on |
| 234 | * every hit would be most of what a crawler still costs. |
| 235 | */ |
| 236 | async function serveCrawler(env: Env, request: Request, ctx: ExecutionContext, family: string): Promise<Response> { |
| 237 | const anonymous = signedOut(request); |
| 238 | const render = () => withRequestPerf(anonymous, async () => finishResponse(anonymous, await requestHandler(anonymous))); |
| 239 | const url = new URL(request.url); |
| 240 | const costly = costlyForCrawlers(url); |
| 241 | const kept = |
| 242 | request.method === "GET" && |
| 243 | !needsBuildRefresh(request, import.meta.env.G1T_BUILD_ID) && |
| 244 | (PUBLIC_TOP.test(url.pathname) || PUBLIC_PROJECT.test(url.pathname) || alsoKeptForCrawlers(url.pathname)); |
| 245 | if (!kept) return (costly ? await crawlerLimited(env, family) : null) ?? render(); |
| 246 | const cache = (caches as unknown as { default: Cache }).default; |
| 247 | const key = crawlerCacheKey(url, pageRepresentation(request.headers.get("accept"))); |
| 248 | const cached = await withDeadline("cache", CACHE_DEADLINE_MS, () => cache.match(key)).catch(() => undefined); |
| 249 | const repository = PUBLIC_TOP.test(url.pathname) ? null : repositoryOfPage(url.pathname); |
| 250 | if (cached && (!repository || (await publicForCrawlers(env, ctx, url.origin, repository)))) { |
| 251 | const age = Math.round((Date.now() - Number(cached.headers.get("x-g1t-kept-at") ?? Date.now())) / 1000); |
| 252 | const answer = new Response(cached.body, cached); |
| 253 | answer.headers.delete("x-g1t-kept-at"); |
| 254 | answer.headers.delete("cache-control"); |
| 255 | answer.headers.set("server-timing", `cache;desc="crawler hit, ${age}s old"`); |
| 256 | return answer; |
| 257 | } |
| 258 | if (costly) { |
| 259 | const limited = await crawlerLimited(env, family); |
| 260 | if (limited) return limited; |
| 261 | } |
| 262 | const fresh = await render(); |
| 263 | if (keptForCrawlers(fresh)) { |
| 264 | const copy = new Response(fresh.clone().body, fresh); |
| 265 | copy.headers.set("x-g1t-kept-at", String(Date.now())); |
| 266 | copy.headers.set("cache-control", `public, max-age=${crawlerKeepSeconds(url)}`); |
| 267 | ctx.waitUntil(cache.put(key, copy).catch(() => undefined)); |
| 268 | } |
| 269 | return fresh; |
| 270 | } |
| 271 | |
| 272 | /** How long whether a repository is public is kept for crawlers. */ |
| 273 | const CRAWLER_VISIBILITY_SECONDS = 60; |
| 274 | |
| 275 | /** Whether the repository is public, asked of repos at most once a minute per data centre. */ |
| 276 | async function publicForCrawlers(env: Env, ctx: ExecutionContext, origin: string, repository: string): Promise<boolean> { |
| 277 | const cache = (caches as unknown as { default: Cache }).default; |
| 278 | const key = new Request(`${origin}/_g1t/crawler-visibility/${encodeURIComponent(repository)}`, { method: "GET" }); |
| 279 | const known = await withDeadline("cache", CACHE_DEADLINE_MS, () => cache.match(key)).catch(() => undefined); |
| 280 | if (known) return (await known.text()) === "public"; |
| 281 | const visible = await isStillPublic(env, repository); |
| 282 | const answer = new Response(visible ? "public" : "not public", { headers: { "cache-control": `public, max-age=${CRAWLER_VISIBILITY_SECONDS}` } }); |
| 283 | ctx.waitUntil(cache.put(key, answer).catch(() => undefined)); |
| 284 | return visible; |
| 285 | } |
| 286 | |
| 287 | /** Whether the repository is there and public; anything else, including no answer, is no. */ |
| 288 | async function isStillPublic(env: Env, repository: string): Promise<boolean> { |
| 289 | try { |
| 290 | return await withDeadline("repos visibility", READ_DEADLINE_MS, async (signal) => { |
| 291 | const answer = await env.REPOS.fetch("https://service/rpc/visibility", { |
| 292 | method: "POST", |
| 293 | headers: { "content-type": "application/json" }, |
| 294 | body: JSON.stringify({ paths: [repository] }), |
| 295 | signal, |
| 296 | }); |
| 297 | return answer.ok && stillPublic(await answer.json(), repository); |
| 298 | }); |
| 299 | } catch { |
| 300 | return false; |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | /** |
| 305 | * A git request, answered by the repos service. Its `Server-Timing` header |
| 306 | * gains `repos`: how long the answer took to start from here, so the time |
| 307 | * between this Worker and the repos service shows beside the steps the |
| 308 | * repos service reports. |
| 309 | */ |
| 310 | async function proxyGit(env: Env, request: Request): Promise<Response> { |
| 311 | const started = Date.now(); |
| 312 | const answer = await env.REPOS.fetch(new Request(request, { redirect: "manual" })); |
| 313 | const response = new Response(answer.body, answer); |
| 314 | response.headers.append("server-timing", `repos;dur=${Date.now() - started}`); |
| 315 | return response; |
| 316 | } |
| 317 | |
| 318 | /** |
| 319 | * A registry request, answered by the packages service as it is: its |
| 320 | * redirects (a large blob sent to storage) go back to the client, which |
| 321 | * follows them itself. One that found nothing under a workspace's old name |
| 322 | * or an alias staff set (`g1t` for `flagon-io`) is sent to the same path |
| 323 | * under the workspace's name: only the not-found answer pays for the lookup. |
| 324 | * Every answer runs nothing in a browser (app/lib/content-safety.ts): what |
| 325 | * a registry serves is its publisher's, on this origin. |
| 326 | */ |
| 327 | async function proxyPackages(env: Env, request: Request): Promise<Response> { |
| 328 | const started = Date.now(); |
| 329 | const answer = await env.PACKAGES.fetch(new Request(request, { redirect: "manual" })); |
| 330 | const moved = answer.status === 404 ? await registryMoved(env, request) : null; |
| 331 | const response = moved ?? new Response(answer.body, answer); |
| 332 | hardenRegistryHeaders(response.headers); |
| 333 | response.headers.append("server-timing", `packages;dur=${Date.now() - started}`); |
| 334 | return response; |
| 335 | } |
| 336 | |
| 337 | /** Where a registry request under an alias or old name goes now, or null. */ |
| 338 | async function registryMoved(env: Env, request: Request): Promise<Response | null> { |
| 339 | const url = new URL(request.url); |
| 340 | const named = registryWorkspace(url.pathname); |
| 341 | if (!named || !isNamespaceShaped(named.slug)) return null; |
| 342 | let current: string | null = null; |
| 343 | try { |
| 344 | current = await identityClient(env.IDENTITY).resolveSlug(named.slug); |
| 345 | } catch { |
| 346 | return null; |
| 347 | } |
| 348 | if (!current || current === named.slug) return null; |
| 349 | const get = request.method === "GET" || request.method === "HEAD"; |
| 350 | // 308 keeps a publish a PUT, for the clients that follow it. |
| 351 | return new Response(null, { status: get ? 301 : 308, headers: { location: named.under(current) + url.search } }); |
| 352 | } |