Skip to content
Sign in

flagon-io/g1t

public
0
229 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

AI Gateway: OpenAI's format, open models, and your own providers1/**
2 * Which model a gateway request names, and where it goes: one of the
3 * workspace's own providers, or g1t's catalogue.
4 *
5 * - **The workspace's own providers come first.** Each connection lists the
6 * models it takes (`gatewayModels`: ids, or prefixes ending in `*`); the
7 * first connection, in the order they were made, whose list matches the
8 * model gets the request. A prefix ending in `/*` is a namespace the
9 * workspace chose, and is taken off before the request is sent.
10 * - **Then g1t's catalogue**, by the provider's own id (`claude-sonnet-5-5`,
11 * `@cf/openai/gpt-oss-120b`) or with its provider in front
12 * (`anthropic/claude-sonnet-5-5`, `workers-ai/@cf/openai/gpt-oss-120b`).
13 */
14
15import type { GatewayModel, GatewayProvider } from "@g1t/contracts";
16
17/** The kinds of request the gateway answers. */
18export type Kind = "chat" | "embeddings";
19
20/** The catalogue's providers, and the API each speaks. */
21export const CATALOGUE_APIS: Record<string, "anthropic" | "openai"> = {
22 anthropic: "anthropic",
23 "workers-ai": "openai",
24};
25
26/** A catalogue model's id with its provider in front: `anthropic/claude-sonnet-5-5`. */
27export function catalogueId(model: Pick<GatewayModel, "provider" | "model">): string {
28 return `${model.provider}/${model.model}`;
29}
30
31/** The model a request names without a catalogue provider in front, when it has one. */
32function unprefixed(requested: string): string | null {
33 for (const provider of Object.keys(CATALOGUE_APIS)) {
34 if (requested.startsWith(`${provider}/`)) return requested.slice(provider.length + 1);
35 }
36 return null;
37}
38
39/**
40 * Whether a connection's pattern takes a model, and the model to send if
41 * so: as named, or without the namespace a `/*` pattern stands for.
42 */
43export function matchPattern(pattern: string, model: string): string | null {
44 if (pattern === "*") return model;
45 if (pattern.endsWith("/*")) {
46 const prefix = pattern.slice(0, -1);
47 return model.startsWith(prefix) && model.length > prefix.length ? model.slice(prefix.length) : null;
48 }
49 if (pattern.endsWith("*")) return model.startsWith(pattern.slice(0, -1)) ? model : null;
50 return pattern === model ? model : null;
51}
52
53/** Where a request goes. */
54export type Route =
55 | { to: "own"; provider: GatewayProvider; model: string }
56 | { to: "g1t"; entry: GatewayModel; api: "anthropic" | "openai"; model: string }
57 | { to: "none"; status: number; message: string };
58
59/** The catalogue model a request names, if any. */
60export function findOffered(requested: string, offered: GatewayModel[]): GatewayModel | null {
61 const bare = unprefixed(requested);
62 if (bare != null) {
63 const provider = requested.slice(0, requested.length - bare.length - 1);
64 return offered.find((row) => row.provider === provider && row.model === bare) ?? null;
65 }
66 return offered.find((row) => row.model === requested) ?? null;
67}
68
69/** The workspace's own provider a request goes to, if any, and the model it is sent as. */
70export function findOwn(requested: string, providers: GatewayProvider[]): { provider: GatewayProvider; model: string } | null {
71 const candidates = [requested];
72 const bare = unprefixed(requested);
73 if (bare) candidates.push(bare);
74 for (const provider of providers) {
75 for (const candidate of candidates) {
76 for (const pattern of provider.patterns) {
77 const model = matchPattern(pattern, candidate);
78 if (model) return { provider, model };
79 }
80 }
81 }
82 return null;
83}
84
85/** Where a request for `requested` goes, or why it goes nowhere. */
86export function routeModel(requested: unknown, kind: Kind, providers: GatewayProvider[], offered: GatewayModel[]): Route {
87 if (typeof requested !== "string" || !requested.trim()) {
88 return { to: "none", status: 400, message: "Name a model: `model` is required." };
89 }
90 const model = requested.trim();
91 const own = findOwn(model, providers);
92 if (own) {
93 if (kind === "embeddings" && own.provider.api !== "openai") {
94 return { to: "none", status: 400, message: `${model} goes to ${own.provider.name}, which speaks Anthropic's API and has no embeddings.` };
95 }
96 return { to: "own", ...own };
97 }
98 const entry = findOffered(model, offered);
99 const api = entry ? CATALOGUE_APIS[entry.provider] : undefined;
100 if (entry && api && (entry.kind ?? "chat") === kind) return { to: "g1t", entry, api, model: entry.model };
101 if (entry && api) {
102 const other = kind === "chat" ? "an embeddings model: send it to /openai/v1/embeddings" : "a chat model: send it to the chat or messages route";
103 return { to: "none", status: 400, message: `${model} is ${other}.` };
104 }
105 const names = offered
106 .filter((row) => (row.kind ?? "chat") === kind && CATALOGUE_APIS[row.provider])
107 .map(catalogueId)
108 .filter((id, i, all) => all.indexOf(id) === i);
109 const own_ = providers.length
110 ? " or a model one of the workspace's own providers under Integrations takes"
111 : ", or connect the workspace's own provider under Integrations";
112 return {
113 to: "none",
114 status: 404,
115 message: `${model} is not offered on the AI Gateway. Name one of ${names.join(", ")}${own_}. See https://docs.g1t.sh/guides/ai-gateway/#models`,
116 };
117}
118
119/** Dollars per million tokens, from millionths. */
120const dollars = (micros: number | undefined) => Math.round(Math.max(0, micros ?? 0)) / 1_000_000;
121
122/** One model, as `GET /openai/v1/models` lists it. */
123export type ListedModel = {
124 id: string;
125 object: "model";
126 created: number;
127 owned_by: string;
128 name: string;
129 kind: Kind;
130 /** `g1t` when g1t's account serves it and the workspace is charged; `workspace` on its own provider. */
131 billed_to: "g1t" | "workspace";
132 connection: string | null;
133 /** g1t's price per million tokens, in dollars; null on the workspace's own provider. */
134 pricing: {
135 currency: "usd";
136 input: number;
137 output: number;
138 cache_read: number;
139 cache_write: number;
140 cache_write_1h: number;
141 long_prompt: { above_tokens: number; input: number; output: number; cache_read: number; cache_write: number; cache_write_1h: number } | null;
142 } | null;
143};
144
145/**
146 * The models a workspace can use, as OpenAI lists models: its own
147 * providers' first (by the ids they take), then g1t's catalogue with its
148 * prices, each model once, cheapest Claude first as the catalogue orders
149 * them.
150 */
151export function listModels(providers: GatewayProvider[], offered: GatewayModel[]): ListedModel[] {
152 const listed: ListedModel[] = [];
153 const seen = new Set<string>();
154 const add = (model: ListedModel) => {
155 if (seen.has(model.id)) return;
156 seen.add(model.id);
157 listed.push(model);
158 };
159 for (const provider of providers) {
160 for (const pattern of provider.patterns) {
161 if (!pattern.includes("*")) {
162 add(own(pattern, provider));
163 continue;
164 }
165 const prefix = pattern.endsWith("/*") ? pattern.slice(0, -1) : "";
166 for (const model of provider.models) {
167 const id = `${prefix}${model}`;
168 if (matchPattern(pattern, id)) add(own(id, provider));
169 }
170 }
171 }
172 const names = new Set<string>();
173 for (const row of offered) {
174 if (!CATALOGUE_APIS[row.provider]) continue;
175 // A dated id for a model already listed is the same model.
176 const key = `${row.provider}:${row.name}`;
177 if (names.has(key)) continue;
178 names.add(key);
179 // A catalogue model one of the workspace's providers takes goes there.
180 const mine = findOwn(catalogueId(row), providers);
181 if (mine) {
182 add({ ...own(catalogueId(row), mine.provider), name: row.name, kind: row.kind ?? "chat" });
183 continue;
184 }
185 add({
186 id: catalogueId(row),
187 object: "model",
188 created: 0,
189 owned_by: row.provider,
190 name: row.name,
191 kind: row.kind ?? "chat",
192 billed_to: "g1t",
193 connection: null,
194 pricing: {
195 currency: "usd",
196 input: dollars(row.inputMicros),
197 output: dollars(row.outputMicros),
198 cache_read: dollars(row.cacheReadMicros),
199 cache_write: dollars(row.cacheWriteMicros),
200 cache_write_1h: dollars(row.cacheWrite1hMicros || row.cacheWriteMicros),
201 long_prompt: row.threshold
202 ? {
203 above_tokens: row.threshold,
204 input: dollars(row.overInputMicros),
205 output: dollars(row.overOutputMicros),
206 cache_read: dollars(row.overCacheReadMicros),
207 cache_write: dollars(row.overCacheWriteMicros),
208 cache_write_1h: dollars(row.overCacheWrite1hMicros || row.overCacheWriteMicros),
209 }
210 : null,
211 },
212 });
213 }
214 return listed;
215}
216
217function own(id: string, provider: GatewayProvider): ListedModel {
218 return {
219 id,
220 object: "model",
221 created: 0,
222 owned_by: provider.provider,
223 name: id,
224 kind: "chat",
225 billed_to: "workspace",
226 connection: provider.name,
227 pricing: null,
228 };
229}