Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/models/src/discover.ts

207 lines9,971 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Models: g1t keeps up with new models, and staff choose each default in sudo1/**
2 * Discovery: which models g1t's providers offer now, so new ones reach the
3 * catalogue without anyone typing an id.
4 *
5 * Once a day (the cron in wrangler.jsonc) and when staff press "Check for
6 * new models" in sudo (the `Discovery` entrypoint in index.ts), this lists:
7 *
8 * - **Anthropic**, with `GET /v1/models` through g1t's AI Gateway (the
9 * same path and credentials g1t's runs use), or straight to Anthropic
10 * with g1t's key where there is no gateway.
11 * - **Workers AI**, with Cloudflare's model search
12 * (`GET /accounts/{account}/ai/models/search`), with `WORKERS_AI_TOKEN`
13 * (Workers AI Read) or else `AI_GATEWAY_TOKEN`.
14 *
15 * Listing models is free: nothing here calls a model. Each provider's list
16 * goes to billing's `record_discovery`, which owns the catalogue: it adds
17 * new ids as `new`, marks ones no longer listed `deprecated`, and emails
18 * staff. A provider that cannot be listed is recorded as a failed check
19 * and changes nothing.
20 */
21import type { DiscoveryResult, ProviderModel } from "@g1t/contracts";
22
23import type { HostedRouting } from "./route.ts";
24
25export type Fetch = (url: string, init?: RequestInit) => Promise<Response>;
26
27/** The version of Anthropic's API the list is read with. */
28export const ANTHROPIC_VERSION = "2023-06-01";
29/** The most pages read from either provider: far more models than either offers. */
30const MAX_PAGES = 20;
31const WORKERS_AI_PAGE = 100;
32
33/** Where Anthropic's list is read, with what; null when g1t has no way to reach Anthropic. */
34export function anthropicRequest(env: HostedRouting, afterId: string | null): { url: string; headers: Headers } | null {
35 const headers = new Headers({ "anthropic-version": ANTHROPIC_VERSION });
36 const query = `?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ""}`;
37 if (env.ANTHROPIC_API_KEY) headers.set("x-api-key", env.ANTHROPIC_API_KEY);
38 if (env.AI_GATEWAY_ID && env.CLOUDFLARE_ACCOUNT_ID && (env.AI_GATEWAY_TOKEN || env.ANTHROPIC_API_KEY)) {
39 if (env.AI_GATEWAY_TOKEN) headers.set("cf-aig-authorization", `Bearer ${env.AI_GATEWAY_TOKEN}`);
40 // Told apart from runs in the gateway's logs.
41 headers.set("cf-aig-metadata", JSON.stringify({ task: "discovery" }));
42 return { url: `https://gateway.ai.cloudflare.com/v1/${env.CLOUDFLARE_ACCOUNT_ID}/${env.AI_GATEWAY_ID}/anthropic/v1/models${query}`, headers };
43 }
44 if (!env.ANTHROPIC_API_KEY) return null;
45 return { url: `https://api.anthropic.com/v1/models${query}`, headers };
46}
47
48type Json = Record<string, unknown>;
49
50const isObject = (value: unknown): value is Json => typeof value === "object" && value !== null && !Array.isArray(value);
51const supported = (node: unknown): boolean => isObject(node) && node.supported === true;
52const count = (value: unknown): number => {
53 const n = typeof value === "string" ? Number(value) : value;
54 return typeof n === "number" && Number.isFinite(n) && n > 0 ? Math.floor(n) : 0;
55};
56
57/**
58 * What an Anthropic model can do, from the capability tree its list
59 * gives: `effort`, `thinking`, `vision`, and `tools` (every Claude model
60 * takes tools). Unknown without the tree.
61 */
62export function anthropicCapabilities(tree: unknown): string[] {
63 if (!isObject(tree)) return [];
64 const out: string[] = [];
65 if (supported(tree.effort)) out.push("effort");
66 if (supported(tree.thinking)) out.push("thinking");
67 out.push("tools");
68 if (supported(tree.image_input)) out.push("vision");
69 return out;
70}
71
72/** One model from Anthropic's list, or null when it has no id. */
73export function fromAnthropic(item: unknown): ProviderModel | null {
74 if (!isObject(item) || typeof item.id !== "string" || !item.id.trim()) return null;
75 return {
76 id: item.id.trim(),
77 name: typeof item.display_name === "string" ? item.display_name : "",
78 kind: "chat",
79 contextWindow: count(item.max_input_tokens),
80 maxOutput: count(item.max_tokens),
81 capabilities: anthropicCapabilities(item.capabilities),
82 price: null,
83 };
84}
85
86/** Every model Anthropic lists for g1t's key, page by page. Throws with what went wrong. */
87export async function listAnthropic(env: HostedRouting, fetcher: Fetch): Promise<ProviderModel[]> {
88 const models: ProviderModel[] = [];
89 let after: string | null = null;
90 for (let page = 0; page < MAX_PAGES; page++) {
91 const request = anthropicRequest(env, after);
92 if (!request) throw new Error("No way to reach Anthropic: set AI_GATEWAY_TOKEN (or ANTHROPIC_API_KEY) on the models service.");
93 const answer = await fetcher(request.url, { headers: request.headers });
94 if (!answer.ok) throw new Error(`Anthropic's model list answered ${answer.status}: ${(await answer.text()).slice(0, 300)}`);
95 const body = (await answer.json()) as Json;
96 const data = Array.isArray(body.data) ? body.data : [];
97 for (const item of data) {
98 const model = fromAnthropic(item);
99 if (model) models.push(model);
100 }
101 const last = typeof body.last_id === "string" ? body.last_id : null;
102 if (body.has_more !== true || !last || data.length === 0) return models;
103 after = last;
104 }
105 return models;
106}
107
108/** A Workers AI model's kind, from its task. */
109export function workersAiKind(task: unknown): string {
110 const name = isObject(task) && typeof task.name === "string" ? task.name.toLowerCase() : "";
111 if (name === "text generation") return "chat";
112 if (name === "text embeddings") return "embeddings";
113 return name ? name.replace(/\s+/g, "-") : "other";
114}
115
116/** Dollars per million tokens to millionths, from Workers AI's price property. */
117function listedPrice(value: unknown): ProviderModel["price"] {
118 if (!Array.isArray(value)) return null;
119 let input = 0;
120 let output = 0;
121 for (const entry of value) {
122 if (!isObject(entry)) continue;
123 const unit = typeof entry.unit === "string" ? entry.unit.toLowerCase() : "";
124 const price = typeof entry.price === "number" ? entry.price : Number(entry.price);
125 if (!Number.isFinite(price) || price <= 0 || !unit.includes("per m")) continue;
126 if (unit.includes("input")) input = Math.round(price * 1_000_000);
127 else if (unit.includes("output")) output = Math.round(price * 1_000_000);
128 }
129 return input > 0 ? { inputMicros: input, outputMicros: output } : null;
130}
131
132/** One model from Workers AI's search, or null when it has no `@cf/…` name. */
133export function fromWorkersAi(item: unknown): ProviderModel | null {
134 if (!isObject(item) || typeof item.name !== "string" || !item.name.startsWith("@")) return null;
135 const properties = new Map<string, unknown>();
136 for (const property of Array.isArray(item.properties) ? item.properties : []) {
137 if (isObject(property) && typeof property.property_id === "string") properties.set(property.property_id, property.value);
138 }
139 const kind = workersAiKind(item.task);
140 const capabilities: string[] = [];
141 if (properties.get("function_calling") === "true" || properties.get("function_calling") === true) capabilities.push("tools");
142 if (kind === "embeddings") capabilities.push("embeddings");
143 return {
144 id: item.name,
145 name: item.name.split("/").pop() ?? item.name,
146 kind,
147 contextWindow: count(properties.get("context_window")),
148 maxOutput: 0,
149 capabilities,
150 price: listedPrice(properties.get("price")),
151 };
152}
153
154/** Every model Workers AI offers on g1t's account. Throws with what went wrong. */
155export async function listWorkersAi(env: HostedRouting, fetcher: Fetch): Promise<ProviderModel[]> {
156 const token = env.WORKERS_AI_TOKEN || env.AI_GATEWAY_TOKEN;
157 if (!token || !env.CLOUDFLARE_ACCOUNT_ID) throw new Error("No Cloudflare token with Workers AI Read: set WORKERS_AI_TOKEN on the models service.");
158 const models: ProviderModel[] = [];
159 for (let page = 1; page <= MAX_PAGES; page++) {
160 const url = `https://api.cloudflare.com/client/v4/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/ai/models/search?per_page=${WORKERS_AI_PAGE}&page=${page}`;
161 const answer = await fetcher(url, { headers: { authorization: `Bearer ${token}` } });
162 if (!answer.ok) throw new Error(`Workers AI's model search answered ${answer.status}: ${(await answer.text()).slice(0, 300)}`);
163 const body = (await answer.json()) as Json;
164 const result = Array.isArray(body.result) ? body.result : [];
165 for (const item of result) {
166 const model = fromWorkersAi(item);
167 if (model) models.push(model);
168 }
169 if (result.length < WORKERS_AI_PAGE) return models;
170 }
171 return models;
172}
173
174/** The providers listed, each with how. */
175export const PROVIDERS: { provider: string; list: (env: HostedRouting, fetcher: Fetch) => Promise<ProviderModel[]> }[] = [
176 { provider: "anthropic", list: listAnthropic },
177 { provider: "workers-ai", list: listWorkersAi },
178];
179
180/** Where each provider's list goes: billing's `record_discovery`. */
181export type Recorder = (provider: string, models: ProviderModel[], by: string, error: string | null) => Promise<DiscoveryResult>;
182
183/**
184 * Lists every provider and records each list with billing, one at a time.
185 * A provider that cannot be listed is recorded as a failed check (nothing
186 * in the catalogue changes); the others still are.
187 */
188export async function discover(env: HostedRouting, fetcher: Fetch, record: Recorder, by: string): Promise<DiscoveryResult[]> {
189 const who = by.trim().slice(0, 200) || "schedule";
190 const results: DiscoveryResult[] = [];
191 for (const { provider, list } of PROVIDERS) {
192 let models: ProviderModel[] = [];
193 let error: string | null = null;
194 try {
195 models = await list(env, fetcher);
196 if (models.length === 0) error = "The list was empty.";
197 } catch (failure) {
198 error = failure instanceof Error ? failure.message : String(failure);
199 }
200 // Never a key in what is recorded.
201 for (const secret of [env.AI_GATEWAY_TOKEN, env.ANTHROPIC_API_KEY, env.WORKERS_AI_TOKEN]) {
202 if (secret && error) error = error.split(secret).join("[secret]");
203 }
204 results.push(await record(provider, error ? [] : models, who, error));
205 }
206 return results;
207}