Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/models/src/discover.test.ts

123 lines6,334 bytesCodeBlame
1import assert from "node:assert/strict";
2import { test } from "node:test";
3
4import type { DiscoveryResult, ProviderModel } from "@g1t/contracts";
5
6import { type Fetch, anthropicRequest, discover, fromAnthropic, fromWorkersAi, listAnthropic, listWorkersAi } from "./discover.ts";
7
8const hosted = { AI_GATEWAY_ID: "g1t", CLOUDFLARE_ACCOUNT_ID: "acct", AI_GATEWAY_TOKEN: "aig-secret", WORKERS_AI_TOKEN: "wai-secret" };
9
10function answering(pages: Record<string, unknown>, seen: { url: string; headers: Headers }[] = []): Fetch {
11 return async (url, init) => {
12 seen.push({ url, headers: new Headers(init?.headers) });
13 const key = Object.keys(pages).find((prefix) => url.includes(prefix));
14 if (!key) return new Response("not found", { status: 404 });
15 return Response.json(pages[key]);
16 };
17}
18
19test("Anthropic's list is read through g1t's gateway, with its token and never a model call", () => {
20 const request = anthropicRequest(hosted, null)!;
21 assert.equal(request.url, "https://gateway.ai.cloudflare.com/v1/acct/g1t/anthropic/v1/models?limit=1000");
22 assert.equal(request.headers.get("cf-aig-authorization"), "Bearer aig-secret");
23 assert.equal(request.headers.get("anthropic-version"), "2023-06-01");
24 assert.equal(anthropicRequest(hosted, "claude-x")!.url.endsWith("&after_id=claude-x"), true);
25 // Without a gateway, straight to Anthropic with g1t's key; with neither, no way.
26 assert.equal(anthropicRequest({ AI_GATEWAY_ID: "", CLOUDFLARE_ACCOUNT_ID: "acct", ANTHROPIC_API_KEY: "sk" }, null)!.url, "https://api.anthropic.com/v1/models?limit=1000");
27 assert.equal(anthropicRequest({ AI_GATEWAY_ID: "", CLOUDFLARE_ACCOUNT_ID: "acct" }, null), null);
28});
29
30test("an Anthropic model's context window and capabilities come from its list entry", () => {
31 const model = fromAnthropic({
32 id: "claude-haiku-6",
33 display_name: "Claude Haiku 6",
34 max_input_tokens: 400000,
35 max_tokens: 64000,
36 capabilities: { effort: { supported: true }, thinking: { supported: true }, image_input: { supported: false } },
37 });
38 assert.deepEqual(model, {
39 id: "claude-haiku-6",
40 name: "Claude Haiku 6",
41 kind: "chat",
42 contextWindow: 400000,
43 maxOutput: 64000,
44 capabilities: ["effort", "thinking", "tools"],
45 price: null,
46 });
47 assert.equal(fromAnthropic({ display_name: "No id" }), null);
48 // An older list with no capability tree: nothing claimed.
49 assert.deepEqual(fromAnthropic({ id: "claude-haiku-4-5-20251001" })!.capabilities, []);
50});
51
52test("Anthropic's list is read page by page", async () => {
53 const seen: { url: string; headers: Headers }[] = [];
54 const fetcher: Fetch = async (url, init) => {
55 seen.push({ url, headers: new Headers(init?.headers) });
56 return url.includes("after_id=claude-b")
57 ? Response.json({ data: [{ id: "claude-c" }], has_more: false, last_id: "claude-c" })
58 : Response.json({ data: [{ id: "claude-a" }, { id: "claude-b" }], has_more: true, last_id: "claude-b" });
59 };
60 const models = await listAnthropic(hosted, fetcher);
61 assert.deepEqual(models.map((m) => m.id), ["claude-a", "claude-b", "claude-c"]);
62 assert.equal(seen.length, 2);
63 await assert.rejects(listAnthropic(hosted, async () => new Response("nope", { status: 401 })), /answered 401/);
64});
65
66test("a Workers AI model's kind, context and price come from its search entry", () => {
67 const chat = fromWorkersAi({
68 name: "@cf/meta/llama-5-70b",
69 task: { name: "Text Generation" },
70 properties: [
71 { property_id: "context_window", value: "131072" },
72 { property_id: "function_calling", value: "true" },
73 { property_id: "price", value: [{ unit: "per M input tokens", price: 0.29 }, { unit: "per M output tokens", price: 2.25 }] },
74 ],
75 })!;
76 assert.deepEqual(chat, {
77 id: "@cf/meta/llama-5-70b",
78 name: "llama-5-70b",
79 kind: "chat",
80 contextWindow: 131072,
81 maxOutput: 0,
82 capabilities: ["tools"],
83 price: { inputMicros: 290000, outputMicros: 2250000 },
84 });
85 const embed = fromWorkersAi({ name: "@cf/baai/bge-large-en-v1.5", task: { name: "Text Embeddings" }, properties: [] })!;
86 assert.equal(embed.kind, "embeddings");
87 assert.deepEqual(embed.capabilities, ["embeddings"]);
88 assert.equal(embed.price, null);
89 assert.equal(fromWorkersAi({ name: "@cf/openai/whisper", task: { name: "Automatic Speech Recognition" } })!.kind, "automatic-speech-recognition");
90});
91
92test("Workers AI's search is read with the Workers AI token", async () => {
93 const seen: { url: string; headers: Headers }[] = [];
94 const models = await listWorkersAi(
95 hosted,
96 answering({ "ai/models/search": { result: [{ name: "@cf/openai/gpt-oss-120b", task: { name: "Text Generation" } }] } }, seen),
97 );
98 assert.deepEqual(models.map((m) => m.id), ["@cf/openai/gpt-oss-120b"]);
99 assert.equal(seen[0].url, "https://api.cloudflare.com/client/v4/accounts/acct/ai/models/search?per_page=100&page=1");
100 assert.equal(seen[0].headers.get("authorization"), "Bearer wai-secret");
101});
102
103test("every provider is recorded, a failed one as a failed check without its key", async () => {
104 const recorded: { provider: string; models: ProviderModel[]; by: string; error: string | null }[] = [];
105 const record = async (provider: string, models: ProviderModel[], by: string, error: string | null): Promise<DiscoveryResult> => {
106 recorded.push({ provider, models, by, error });
107 return { provider, checkedAt: "", by, listed: models.length, added: [], deprecated: [], restored: [], error };
108 };
109 const fetcher: Fetch = async (url) =>
110 url.includes("anthropic")
111 ? Response.json({ data: [{ id: "claude-haiku-5-5" }], has_more: false })
112 : new Response("token wai-secret is not allowed", { status: 403 });
113 const results = await discover(hosted, fetcher, record, "staff@g1t.sh");
114 assert.deepEqual(results.map((r) => [r.provider, r.listed]), [["anthropic", 1], ["workers-ai", 0]]);
115 assert.equal(recorded[0].by, "staff@g1t.sh");
116 assert.equal(recorded[0].error, null);
117 assert.match(recorded[1].error!, /answered 403/);
118 assert.doesNotMatch(recorded[1].error!, /wai-secret/);
119 // An empty list is a failed check, never every model gone.
120 await discover(hosted, async () => Response.json({ data: [], result: [] }), record, "");
121 assert.equal(recorded[2].error, "The list was empty.");
122 assert.equal(recorded[2].by, "schedule");
123});