Skip to content
Sign up

flagon-io/g1t

public
0
247 linesCodeBlameRaw
1/**
2 * Reading marketplaces and plugins from where they live
3 * (docs.g1t.sh/guides/customize/, "Where plugins come from"): a GitHub
4 * repository (public, by its API for the tree and raw files for content),
5 * a repository on this g1t (the repos service), a direct link to a
6 * `marketplace.json`, or a zip archive. Every read is bounded: at most
7 * `MAX_PLUGIN_FILES` files and `MAX_PLUGIN_BYTES` together, and a file over
8 * `MAX_FILE_BYTES` is left out and named, since Claude Code on a runner
9 * fetches the plugin whole for itself anyway.
10 *
11 * Pure apart from `fetch` and the repos ports, so it is tested with fakes.
12 */
13import type { FetchPlan, MarketplaceSource } from "../../../packages/contracts/src/claude-plugin.ts";
14import type { SkillFile } from "../../../packages/contracts/src/skill-format.ts";
15import { asSkillFile, unzip } from "./skill-zip.ts";
16
17/** The most a plugin read here holds. */
18export const MAX_PLUGIN_FILES = 2000;
19export const MAX_PLUGIN_BYTES = 24 * 1024 * 1024;
20/** A file larger than this is listed but not read. */
21export const MAX_FILE_BYTES = 2 * 1024 * 1024;
22/** How long one fetch may take. */
23const TIMEOUT_MS = 20_000;
24/** Raw files fetched at once. */
25const PARALLEL = 16;
26
27export type Fetch = (url: string, init?: RequestInit) => Promise<Response>;
28
29/** The repos service, as the fetcher needs it: a repository the viewer may read, its files at a ref, and blobs. */
30export type RepoPorts = {
31 repo(full: string): Promise<{ id: string; default_branch: string } | null>;
32 listFiles(repoId: string, ref: string | null): Promise<{ commit: string | null; files: { path: string; hash: string | null }[]; truncated: boolean }>;
33 blobs(repoId: string, hashes: string[]): Promise<{ hash: string; data: string | null }[]>;
34};
35
36export type FetchDeps = {
37 fetchFn?: Fetch;
38 /** A GitHub token for the API (optional: the API allows a few reads an hour without one). */
39 githubToken?: string | null;
40 repos?: RepoPorts | null;
41};
42
43export type Fetched = { ok: true; files: SkillFile[]; sha: string | null; omitted: string[] } | { ok: false; message: string };
44
45const fail = (message: string): { ok: false; message: string } => ({ ok: false, message });
46
47async function get(fetchFn: Fetch, url: string, headers: Record<string, string> = {}): Promise<Response | null> {
48 try {
49 return await fetchFn(url, { headers: { "user-agent": "g1t-customize", ...headers }, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "follow" });
50 } catch {
51 return null;
52 }
53}
54
55function githubHeaders(token: string | null | undefined): Record<string, string> {
56 return { accept: "application/vnd.github+json", "x-github-api-version": "2022-11-28", ...(token ? { authorization: `Bearer ${token}` } : {}) };
57}
58
59// ── GitHub ───────────────────────────────────────────────────────────────
60
61/** The commit `ref` names (or the default branch's head), as a 40-character sha. */
62async function githubCommit(fetchFn: Fetch, token: string | null | undefined, repo: string, ref: string | null): Promise<{ ok: true; sha: string } | { ok: false; message: string }> {
63 const response = await get(fetchFn, `https://api.github.com/repos/${repo}/commits/${encodeURIComponent(ref ?? "HEAD")}`, githubHeaders(token));
64 if (!response) return fail(`GitHub couldn't be reached for ${repo}.`);
65 if (response.status === 404) return fail(`${repo}${ref ? ` at ${ref}` : ""} isn't on GitHub, or isn't public.`);
66 if (response.status === 403 || response.status === 429) return fail("GitHub is limiting how often g1t may read right now. Try again in a little while.");
67 if (!response.ok) return fail(`GitHub answered ${response.status} for ${repo}.`);
68 const body = (await response.json().catch(() => null)) as { sha?: string } | null;
69 return body?.sha && /^[0-9a-f]{40}$/.test(body.sha) ? { ok: true, sha: body.sha } : fail(`GitHub didn't say which commit ${repo} is at.`);
70}
71
72/** Every file under `path` at `sha`, from the repository's tree. */
73async function githubTree(fetchFn: Fetch, token: string | null | undefined, repo: string, sha: string, path: string): Promise<{ ok: true; files: { path: string; size: number }[]; truncated: boolean } | { ok: false; message: string }> {
74 const response = await get(fetchFn, `https://api.github.com/repos/${repo}/git/trees/${sha}?recursive=1`, githubHeaders(token));
75 if (!response) return fail(`GitHub couldn't be reached for ${repo}.`);
76 if (response.status === 403 || response.status === 429) return fail("GitHub is limiting how often g1t may read right now. Try again in a little while.");
77 if (!response.ok) return fail(`GitHub answered ${response.status} for ${repo}'s files.`);
78 const body = (await response.json().catch(() => null)) as { tree?: { path: string; type: string; size?: number }[]; truncated?: boolean } | null;
79 if (!body?.tree) return fail(`GitHub didn't list ${repo}'s files.`);
80 const prefix = path ? `${path.replace(/\/+$/, "")}/` : "";
81 const files = body.tree.filter((e) => e.type === "blob" && e.path.startsWith(prefix)).map((e) => ({ path: e.path.slice(prefix.length), size: e.size ?? 0 }));
82 return { ok: true, files, truncated: !!body.truncated };
83}
84
85/** Raw files from GitHub, in parallel, each as text or base64. */
86async function githubFiles(fetchFn: Fetch, repo: string, sha: string, base: string, files: { path: string; size: number }[]): Promise<{ ok: true; files: SkillFile[]; omitted: string[] } | { ok: false; message: string }> {
87 const out: SkillFile[] = [];
88 const omitted: string[] = [];
89 const prefix = base ? `${base}/` : "";
90 const wanted = files.filter((f) => (f.size <= MAX_FILE_BYTES ? true : (omitted.push(f.path), false)));
91 for (let i = 0; i < wanted.length; i += PARALLEL) {
92 const batch = wanted.slice(i, i + PARALLEL);
93 const results = await Promise.all(
94 batch.map(async (file) => {
95 const response = await get(fetchFn, `https://raw.githubusercontent.com/${repo}/${sha}/${prefix}${file.path.split("/").map(encodeURIComponent).join("/")}`);
96 if (!response?.ok) return { file, bytes: null as Uint8Array | null };
97 return { file, bytes: new Uint8Array(await response.arrayBuffer()) };
98 }),
99 );
100 for (const { file, bytes } of results) {
101 if (!bytes) return fail(`${file.path} in ${repo} couldn't be read from GitHub.`);
102 out.push(asSkillFile(file.path, bytes));
103 }
104 }
105 return { ok: true, files: out, omitted };
106}
107
108// ── This g1t ─────────────────────────────────────────────────────────────
109
110function blobFile(path: string, data: string): SkillFile {
111 const binary = atob(data);
112 const bytes = new Uint8Array(binary.length);
113 for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
114 return asSkillFile(path, bytes);
115}
116
117async function g1tFiles(repos: RepoPorts, repo: string, ref: string | null, path: string): Promise<Fetched> {
118 const found = await repos.repo(repo).catch(() => null);
119 if (!found) return fail(`${repo} isn't a repository here that you can read.`);
120 const listing = await repos.listFiles(found.id, ref).catch(() => null);
121 if (!listing) return fail(`${repo}${ref ? ` at ${ref}` : ""} couldn't be read.`);
122 const prefix = path ? `${path.replace(/\/+$/, "")}/` : "";
123 const wanted = listing.files.filter((f): f is { path: string; hash: string } => !!f.hash && f.path.startsWith(prefix));
124 if (!wanted.length) return fail(`There are no files in ${path || "the top folder"} of ${repo}.`);
125 if (wanted.length > MAX_PLUGIN_FILES) return fail(`${path || repo} has ${wanted.length} files; a plugin holds at most ${MAX_PLUGIN_FILES}.`);
126 const data = new Map<string, string | null>();
127 const hashes = [...new Set(wanted.map((f) => f.hash))];
128 for (let i = 0; i < hashes.length; i += 100) for (const blob of await repos.blobs(found.id, hashes.slice(i, i + 100))) data.set(blob.hash, blob.data);
129 const files: SkillFile[] = [];
130 const omitted: string[] = [];
131 let bytes = 0;
132 for (const file of wanted) {
133 const content = data.get(file.hash);
134 const relative = file.path.slice(prefix.length);
135 if (content == null) {
136 omitted.push(relative);
137 continue;
138 }
139 const one = blobFile(relative, content);
140 bytes += one.encoding === "base64" ? Math.floor((one.content.length * 3) / 4) : new TextEncoder().encode(one.content).length;
141 if (bytes > MAX_PLUGIN_BYTES) return fail(`${path || repo} is larger than ${Math.round(MAX_PLUGIN_BYTES / 1024 / 1024)} MB, the most a plugin read here holds.`);
142 files.push(one);
143 }
144 return { ok: true, files, sha: listing.commit, omitted };
145}
146
147// ── Archives ─────────────────────────────────────────────────────────────
148
149async function archiveFiles(fetchFn: Fetch, url: string, sha256: string | null): Promise<Fetched> {
150 const response = await get(fetchFn, url);
151 if (!response?.ok) return fail(`The archive at ${url} couldn't be fetched${response ? ` (${response.status})` : ""}.`);
152 const bytes = new Uint8Array(await response.arrayBuffer());
153 if (bytes.length > MAX_PLUGIN_BYTES) return fail(`The archive is larger than ${Math.round(MAX_PLUGIN_BYTES / 1024 / 1024)} MB, the most a plugin read here holds.`);
154 if (sha256) {
155 const digest = [...new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))].map((b) => b.toString(16).padStart(2, "0")).join("");
156 if (digest !== sha256) return fail("The archive isn't the one the marketplace lists: its checksum differs.");
157 }
158 const unpacked = await unzip(bytes, { files: MAX_PLUGIN_FILES, bytes: MAX_PLUGIN_BYTES });
159 if (!unpacked.ok) return fail(unpacked.message);
160 return { ok: true, files: unpacked.files, sha: null, omitted: [] };
161}
162
163// ── Plugins ──────────────────────────────────────────────────────────────
164
165/** A plugin's files, read from where its plan says, with the commit they were read at when there is one. */
166export async function fetchPlugin(plan: FetchPlan, deps: FetchDeps = {}): Promise<Fetched> {
167 const fetchFn = deps.fetchFn ?? ((url, init) => fetch(url, init));
168 switch (plan.kind) {
169 case "unsupported":
170 return fail(plan.why);
171 case "archive":
172 return archiveFiles(fetchFn, plan.url, plan.sha256);
173 case "g1t":
174 if (!deps.repos) return fail("Repositories here can't be read just now.");
175 return g1tFiles(deps.repos, plan.repo, plan.ref, plan.path);
176 case "github": {
177 const sha = plan.sha ? { ok: true as const, sha: plan.sha } : await githubCommit(fetchFn, deps.githubToken, plan.repo, plan.ref);
178 if (!sha.ok) return sha;
179 const tree = await githubTree(fetchFn, deps.githubToken, plan.repo, sha.sha, plan.path);
180 if (!tree.ok) return tree;
181 if (!tree.files.length) return fail(`There are no files in ${plan.path || "the top folder"} of ${plan.repo}.`);
182 if (tree.files.length > MAX_PLUGIN_FILES) return fail(`${plan.path || plan.repo} has ${tree.files.length} files; a plugin holds at most ${MAX_PLUGIN_FILES}.`);
183 const total = tree.files.reduce((n, f) => n + Math.min(f.size, MAX_FILE_BYTES), 0);
184 if (total > MAX_PLUGIN_BYTES) return fail(`${plan.path || plan.repo} is larger than ${Math.round(MAX_PLUGIN_BYTES / 1024 / 1024)} MB, the most a plugin read here holds.`);
185 const files = await githubFiles(fetchFn, plan.repo, sha.sha, plan.path, tree.files);
186 if (!files.ok) return files;
187 return { ok: true, files: files.files, sha: sha.sha, omitted: [...files.omitted, ...(tree.truncated ? ["(GitHub listed only part of a very large repository)"] : [])] };
188 }
189 }
190}
191
192// ── Marketplaces ─────────────────────────────────────────────────────────
193
194export type FetchedManifest = { ok: true; json: unknown; sha: string | null } | { ok: false; message: string };
195
196const MANIFEST_PATH = ".claude-plugin/marketplace.json";
197
198/** A marketplace's `marketplace.json`, read from where it is. */
199export async function fetchMarketplace(source: MarketplaceSource, deps: FetchDeps = {}, host: string | null = null): Promise<FetchedManifest> {
200 const fetchFn = deps.fetchFn ?? ((url, init) => fetch(url, init));
201 const parse = async (response: Response | null, where: string): Promise<FetchedManifest> => {
202 if (!response) return fail(`${where} couldn't be reached.`);
203 if (response.status === 404) return fail(`There is no ${MANIFEST_PATH} at ${where}.`);
204 if (!response.ok) return fail(`${where} answered ${response.status}.`);
205 const text = await response.text().catch(() => "");
206 if (text.length > 4 * 1024 * 1024) return fail("That marketplace.json is larger than 4 MB.");
207 try {
208 return { ok: true, json: JSON.parse(text), sha: null };
209 } catch {
210 return fail(`${where} didn't give JSON.`);
211 }
212 };
213 switch (source.source) {
214 case "builtin":
215 return fail("A built-in marketplace is read by its own source.");
216 case "url":
217 return parse(await get(fetchFn, source.url), source.url);
218 case "github": {
219 const read = await parse(await get(fetchFn, `https://raw.githubusercontent.com/${source.repo}/${encodeURIComponent(source.ref ?? "HEAD")}/${MANIFEST_PATH}`), `${source.repo} on GitHub`);
220 if (!read.ok) return read;
221 const sha = await githubCommit(fetchFn, deps.githubToken, source.repo, source.ref ?? null);
222 return { ...read, sha: sha.ok ? sha.sha : null };
223 }
224 case "git": {
225 const github = /github\.com[/:]([^/]+\/[^/]+?)(?:\.git)?\/?$/.exec(source.url);
226 if (github) return fetchMarketplace({ source: "github", repo: github[1]!, ...(source.ref ? { ref: source.ref } : {}) }, deps, host);
227 const own = host ? new RegExp(`^https?://(?:www\\.)?${host.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/([^/]+/[^/]+?)(?:\\.git)?/?$`, "i").exec(source.url) : null;
228 if (own) return fetchMarketplace({ source: "g1t", repo: own[1]!, ...(source.ref ? { ref: source.ref } : {}) }, deps, host);
229 return fail("g1t reads marketplaces from GitHub, from repositories here, and from a link to their marketplace.json. That git host isn't one of them yet.");
230 }
231 case "g1t": {
232 if (!deps.repos) return fail("Repositories here can't be read just now.");
233 const found = await deps.repos.repo(source.repo).catch(() => null);
234 if (!found) return fail(`${source.repo} isn't a repository here that you can read.`);
235 const listing = await deps.repos.listFiles(found.id, source.ref ?? null).catch(() => null);
236 const entry = listing?.files.find((f) => f.path === MANIFEST_PATH);
237 if (!listing || !entry?.hash) return fail(`There is no ${MANIFEST_PATH} in ${source.repo}.`);
238 const [blob] = await deps.repos.blobs(found.id, [entry.hash]);
239 if (!blob?.data) return fail(`${MANIFEST_PATH} in ${source.repo} couldn't be read.`);
240 try {
241 return { ok: true, json: JSON.parse(atob(blob.data)), sha: listing.commit };
242 } catch {
243 return fail(`${MANIFEST_PATH} in ${source.repo} isn't JSON.`);
244 }
245 }
246 }
247}