| 1 | /** |
| 2 | * What /robots.txt says: everything may be crawled but a file's blame |
| 3 | * (`?blame=1`, and the `/<owner>/<repo>/blame/…` data it fetches) and the |
| 4 | * cards over references (`/<owner>/<repo>/ref/…`), and where the site's |
| 5 | * maps are. |
| 6 | * |
| 7 | * Blame is the costliest file view to work out, one per file per commit a |
| 8 | * crawler finds, and adds nothing an index needs that the file itself does |
| 9 | * not have: on 2026-10-14 crawlers asked for 1,150 blame pages in four |
| 10 | * hours, against a handful from people. The links to it are `nofollow` too, |
| 11 | * and its page is `noindex` (routes/repo/blob.tsx). |
| 12 | */ |
| 13 | |
| 14 | /** g1t.sh's docs keep their own map; a self-hosted g1t has no docs site. */ |
| 15 | const DOCS_SITEMAP = "https://docs.g1t.sh/sitemap-index.xml"; |
| 16 | |
| 17 | /** Paths crawlers are kept from, wherever the query puts `blame`. */ |
| 18 | export const DISALLOWED = ["/*?blame=", "/*&blame=", "/*/*/blame/", "/*/*/ref/"]; |
| 19 | |
| 20 | export function robotsTxt(site: string): string { |
| 21 | const lines = ["User-agent: *", "Allow: /", ...DISALLOWED.map((path) => `Disallow: ${path}`), "", `Sitemap: ${site}/sitemap.xml`]; |
| 22 | if (site === "https://g1t.sh") lines.push(`Sitemap: ${DOCS_SITEMAP}`); |
| 23 | return `${lines.join("\n")}\n`; |
| 24 | } |