A file's blame is fetched by its page, so a slow first one still arrives. 834f2998 streamed blame from the blob loader, but the site keeps a page's stream open for 5 seconds (entry.server.tsx streamTimeout), and a first blame can take up to about 6 (repos' walk is cut at 6 s): live on flagon-io/g1t's services/actions/src/plan.rs at 834f2998 the stream ended at 5.7 s with the blame cut off, the browser logged Unexpected Server Error, and the page said "Working out who changed each line" for good. The page now opens with the file and fetches the blame from /<owner>/<repo>/blame/<ref>/<path> (routes/repo/blame-data.ts, JSON, private and noindex), on a request of its own with no stream to outlast; it says the blame could not be worked out when the request fails. robots.txt keeps crawlers off that path too (Disallow: /*/*/blame/). Checked: the site's typecheck (tsc -b --force) and unit tests (950 passing, the robots test with the new rule).
- G1t-Session:
- ses_01m4y12gdde2r9q832m7xk87kd
6 files+72−240/6 viewed
| 101 | 101 | fills in when it is ready. A file's blame at a commit is kept once worked | |
| 102 | 102 | out, so opening it again, or from a commit's link, shows at once. Search | |
| 103 | 103 | engines and other crawlers are asked to leave blame pages alone | |
| 104 | − | (`Disallow: /*?blame=` in `/robots.txt`); the file itself stays open to | |
| 105 | − | them. | |
| 104 | + | (`Disallow: /*?blame=`, and `/*/*/blame/` for the blame data the view | |
| 105 | + | fetches, in `/robots.txt`); the file itself stays open to them. | |
| 106 | 106 | ||
| 107 | 107 | ### Why over the API | |
| 108 | 108 |
| 8 | 8 | assert.match(text, /^User-agent: \*\nAllow: \/\n/); | |
| 9 | 9 | assert.match(text, /^Disallow: \/\*\?blame=$/m); | |
| 10 | 10 | assert.match(text, /^Disallow: \/\*&blame=$/m); | |
| 11 | + | assert.match(text, /^Disallow: \/\*\/\*\/blame\/$/m); | |
| 11 | 12 | assert.match(text, /^Sitemap: https:\/\/g1t\.example\/sitemap\.xml$/m); | |
| 12 | 13 | assert.doesNotMatch(text, /docs\.g1t\.sh/, "a self-hosted g1t has no docs site"); | |
| 13 | 14 | assert.ok(text.endsWith("\n")); |
| 1 | 1 | /** | |
| 2 | 2 | * What /robots.txt says: everything may be crawled but a file's blame | |
| 3 | − | * (`?blame=1`), and where the site's maps are. | |
| 3 | + | * (`?blame=1`, and the `/<owner>/<repo>/blame/…` data it fetches), and | |
| 4 | + | * where the site's maps are. | |
| 4 | 5 | * | |
| 5 | 6 | * Blame is the costliest file view to work out, one per file per commit a | |
| 6 | 7 | * crawler finds, and adds nothing an index needs that the file itself does | |
| ⋯ | |||
| 13 | 14 | const DOCS_SITEMAP = "https://docs.g1t.sh/sitemap-index.xml"; | |
| 14 | 15 | ||
| 15 | 16 | /** Paths crawlers are kept from, wherever the query puts `blame`. */ | |
| 16 | − | export const DISALLOWED = ["/*?blame=", "/*&blame="]; | |
| 17 | + | export const DISALLOWED = ["/*?blame=", "/*&blame=", "/*/*/blame/"]; | |
| 17 | 18 | ||
| 18 | 19 | export function robotsTxt(site: string): string { | |
| 19 | 20 | const lines = ["User-agent: *", "Allow: /", ...DISALLOWED.map((path) => `Disallow: ${path}`), "", `Sitemap: ${site}/sitemap.xml`]; | |
| 248 | 248 | route(":owner/:repo/production-elsewhere.jpg", "routes/repo/production-elsewhere.ts"), | |
| 249 | 249 | // A file as it is, sent on to the usercontent origin (lib/usercontent.ts). | |
| 250 | 250 | route(":owner/:repo/raw/:ref/*", "routes/repo/raw.ts"), | |
| 251 | + | // A file's blame as JSON, fetched by its blame view (routes/repo/blob.tsx). | |
| 252 | + | route(":owner/:repo/blame/:ref/*", "routes/repo/blame-data.ts"), | |
| 251 | 253 | // A project: its overview first, its repository's code under Code. The | |
| 252 | 254 | // 1:1 project of a repository has the repository's name, so every | |
| 253 | 255 | // repository address below keeps working. |
| 1 | + | /** | |
| 2 | + | * A file's blame at a ref, as JSON, for the blame view to fetch once the | |
| 3 | + | * file is on screen: `/<owner>/<repo>/blame/<ref>/<path>`. A first blame | |
| 4 | + | * walks the file's history in repos and can take longer than the page's | |
| 5 | + | * stream may stay open (entry.server.tsx `streamTimeout`), so it is asked | |
| 6 | + | * for on its own request instead; once worked out, repos keeps it and it | |
| 7 | + | * comes back at once. Kept out of indexes like the blame view | |
| 8 | + | * (lib/robots.ts). | |
| 9 | + | */ | |
| 10 | + | import type { Route } from "./+types/blame-data"; | |
| 11 | + | import { showCommits } from "../../lib/commit-people.server"; | |
| 12 | + | import { repos } from "../../lib/services.server"; | |
| 13 | + | import { getViewer } from "../../lib/session.server"; | |
| 14 | + | ||
| 15 | + | const HEADERS = { | |
| 16 | + | "cache-control": "private, no-store", | |
| 17 | + | "x-robots-tag": "noindex, nofollow", | |
| 18 | + | }; | |
| 19 | + | ||
| 20 | + | export async function loader({ params, context }: Route.LoaderArgs) { | |
| 21 | + | const viewer = getViewer(context); | |
| 22 | + | const path = { namespace: params.owner, name: params.repo }; | |
| 23 | + | const file = params["*"] ?? ""; | |
| 24 | + | const found = await repos.blame(path, viewer, params.ref, file).catch(() => null); | |
| 25 | + | if (!found?.ok) { | |
| 26 | + | const status = found?.error.code === "not_found" ? 404 : 502; | |
| 27 | + | return Response.json({ blame: null }, { status, headers: HEADERS }); | |
| 28 | + | } | |
| 29 | + | // Every commit's people at once. | |
| 30 | + | const blame = { ...found.value, commits: await showCommits(found.value.commits) }; | |
| 31 | + | return Response.json({ blame }, { headers: HEADERS }); | |
| 32 | + | } |
| 1 | 1 | import { isCodeownersPath } from "@g1t/contracts"; | |
| 2 | + | import { useEffect, useState } from "react"; | |
| 2 | 3 | ||
| 3 | 4 | import type { Route } from "./+types/blob"; | |
| 4 | 5 | import { page } from "../../lib/meta"; | |
| 5 | 6 | import { CodeownersFileErrors } from "../../components/codeowners"; | |
| 6 | 7 | import { BlobView } from "../../components/repo-view"; | |
| 7 | 8 | import type { ShownBlame } from "../../lib/commit-people"; | |
| 8 | − | import { useArrived } from "../../lib/use-arrived"; | |
| 9 | 9 | import { highlightLines } from "../../lib/highlight.server"; | |
| 10 | 10 | import { repos, work } from "../../lib/services.server"; | |
| 11 | − | import { showCommits } from "../../lib/commit-people.server"; | |
| 12 | 11 | import { redirectIfBranchRenamed } from "../../lib/branch-redirect.server"; | |
| 13 | 12 | import { getViewer, unwrap } from "../../lib/session.server"; | |
| 14 | 13 | ||
| ⋯ | |||
| 24 | 23 | const path = { namespace: params.owner, name: params.repo }; | |
| 25 | 24 | const file = params["*"] ?? ""; | |
| 26 | 25 | const wantsBlame = new URL(request.url).searchParams.has("blame"); | |
| 27 | − | // A first blame can take seconds (repos walks the file's history), so it | |
| 28 | − | // streams in after the file; a later one comes from repos' cache at once. | |
| 29 | − | const blameFound = wantsBlame ? repos.blame(path, viewer, params.ref, file) : null; | |
| 30 | − | blameFound?.catch(() => null); | |
| 31 | 26 | const [found, list, checked] = await Promise.all([ | |
| 32 | 27 | repos.blob(path, viewer, params.ref, file), | |
| 33 | 28 | // For the branch menu; the page still shows without it. | |
| ⋯ | |||
| 41 | 36 | if (!found.ok && found.error.code === "not_found") await redirectIfBranchRenamed(request, path, viewer, params.ref); | |
| 42 | 37 | const blob = unwrap(found); | |
| 43 | 38 | const html = blob.text == null ? null : await highlightLines(blob.path, blob.text); | |
| 44 | − | // Each line's commit, with every commit's people at once; null when it | |
| 45 | − | // could not be worked out, and the file shows as it is. | |
| 46 | − | const blame: Promise<ShownBlame | null> | null = | |
| 47 | − | blameFound && blob.text != null | |
| 48 | − | ? blameFound | |
| 49 | − | .then(async (answered) => (answered.ok ? { ...answered.value, commits: await showCommits(answered.value.commits) } : null)) | |
| 50 | − | .catch(() => null) | |
| 51 | − | : null; | |
| 52 | − | return { blob, html, blame, branches, codeowners }; | |
| 39 | + | // The blame itself is fetched by the page once the file shows | |
| 40 | + | // (routes/repo/blame-data.ts): a first one can take longer than the | |
| 41 | + | // page's stream may stay open. | |
| 42 | + | return { blob, html, wantsBlame: wantsBlame && blob.text != null, branches, codeowners }; | |
| 53 | 43 | } | |
| 54 | 44 | ||
| 55 | 45 | export default function Blob({ loaderData, params }: Route.ComponentProps) { | |
| 56 | − | const { codeowners, blob, html } = loaderData; | |
| 46 | + | const { codeowners, blob, html, wantsBlame } = loaderData; | |
| 57 | 47 | // Its errors' lines are marked, when they are this file's. | |
| 58 | 48 | const own = codeowners != null && codeowners.path === blob.path; | |
| 59 | − | // undefined while the blame is on its way, null when it could not be had. | |
| 60 | − | const blame = useArrived(loaderData.blame ?? NO_BLAME, undefined); | |
| 49 | + | const blame = useBlame(wantsBlame ? `/${params.owner}/${params.repo}/blame/${encodeURIComponent(blob.ref)}/${encodePath(blob.path)}` : null); | |
| 61 | 50 | return ( | |
| 62 | 51 | <BlobView | |
| 63 | 52 | blob={blob} | |
| 64 | 53 | html={html} | |
| 65 | − | blame={loaderData.blame == null ? null : blame === undefined ? "waiting" : blame === null ? "failed" : { blame, lines: html }} | |
| 54 | + | blame={blame === null ? null : typeof blame === "string" ? blame : { blame, lines: html }} | |
| 66 | 55 | branches={loaderData.branches} | |
| 67 | 56 | notice={ | |
| 68 | 57 | codeowners && ( | |
| ⋯ | |||
| 74 | 63 | ); | |
| 75 | 64 | } | |
| 76 | 65 | ||
| 77 | − | const NO_BLAME: Promise<null> = Promise.resolve(null); | |
| 66 | + | /** The blame at `url`: "waiting" until it comes, "failed" when it could not | |
| 67 | + | * be had, null when none was asked for. */ | |
| 68 | + | function useBlame(url: string | null): ShownBlame | "waiting" | "failed" | null { | |
| 69 | + | const [answered, setAnswered] = useState<{ url: string; blame: ShownBlame | "failed" } | null>(null); | |
| 70 | + | useEffect(() => { | |
| 71 | + | if (!url) return; | |
| 72 | + | const stop = new AbortController(); | |
| 73 | + | fetch(url, { signal: stop.signal, headers: { accept: "application/json" } }) | |
| 74 | + | .then(async (response) => { | |
| 75 | + | const body = (await response.json()) as { blame: ShownBlame | null }; | |
| 76 | + | setAnswered({ url, blame: response.ok && body.blame ? body.blame : "failed" }); | |
| 77 | + | }) | |
| 78 | + | .catch(() => { | |
| 79 | + | if (!stop.signal.aborted) setAnswered({ url, blame: "failed" }); | |
| 80 | + | }); | |
| 81 | + | return () => stop.abort(); | |
| 82 | + | }, [url]); | |
| 83 | + | if (!url) return null; | |
| 84 | + | return answered?.url === url ? answered.blame : "waiting"; | |
| 85 | + | } | |
| 86 | + | ||
| 87 | + | function encodePath(path: string): string { | |
| 88 | + | return path.split("/").map(encodeURIComponent).join("/"); | |
| 89 | + | } | |
Comments
Sign in to comment.