g1t/services/context/src/scorecards.ts
| 1 | /** |
| 2 | * Scorecards: a few rules every project should meet, each checked from the |
| 3 | * catalog and the project's deployments, and each failing one ready to |
| 4 | * become an issue an agent can fix. Pure: given what is known, the same |
| 5 | * card. |
| 6 | */ |
| 7 | |
| 8 | import type { RuleResult, ScoreRule } from "@g1t/contracts"; |
| 9 | |
| 10 | export type CardInput = { |
| 11 | name: string; |
| 12 | /** Owners named in its files or found among its authors. */ |
| 13 | owners: string[]; |
| 14 | /** The paths of its docs, relative to its root. */ |
| 15 | docs: string[]; |
| 16 | /** Whether one of its workflows runs tests. */ |
| 17 | tests: boolean; |
| 18 | /** Its test command, when its manifests say. */ |
| 19 | testCommand: string | null; |
| 20 | deploy: { |
| 21 | enabled: boolean; |
| 22 | production: { url: string } | null; |
| 23 | latest: { kind: string; status: string; error: string | null } | null; |
| 24 | } | null; |
| 25 | /** Open secret findings, or null when nothing reports them. */ |
| 26 | secretFindings: number | null; |
| 27 | }; |
| 28 | |
| 29 | export const RULE_TITLES: Record<ScoreRule, string> = { |
| 30 | has_owner: "Has an owner", |
| 31 | has_readme: "Has a README", |
| 32 | has_agents_md: "Has an AGENTS.md", |
| 33 | tests_in_ci: "Tests run in checks", |
| 34 | production_green: "Production deploy is green", |
| 35 | no_secret_findings: "No open secret findings", |
| 36 | }; |
| 37 | |
| 38 | const isReadme = (path: string) => /^readme(\.(md|markdown|txt))?$/i.test(path); |
| 39 | const isAgents = (path: string) => /^(agents|claude)\.md$/i.test(path); |
| 40 | |
| 41 | export function evaluate(card: CardInput): RuleResult[] { |
| 42 | const result = (rule: ScoreRule, status: RuleResult["status"], detail: string, fix: RuleResult["fix"] = null): RuleResult => ({ |
| 43 | rule, |
| 44 | title: RULE_TITLES[rule], |
| 45 | status, |
| 46 | detail, |
| 47 | fix: status === "fail" ? fix : null, |
| 48 | }); |
| 49 | const rules: RuleResult[] = []; |
| 50 | |
| 51 | rules.push( |
| 52 | card.owners.length |
| 53 | ? result("has_owner", "pass", `Owned by ${card.owners.join(", ")}.`) |
| 54 | : result("has_owner", "fail", "No owner is named, and no member wrote most of it.", { |
| 55 | title: `Name the owners of ${card.name}`, |
| 56 | body: [ |
| 57 | `${card.name} has no owner in g1t's catalog. Add an \`owners\` list to \`.g1t/project.yml\` (create it if it is missing) naming the workspace members who own this project, for example:`, |
| 58 | "", |
| 59 | "```yaml", |
| 60 | "owners:", |
| 61 | " - ana", |
| 62 | "```", |
| 63 | "", |
| 64 | "Pick the people from the history of the project: who wrote and reviewed most of it recently. Keep any `dependsOn` the file already has.", |
| 65 | ].join("\n"), |
| 66 | checks: ["grep -q '^owners:' .g1t/project.yml"], |
| 67 | }), |
| 68 | ); |
| 69 | |
| 70 | rules.push( |
| 71 | card.docs.some(isReadme) |
| 72 | ? result("has_readme", "pass", "It has a README.") |
| 73 | : result("has_readme", "fail", "There is no README at its root.", { |
| 74 | title: `Write a README for ${card.name}`, |
| 75 | body: `${card.name} has no README. Write \`README.md\` at its root: what it is and who uses it, how to run it locally, how to test it, and how it is deployed. Take the commands from its manifests and workflows, not from guesses.`, |
| 76 | checks: ["test -f README.md"], |
| 77 | }), |
| 78 | ); |
| 79 | |
| 80 | rules.push( |
| 81 | card.docs.some(isAgents) |
| 82 | ? result("has_agents_md", "pass", "It has instructions for agents.") |
| 83 | : result("has_agents_md", "fail", "There is no AGENTS.md telling agents how to work here.", { |
| 84 | title: `Add an AGENTS.md to ${card.name}`, |
| 85 | body: `${card.name} has no AGENTS.md. Write one at its root for agents working here: how to build and test (exact commands), the conventions the code follows, where things live, and what not to touch. Keep each point to one line; g1t keeps what it says as memory.`, |
| 86 | checks: ["test -f AGENTS.md"], |
| 87 | }), |
| 88 | ); |
| 89 | |
| 90 | rules.push( |
| 91 | card.tests |
| 92 | ? result("tests_in_ci", "pass", "A workflow runs its tests.") |
| 93 | : result("tests_in_ci", "fail", "No workflow in .g1t/workflows or .github/workflows runs tests.", { |
| 94 | title: `Run ${card.name}'s tests on every push`, |
| 95 | body: `Add a workflow under \`.g1t/workflows/\` that runs ${card.name}'s tests on every push and pull request${ |
| 96 | card.testCommand ? ` (\`${card.testCommand}\`)` : "" |
| 97 | }. If it has no tests yet, add a first meaningful one with it.`, |
| 98 | checks: ["grep -rqsiE 'test' .g1t/workflows .github/workflows"], |
| 99 | }), |
| 100 | ); |
| 101 | |
| 102 | const deploy = card.deploy; |
| 103 | if (!deploy?.enabled) { |
| 104 | rules.push(result("production_green", "na", "It does not deploy on g1t.")); |
| 105 | } else if (deploy.latest?.kind === "production" && deploy.latest.status === "failed") { |
| 106 | rules.push( |
| 107 | result("production_green", "fail", `The latest production build failed${deploy.latest.error ? `: ${deploy.latest.error}` : "."}`, { |
| 108 | title: `Fix ${card.name}'s production build`, |
| 109 | body: `The latest production build of ${card.name} failed${deploy.latest.error ? ` with:\n\n\`\`\`\n${deploy.latest.error}\n\`\`\`\n` : "."} Find out why from the build's log on the project's Deployments page, fix it, and make sure the build passes.`, |
| 110 | checks: [], |
| 111 | }), |
| 112 | ); |
| 113 | } else if (!deploy.production) { |
| 114 | rules.push(result("production_green", "fail", "Deployments are on, but production is not live.", { |
| 115 | title: `Get ${card.name} live in production`, |
| 116 | body: `Deployments are on for ${card.name}, but nothing is live in production. Make the default branch build and serve, then check the project's Deployments page.`, |
| 117 | checks: [], |
| 118 | })); |
| 119 | } else { |
| 120 | rules.push(result("production_green", "pass", `Live at ${deploy.production.url}.`)); |
| 121 | } |
| 122 | |
| 123 | rules.push( |
| 124 | card.secretFindings == null |
| 125 | ? result("no_secret_findings", "na", "Secret scanning has not reported on it yet.") |
| 126 | : card.secretFindings === 0 |
| 127 | ? result("no_secret_findings", "pass", "No open secret findings.") |
| 128 | : result("no_secret_findings", "fail", `${card.secretFindings} open secret finding${card.secretFindings === 1 ? "" : "s"}.`, { |
| 129 | title: `Remove the secrets found in ${card.name}`, |
| 130 | body: `Secret scanning found ${card.secretFindings} secret${card.secretFindings === 1 ? "" : "s"} in ${card.name}. Take each out of the code and read it from a secret instead (Settings, Secrets and variables). Say in the pull request which credentials must be rotated; do not paste them.`, |
| 131 | checks: [], |
| 132 | }), |
| 133 | ); |
| 134 | return rules; |
| 135 | } |