Skip to content
Sign up

flagon-io/g1t

public
0
522 linesCodeBlameRaw
1//! Which language each file is written in, and which files count towards a
2//! repository's languages: a small version of what people know from the
3//! language bar on code hosts. Programming and markup languages count;
4//! data (JSON, YAML) and prose (Markdown) do not, unless `.gitattributes`
5//! says `linguist-detectable`. Vendored, generated and documentation files
6//! never count, unless `.gitattributes` says otherwise:
7//!
8//! ```text
9//! third_party/** -linguist-vendored
10//! *.gen.ts linguist-generated
11//! docs/** -linguist-documentation
12//! *.inc linguist-language=PHP
13//! *.sql linguist-detectable
14//! ```
15
16use g1t_contracts::about::LanguageShare;
17
18/// What a language is, as it decides whether it counts by default.
19#[derive(Clone, Copy, Debug, PartialEq, Eq)]
20pub enum Kind {
21 Programming,
22 Markup,
23 Data,
24 Prose,
25}
26
27#[derive(Debug, PartialEq, Eq)]
28pub struct Language {
29 pub name: &'static str,
30 pub color: Option<&'static str>,
31 pub kind: Kind,
32}
33
34const fn lang(name: &'static str, color: &'static str, kind: Kind) -> Language {
35 Language { name, color: Some(color), kind }
36}
37
38use Kind::{Data, Markup, Programming, Prose};
39
40/// Every language g1t knows, with the colors people know them by.
41pub static LANGUAGES: &[Language] = &[
42 lang("Assembly", "#6E4C13", Programming),
43 lang("Astro", "#ff5a03", Markup),
44 lang("Batchfile", "#C1F12E", Programming),
45 lang("Blade", "#f7523f", Markup),
46 lang("C", "#555555", Programming),
47 lang("C#", "#178600", Programming),
48 lang("C++", "#f34b7d", Programming),
49 lang("Clojure", "#db5855", Programming),
50 lang("CMake", "#DA3434", Programming),
51 lang("Common Lisp", "#3fb68b", Programming),
52 lang("Crystal", "#000100", Programming),
53 lang("CSS", "#663399", Markup),
54 lang("Cuda", "#3A4E3A", Programming),
55 lang("Dart", "#00B4AB", Programming),
56 lang("Dockerfile", "#384d54", Programming),
57 lang("EJS", "#a91e50", Markup),
58 lang("Elixir", "#6e4a7e", Programming),
59 lang("Elm", "#60B5CC", Programming),
60 lang("Emacs Lisp", "#c065db", Programming),
61 lang("Erlang", "#B83998", Programming),
62 lang("F#", "#b845fc", Programming),
63 lang("Fortran", "#4d41b1", Programming),
64 lang("Gleam", "#ffaff3", Programming),
65 lang("GLSL", "#5686a5", Programming),
66 lang("Go", "#00ADD8", Programming),
67 lang("GraphQL", "#e10098", Data),
68 lang("Groovy", "#4298b8", Programming),
69 lang("Handlebars", "#f7931e", Markup),
70 lang("Haskell", "#5e5086", Programming),
71 lang("HCL", "#844FBA", Programming),
72 lang("HTML", "#e34c26", Markup),
73 lang("Java", "#b07219", Programming),
74 lang("JavaScript", "#f1e05a", Programming),
75 lang("JSON", "#292929", Data),
76 lang("Jsonnet", "#0064bd", Programming),
77 lang("Julia", "#a270ba", Programming),
78 lang("Jupyter Notebook", "#DA5B0B", Markup),
79 lang("Kotlin", "#A97BFF", Programming),
80 lang("Less", "#1d365d", Markup),
81 lang("Liquid", "#67b8de", Markup),
82 lang("Lua", "#000080", Programming),
83 lang("Makefile", "#427819", Programming),
84 lang("Markdown", "#083fa1", Prose),
85 lang("MDX", "#fcb32c", Markup),
86 lang("Nim", "#ffc200", Programming),
87 lang("Nix", "#7e7eff", Programming),
88 lang("Objective-C", "#438eff", Programming),
89 lang("Objective-C++", "#6866fb", Programming),
90 lang("OCaml", "#ef7a08", Programming),
91 lang("Perl", "#0298c3", Programming),
92 lang("PHP", "#4F5D95", Programming),
93 lang("PLpgSQL", "#336790", Programming),
94 lang("PowerShell", "#012456", Programming),
95 lang("Pug", "#a86454", Markup),
96 lang("Python", "#3572A5", Programming),
97 lang("R", "#198CE7", Programming),
98 lang("Racket", "#3c5caa", Programming),
99 lang("Ruby", "#701516", Programming),
100 lang("Rust", "#dea584", Programming),
101 lang("Sass", "#a53b70", Markup),
102 lang("Scala", "#c22d40", Programming),
103 lang("Scheme", "#1e4aff", Programming),
104 lang("SCSS", "#c6538c", Markup),
105 lang("Shell", "#89e051", Programming),
106 lang("Solidity", "#AA6746", Programming),
107 lang("SQL", "#e38c00", Data),
108 lang("Starlark", "#76d275", Programming),
109 lang("Stylus", "#ff6347", Markup),
110 lang("Svelte", "#ff3e00", Markup),
111 lang("Swift", "#F05138", Programming),
112 lang("TOML", "#9c4221", Data),
113 lang("Twig", "#c1d026", Markup),
114 lang("TypeScript", "#3178c6", Programming),
115 lang("Vim Script", "#199f4b", Programming),
116 lang("Visual Basic .NET", "#945db7", Programming),
117 lang("Vue", "#41b883", Markup),
118 lang("WebAssembly", "#04133b", Programming),
119 lang("XML", "#0060ac", Data),
120 lang("YAML", "#cb171e", Data),
121 lang("Zig", "#ec915c", Programming),
122];
123
124/// A language by name, any case.
125pub fn named(name: &str) -> Option<&'static Language> {
126 LANGUAGES.iter().find(|language| language.name.eq_ignore_ascii_case(name))
127}
128
129/// Files known by their whole name.
130fn by_filename(name: &str) -> Option<&'static str> {
131 Some(match name {
132 "Dockerfile" | "Containerfile" => "Dockerfile",
133 "Makefile" | "GNUmakefile" | "makefile" => "Makefile",
134 "CMakeLists.txt" => "CMake",
135 "Rakefile" | "Gemfile" | "Vagrantfile" | "Podfile" | "Brewfile" => "Ruby",
136 "Jenkinsfile" => "Groovy",
137 "BUILD" | "BUILD.bazel" | "WORKSPACE" | "WORKSPACE.bazel" | "Tiltfile" => "Starlark",
138 _ if name.starts_with("Dockerfile.") => "Dockerfile",
139 _ => return None,
140 })
141}
142
143/// Files known by their extension, lowercased, without the dot.
144fn by_extension(extension: &str) -> Option<&'static str> {
145 Some(match extension {
146 "asm" | "s" | "nasm" => "Assembly",
147 "astro" => "Astro",
148 "bat" | "cmd" => "Batchfile",
149 "c" | "h" => "C",
150 "cs" | "csx" => "C#",
151 "cc" | "cpp" | "cxx" | "c++" | "hpp" | "hh" | "hxx" | "h++" | "ipp" | "tpp" => "C++",
152 "clj" | "cljs" | "cljc" => "Clojure",
153 "cmake" => "CMake",
154 "lisp" | "lsp" => "Common Lisp",
155 "cr" => "Crystal",
156 "css" => "CSS",
157 "cu" | "cuh" => "Cuda",
158 "dart" => "Dart",
159 "dockerfile" => "Dockerfile",
160 "ejs" => "EJS",
161 "ex" | "exs" => "Elixir",
162 "elm" => "Elm",
163 "el" => "Emacs Lisp",
164 "erl" | "hrl" => "Erlang",
165 "fs" | "fsi" | "fsx" => "F#",
166 "f" | "f77" | "f90" | "f95" | "f03" | "for" => "Fortran",
167 "gleam" => "Gleam",
168 "glsl" | "vert" | "frag" | "geom" | "comp" => "GLSL",
169 "go" => "Go",
170 "graphql" | "gql" | "graphqls" => "GraphQL",
171 "groovy" | "gradle" | "gvy" => "Groovy",
172 "hbs" | "handlebars" | "mustache" => "Handlebars",
173 "hs" | "lhs" => "Haskell",
174 "hcl" | "tf" | "tfvars" => "HCL",
175 "html" | "htm" | "xhtml" => "HTML",
176 "java" => "Java",
177 "js" | "mjs" | "cjs" | "jsx" => "JavaScript",
178 "json" | "jsonc" | "json5" => "JSON",
179 "jsonnet" | "libsonnet" => "Jsonnet",
180 "jl" => "Julia",
181 "ipynb" => "Jupyter Notebook",
182 "kt" | "kts" => "Kotlin",
183 "less" => "Less",
184 "liquid" => "Liquid",
185 "lua" => "Lua",
186 "mk" | "mak" => "Makefile",
187 "md" | "markdown" | "mdown" => "Markdown",
188 "mdx" => "MDX",
189 "nim" | "nims" => "Nim",
190 "nix" => "Nix",
191 "m" => "Objective-C",
192 "mm" => "Objective-C++",
193 "ml" | "mli" => "OCaml",
194 "pl" | "pm" => "Perl",
195 "php" | "phtml" => "PHP",
196 "pgsql" | "plpgsql" => "PLpgSQL",
197 "ps1" | "psm1" | "psd1" => "PowerShell",
198 "pug" | "jade" => "Pug",
199 "py" | "pyi" | "pyw" => "Python",
200 "r" => "R",
201 "rkt" => "Racket",
202 "rb" | "gemspec" | "rake" => "Ruby",
203 "rs" => "Rust",
204 "sass" => "Sass",
205 "scala" | "sc" => "Scala",
206 "scm" | "ss" => "Scheme",
207 "scss" => "SCSS",
208 "sh" | "bash" | "zsh" | "ksh" => "Shell",
209 "sol" => "Solidity",
210 "sql" => "SQL",
211 "bzl" | "star" => "Starlark",
212 "styl" => "Stylus",
213 "svelte" => "Svelte",
214 "swift" => "Swift",
215 "toml" => "TOML",
216 "twig" => "Twig",
217 // TSX is TypeScript, as JSX is JavaScript: one language on the bar.
218 "ts" | "tsx" | "mts" | "cts" => "TypeScript",
219 "vim" => "Vim Script",
220 "vb" => "Visual Basic .NET",
221 "vue" => "Vue",
222 "wat" | "wast" => "WebAssembly",
223 "xml" | "xsd" | "xsl" | "plist" | "csproj" | "fsproj" | "vbproj" => "XML",
224 "yml" | "yaml" => "YAML",
225 "zig" => "Zig",
226 _ => return None,
227 })
228}
229
230/// The language of a file by its path, if g1t knows it.
231pub fn language_of(path: &str) -> Option<&'static Language> {
232 let name = path.rsplit('/').next().unwrap_or(path);
233 if name.ends_with(".blade.php") {
234 return named("Blade");
235 }
236 let found = by_filename(name).or_else(|| {
237 let (stem, extension) = name.rsplit_once('.')?;
238 if stem.is_empty() {
239 return None;
240 }
241 by_extension(&extension.to_ascii_lowercase())
242 })?;
243 named(found)
244}
245
246/// Directories whose code someone else wrote.
247const VENDORED_DIRS: &[&str] = &[
248 "node_modules",
249 "vendor",
250 "vendors",
251 "third_party",
252 "third-party",
253 "3rdparty",
254 "bower_components",
255 "jspm_packages",
256 "Pods",
257 "Carthage",
258 "site-packages",
259 "venv",
260 "__pypackages__",
261];
262
263/// Directories that hold docs and examples.
264const DOCUMENTATION_DIRS: &[&str] = &["docs", "doc", "documentation", "examples", "example", "samples"];
265
266/// Directories of output that a build makes.
267const GENERATED_DIRS: &[&str] = &["dist", "__generated__", "generated", "__pycache__", ".next", ".nuxt", ".svelte-kit"];
268
269fn directories(path: &str) -> impl Iterator<Item = &str> {
270 let mut parts: Vec<&str> = path.split('/').collect();
271 parts.pop();
272 parts.into_iter()
273}
274
275/// Whether a file is someone else's code kept in the repository, or the
276/// configuration under a dot-directory (`.github`, `.vscode`).
277pub fn vendored(path: &str) -> bool {
278 let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
279 directories(path).any(|dir| dir.starts_with('.') || VENDORED_DIRS.iter().any(|vendored| dir.eq_ignore_ascii_case(vendored)))
280 || name.starts_with("jquery")
281 || name.starts_with("bootstrap") && (name.ends_with(".js") || name.ends_with(".css"))
282 || matches!(name.as_str(), "gradlew" | "gradlew.bat" | "mvnw" | "mvnw.cmd")
283}
284
285/// Whether a file was made by a tool rather than written: minified,
286/// compiled from a schema, or a lockfile.
287pub fn generated(path: &str) -> bool {
288 let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
289 directories(path).any(|dir| GENERATED_DIRS.iter().any(|generated| dir.eq_ignore_ascii_case(generated)))
290 || [".min.js", ".min.css", ".min.mjs", ".map", ".pb.go", "_pb2.py", "_pb2_grpc.py", ".pb.cc", ".pb.h", ".g.dart", ".freezed.dart", ".designer.cs", ".lock"]
291 .iter()
292 .any(|suffix| name.ends_with(suffix))
293 || name.contains(".generated.")
294 || matches!(name.as_str(), "package-lock.json" | "pnpm-lock.yaml" | "npm-shrinkwrap.json" | "go.sum" | "composer.lock")
295}
296
297/// Whether a file is documentation or an example.
298pub fn documentation(path: &str) -> bool {
299 directories(path).any(|dir| DOCUMENTATION_DIRS.iter().any(|docs| dir.eq_ignore_ascii_case(docs)))
300}
301
302/// What `.gitattributes` says of a path. Each is None when it says nothing.
303#[derive(Clone, Debug, Default, PartialEq, Eq)]
304pub struct Said {
305 pub vendored: Option<bool>,
306 pub generated: Option<bool>,
307 pub documentation: Option<bool>,
308 pub detectable: Option<bool>,
309 pub language: Option<String>,
310}
311
312/// The `linguist-*` lines of a `.gitattributes` file, in order: a later
313/// line that matches wins, as in git.
314#[derive(Clone, Debug, Default)]
315pub struct Attributes {
316 rules: Vec<(String, Said)>,
317}
318
319/// An attribute set (`attr`, `attr=true`), unset (`-attr`, `attr=false`) or
320/// not this one.
321fn switch(word: &str, attribute: &str) -> Option<bool> {
322 if let Some(rest) = word.strip_prefix('-') {
323 return (rest == attribute).then_some(false);
324 }
325 if word == attribute {
326 return Some(true);
327 }
328 let (name, value) = word.split_once('=')?;
329 (name == attribute).then_some(!matches!(value, "false" | "0"))
330}
331
332impl Attributes {
333 pub fn parse(text: &str) -> Self {
334 let mut rules = Vec::new();
335 for line in text.lines() {
336 let line = line.trim();
337 if line.is_empty() || line.starts_with('#') {
338 continue;
339 }
340 let mut words = line.split_whitespace();
341 let Some(pattern) = words.next() else { continue };
342 let mut said = Said::default();
343 for word in words {
344 if let Some(on) = switch(word, "linguist-vendored") {
345 said.vendored = Some(on);
346 } else if let Some(on) = switch(word, "linguist-generated") {
347 said.generated = Some(on);
348 } else if let Some(on) = switch(word, "linguist-documentation") {
349 said.documentation = Some(on);
350 } else if let Some(on) = switch(word, "linguist-detectable") {
351 said.detectable = Some(on);
352 } else if let Some(name) = word.strip_prefix("linguist-language=") {
353 said.language = Some(name.replace("-", " "));
354 }
355 }
356 if said != Said::default() {
357 rules.push((pattern.to_owned(), said));
358 }
359 }
360 Attributes { rules }
361 }
362
363 /// What the file's matching lines say, the later over the earlier.
364 pub fn of(&self, path: &str) -> Said {
365 let mut said = Said::default();
366 for (pattern, rule) in &self.rules {
367 if !g1t_rules::glob::path_matches(pattern, path) {
368 continue;
369 }
370 said.vendored = rule.vendored.or(said.vendored);
371 said.generated = rule.generated.or(said.generated);
372 said.documentation = rule.documentation.or(said.documentation);
373 said.detectable = rule.detectable.or(said.detectable);
374 if rule.language.is_some() {
375 said.language = rule.language.clone();
376 }
377 }
378 said
379 }
380}
381
382/// The language a file counts towards, or None when it does not count.
383pub fn counted(path: &str, attributes: &Attributes) -> Option<&'static Language> {
384 let said = attributes.of(path);
385 let language = match &said.language {
386 Some(name) => named(name).or_else(|| language_of(path))?,
387 None => language_of(path)?,
388 };
389 if said.vendored.unwrap_or_else(|| vendored(path))
390 || said.generated.unwrap_or_else(|| generated(path))
391 || said.documentation.unwrap_or_else(|| documentation(path))
392 {
393 return None;
394 }
395 let detectable = said.detectable.unwrap_or(matches!(language.kind, Kind::Programming | Kind::Markup));
396 detectable.then_some(language)
397}
398
399/// Each language's share of the bytes, largest first, to one decimal place.
400pub fn shares(bytes: impl IntoIterator<Item = (&'static Language, u64)>) -> Vec<LanguageShare> {
401 let mut by_language: Vec<(&'static Language, u64)> = Vec::new();
402 for (language, size) in bytes {
403 match by_language.iter_mut().find(|(known, _)| known.name == language.name) {
404 Some((_, total)) => *total += size,
405 None => by_language.push((language, size)),
406 }
407 }
408 let total: u64 = by_language.iter().map(|(_, size)| size).sum();
409 by_language.retain(|(_, size)| *size > 0);
410 by_language.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.name.cmp(b.0.name)));
411 by_language
412 .into_iter()
413 .map(|(language, size)| LanguageShare {
414 name: language.name.to_owned(),
415 color: language.color.map(str::to_owned),
416 bytes: size,
417 percent: if total == 0 { 0.0 } else { (size as f64 * 1000.0 / total as f64).round() / 10.0 },
418 })
419 .collect()
420}
421
422#[cfg(test)]
423mod tests {
424 use super::*;
425
426 fn name(path: &str) -> Option<&'static str> {
427 counted(path, &Attributes::default()).map(|language| language.name)
428 }
429
430 #[test]
431 fn files_are_known_by_extension_and_name() {
432 assert_eq!(name("src/main.rs"), Some("Rust"));
433 assert_eq!(name("app/routes/home.tsx"), Some("TypeScript"));
434 assert_eq!(name("lib/x.mts"), Some("TypeScript"));
435 assert_eq!(name("Dockerfile"), Some("Dockerfile"));
436 assert_eq!(name("deploy/Dockerfile.prod"), Some("Dockerfile"));
437 assert_eq!(name("Makefile"), Some("Makefile"));
438 assert_eq!(name("resources/views/home.blade.php"), Some("Blade"));
439 assert_eq!(name("SCRIPT.SH"), Some("Shell"));
440 assert_eq!(name(".bashrc"), None, "a dotfile has no extension");
441 assert_eq!(name("LICENSE"), None);
442 }
443
444 #[test]
445 fn data_prose_vendored_generated_and_docs_do_not_count() {
446 assert_eq!(name("package.json"), None);
447 assert_eq!(name("README.md"), None);
448 assert_eq!(name("config.yml"), None);
449 assert_eq!(name("node_modules/react/index.js"), None);
450 assert_eq!(name("web/vendor/lib.go"), None);
451 assert_eq!(name("static/jquery-3.7.1.js"), None);
452 assert_eq!(name("public/app.min.js"), None);
453 assert_eq!(name("dist/index.js"), None);
454 assert_eq!(name("api/v1/service.pb.go"), None);
455 assert_eq!(name("docs/src/plugin.ts"), None);
456 assert_eq!(name("examples/hello.rs"), None);
457 assert_eq!(name(".github/scripts/release.sh"), None);
458 assert_eq!(name("tests/it.rs"), Some("Rust"), "tests count");
459 }
460
461 #[test]
462 fn gitattributes_overrides_what_is_counted() {
463 let attributes = Attributes::parse(
464 "# ours\n\
465 vendor/** -linguist-vendored\n\
466 *.gen.ts linguist-generated=true\n\
467 docs/** linguist-documentation=false\n\
468 *.inc linguist-language=PHP\n\
469 *.sql linguist-detectable\n\
470 src/legacy/*.js -linguist-detectable\n",
471 );
472 let of = |path: &str| counted(path, &attributes).map(|language| language.name);
473 assert_eq!(of("vendor/ours.go"), Some("Go"));
474 assert_eq!(of("src/api.gen.ts"), None);
475 assert_eq!(of("docs/site.ts"), Some("TypeScript"));
476 assert_eq!(of("lib/header.inc"), Some("PHP"));
477 assert_eq!(of("db/schema.sql"), Some("SQL"));
478 assert_eq!(of("src/legacy/old.js"), None);
479 assert_eq!(of("src/new.js"), Some("JavaScript"));
480 }
481
482 #[test]
483 fn a_later_line_wins() {
484 let attributes = Attributes::parse("*.js linguist-vendored\nsrc/*.js -linguist-vendored\n");
485 assert_eq!(attributes.of("src/a.js").vendored, Some(false));
486 assert_eq!(attributes.of("lib/a.js").vendored, Some(true));
487 }
488
489 #[test]
490 fn shares_add_up_by_language() {
491 let rust = named("Rust").unwrap();
492 let ts = named("TypeScript").unwrap();
493 let shares = shares([(rust, 600), (ts, 300), (rust, 100), (ts, 0)]);
494 assert_eq!(shares.len(), 2);
495 assert_eq!(shares[0].name, "Rust");
496 assert_eq!(shares[0].bytes, 700);
497 assert_eq!(shares[0].percent, 70.0);
498 assert_eq!(shares[1].percent, 30.0);
499 assert_eq!(shares[0].color.as_deref(), Some("#dea584"));
500 assert!(super::shares(Vec::new()).is_empty());
501 }
502
503 #[test]
504 fn every_extension_names_a_known_language() {
505 let one_of_each = "asm astro bat c cs cpp clj cmake lisp cr css cu dart dockerfile ejs ex elm el erl fs f90 gleam glsl go graphql \
506 groovy hbs hs tf html java js json jsonnet jl ipynb kt less liquid lua mk md mdx nim nix m mm ml pl php pgsql ps1 pug py r \
507 rkt rb rs sass scala scm scss sh sol sql bzl styl svelte swift toml tsx twig ts vim vb vue wat xml yml zig";
508 for extension in one_of_each.split_whitespace() {
509 assert!(language_of(&format!("a.{extension}")).is_some(), "{extension}");
510 }
511 for name in ["Dockerfile", "Makefile", "CMakeLists.txt", "Gemfile", "Jenkinsfile", "BUILD.bazel", "x.blade.php"] {
512 assert!(language_of(name).is_some(), "{name}");
513 }
514 for path in ["a.asm", "a.tsx", "a.ipynb", "a.vue", "a.ex", "a.hcl", "a.sol", "a.zig", "a.wat", "a.bzl", "a.vb", "a.pgsql"] {
515 assert!(language_of(path).is_some(), "{path}");
516 }
517 let mut names: Vec<&str> = LANGUAGES.iter().map(|language| language.name).collect();
518 names.sort();
519 names.dedup();
520 assert_eq!(names.len(), LANGUAGES.len(), "each language once");
521 }
522}