Skip to content
Sign in

flagon-io/g1t

public
0

g1t/services/repos/src/languages.rs

523 lines20,655 bytesCodeBlame
1//! Which language each file is written in, and which files count towards a
2//! repository's languages: a small version of what people know from the
3//! language bar on code hosts. Programming and markup languages count;
4//! data (JSON, YAML) and prose (Markdown) do not, unless `.gitattributes`
5//! says `linguist-detectable`. Vendored, generated and documentation files
6//! never count, unless `.gitattributes` says otherwise:
7//!
8//! ```text
9//! third_party/** -linguist-vendored
10//! *.gen.ts linguist-generated
11//! docs/** -linguist-documentation
12//! *.inc linguist-language=PHP
13//! *.sql linguist-detectable
14//! ```
15
16use g1t_contracts::about::LanguageShare;
17
18/// What a language is, as it decides whether it counts by default.
19#[derive(Clone, Copy, Debug, PartialEq, Eq)]
20pub enum Kind {
21 Programming,
22 Markup,
23 Data,
24 Prose,
25}
26
27#[derive(Debug, PartialEq, Eq)]
28pub struct Language {
29 pub name: &'static str,
30 pub color: Option<&'static str>,
31 pub kind: Kind,
32}
33
34const fn lang(name: &'static str, color: &'static str, kind: Kind) -> Language {
35 Language { name, color: Some(color), kind }
36}
37
38use Kind::{Data, Markup, Programming, Prose};
39
40/// Every language g1t knows, with the colors people know them by.
41pub static LANGUAGES: &[Language] = &[
42 lang("Assembly", "#6E4C13", Programming),
43 lang("Astro", "#ff5a03", Markup),
44 lang("Batchfile", "#C1F12E", Programming),
45 lang("Blade", "#f7523f", Markup),
46 lang("C", "#555555", Programming),
47 lang("C#", "#178600", Programming),
48 lang("C++", "#f34b7d", Programming),
49 lang("Clojure", "#db5855", Programming),
50 lang("CMake", "#DA3434", Programming),
51 lang("Common Lisp", "#3fb68b", Programming),
52 lang("Crystal", "#000100", Programming),
53 lang("CSS", "#663399", Markup),
54 lang("Cuda", "#3A4E3A", Programming),
55 lang("Dart", "#00B4AB", Programming),
56 lang("Dockerfile", "#384d54", Programming),
57 lang("EJS", "#a91e50", Markup),
58 lang("Elixir", "#6e4a7e", Programming),
59 lang("Elm", "#60B5CC", Programming),
60 lang("Emacs Lisp", "#c065db", Programming),
61 lang("Erlang", "#B83998", Programming),
62 lang("F#", "#b845fc", Programming),
63 lang("Fortran", "#4d41b1", Programming),
64 lang("Gleam", "#ffaff3", Programming),
65 lang("GLSL", "#5686a5", Programming),
66 lang("Go", "#00ADD8", Programming),
67 lang("GraphQL", "#e10098", Data),
68 lang("Groovy", "#4298b8", Programming),
69 lang("Handlebars", "#f7931e", Markup),
70 lang("Haskell", "#5e5086", Programming),
71 lang("HCL", "#844FBA", Programming),
72 lang("HTML", "#e34c26", Markup),
73 lang("Java", "#b07219", Programming),
74 lang("JavaScript", "#f1e05a", Programming),
75 lang("JSON", "#292929", Data),
76 lang("Jsonnet", "#0064bd", Programming),
77 lang("Julia", "#a270ba", Programming),
78 lang("Jupyter Notebook", "#DA5B0B", Markup),
79 lang("Kotlin", "#A97BFF", Programming),
80 lang("Less", "#1d365d", Markup),
81 lang("Liquid", "#67b8de", Markup),
82 lang("Lua", "#000080", Programming),
83 lang("Makefile", "#427819", Programming),
84 lang("Markdown", "#083fa1", Prose),
85 lang("MDX", "#fcb32c", Markup),
86 lang("Nim", "#ffc200", Programming),
87 lang("Nix", "#7e7eff", Programming),
88 lang("Objective-C", "#438eff", Programming),
89 lang("Objective-C++", "#6866fb", Programming),
90 lang("OCaml", "#ef7a08", Programming),
91 lang("Perl", "#0298c3", Programming),
92 lang("PHP", "#4F5D95", Programming),
93 lang("PLpgSQL", "#336790", Programming),
94 lang("PowerShell", "#012456", Programming),
95 lang("Pug", "#a86454", Markup),
96 lang("Python", "#3572A5", Programming),
97 lang("R", "#198CE7", Programming),
98 lang("Racket", "#3c5caa", Programming),
99 lang("Ruby", "#701516", Programming),
100 lang("Rust", "#dea584", Programming),
101 lang("Sass", "#a53b70", Markup),
102 lang("Scala", "#c22d40", Programming),
103 lang("Scheme", "#1e4aff", Programming),
104 lang("SCSS", "#c6538c", Markup),
105 lang("Shell", "#89e051", Programming),
106 lang("Solidity", "#AA6746", Programming),
107 lang("SQL", "#e38c00", Data),
108 lang("Starlark", "#76d275", Programming),
109 lang("Stylus", "#ff6347", Markup),
110 lang("Svelte", "#ff3e00", Markup),
111 lang("Swift", "#F05138", Programming),
112 lang("TOML", "#9c4221", Data),
113 lang("TSX", "#3178c6", Programming),
114 lang("Twig", "#c1d026", Markup),
115 lang("TypeScript", "#3178c6", Programming),
116 lang("Vim Script", "#199f4b", Programming),
117 lang("Visual Basic .NET", "#945db7", Programming),
118 lang("Vue", "#41b883", Markup),
119 lang("WebAssembly", "#04133b", Programming),
120 lang("XML", "#0060ac", Data),
121 lang("YAML", "#cb171e", Data),
122 lang("Zig", "#ec915c", Programming),
123];
124
125/// A language by name, any case.
126pub fn named(name: &str) -> Option<&'static Language> {
127 LANGUAGES.iter().find(|language| language.name.eq_ignore_ascii_case(name))
128}
129
130/// Files known by their whole name.
131fn by_filename(name: &str) -> Option<&'static str> {
132 Some(match name {
133 "Dockerfile" | "Containerfile" => "Dockerfile",
134 "Makefile" | "GNUmakefile" | "makefile" => "Makefile",
135 "CMakeLists.txt" => "CMake",
136 "Rakefile" | "Gemfile" | "Vagrantfile" | "Podfile" | "Brewfile" => "Ruby",
137 "Jenkinsfile" => "Groovy",
138 "BUILD" | "BUILD.bazel" | "WORKSPACE" | "WORKSPACE.bazel" | "Tiltfile" => "Starlark",
139 _ if name.starts_with("Dockerfile.") => "Dockerfile",
140 _ => return None,
141 })
142}
143
144/// Files known by their extension, lowercased, without the dot.
145fn by_extension(extension: &str) -> Option<&'static str> {
146 Some(match extension {
147 "asm" | "s" | "nasm" => "Assembly",
148 "astro" => "Astro",
149 "bat" | "cmd" => "Batchfile",
150 "c" | "h" => "C",
151 "cs" | "csx" => "C#",
152 "cc" | "cpp" | "cxx" | "c++" | "hpp" | "hh" | "hxx" | "h++" | "ipp" | "tpp" => "C++",
153 "clj" | "cljs" | "cljc" => "Clojure",
154 "cmake" => "CMake",
155 "lisp" | "lsp" => "Common Lisp",
156 "cr" => "Crystal",
157 "css" => "CSS",
158 "cu" | "cuh" => "Cuda",
159 "dart" => "Dart",
160 "dockerfile" => "Dockerfile",
161 "ejs" => "EJS",
162 "ex" | "exs" => "Elixir",
163 "elm" => "Elm",
164 "el" => "Emacs Lisp",
165 "erl" | "hrl" => "Erlang",
166 "fs" | "fsi" | "fsx" => "F#",
167 "f" | "f77" | "f90" | "f95" | "f03" | "for" => "Fortran",
168 "gleam" => "Gleam",
169 "glsl" | "vert" | "frag" | "geom" | "comp" => "GLSL",
170 "go" => "Go",
171 "graphql" | "gql" | "graphqls" => "GraphQL",
172 "groovy" | "gradle" | "gvy" => "Groovy",
173 "hbs" | "handlebars" | "mustache" => "Handlebars",
174 "hs" | "lhs" => "Haskell",
175 "hcl" | "tf" | "tfvars" => "HCL",
176 "html" | "htm" | "xhtml" => "HTML",
177 "java" => "Java",
178 "js" | "mjs" | "cjs" | "jsx" => "JavaScript",
179 "json" | "jsonc" | "json5" => "JSON",
180 "jsonnet" | "libsonnet" => "Jsonnet",
181 "jl" => "Julia",
182 "ipynb" => "Jupyter Notebook",
183 "kt" | "kts" => "Kotlin",
184 "less" => "Less",
185 "liquid" => "Liquid",
186 "lua" => "Lua",
187 "mk" | "mak" => "Makefile",
188 "md" | "markdown" | "mdown" => "Markdown",
189 "mdx" => "MDX",
190 "nim" | "nims" => "Nim",
191 "nix" => "Nix",
192 "m" => "Objective-C",
193 "mm" => "Objective-C++",
194 "ml" | "mli" => "OCaml",
195 "pl" | "pm" => "Perl",
196 "php" | "phtml" => "PHP",
197 "pgsql" | "plpgsql" => "PLpgSQL",
198 "ps1" | "psm1" | "psd1" => "PowerShell",
199 "pug" | "jade" => "Pug",
200 "py" | "pyi" | "pyw" => "Python",
201 "r" => "R",
202 "rkt" => "Racket",
203 "rb" | "gemspec" | "rake" => "Ruby",
204 "rs" => "Rust",
205 "sass" => "Sass",
206 "scala" | "sc" => "Scala",
207 "scm" | "ss" => "Scheme",
208 "scss" => "SCSS",
209 "sh" | "bash" | "zsh" | "ksh" => "Shell",
210 "sol" => "Solidity",
211 "sql" => "SQL",
212 "bzl" | "star" => "Starlark",
213 "styl" => "Stylus",
214 "svelte" => "Svelte",
215 "swift" => "Swift",
216 "toml" => "TOML",
217 "tsx" => "TSX",
218 "twig" => "Twig",
219 "ts" | "mts" | "cts" => "TypeScript",
220 "vim" => "Vim Script",
221 "vb" => "Visual Basic .NET",
222 "vue" => "Vue",
223 "wat" | "wast" => "WebAssembly",
224 "xml" | "xsd" | "xsl" | "plist" | "csproj" | "fsproj" | "vbproj" => "XML",
225 "yml" | "yaml" => "YAML",
226 "zig" => "Zig",
227 _ => return None,
228 })
229}
230
231/// The language of a file by its path, if g1t knows it.
232pub fn language_of(path: &str) -> Option<&'static Language> {
233 let name = path.rsplit('/').next().unwrap_or(path);
234 if name.ends_with(".blade.php") {
235 return named("Blade");
236 }
237 let found = by_filename(name).or_else(|| {
238 let (stem, extension) = name.rsplit_once('.')?;
239 if stem.is_empty() {
240 return None;
241 }
242 by_extension(&extension.to_ascii_lowercase())
243 })?;
244 named(found)
245}
246
247/// Directories whose code someone else wrote.
248const VENDORED_DIRS: &[&str] = &[
249 "node_modules",
250 "vendor",
251 "vendors",
252 "third_party",
253 "third-party",
254 "3rdparty",
255 "bower_components",
256 "jspm_packages",
257 "Pods",
258 "Carthage",
259 "site-packages",
260 "venv",
261 "__pypackages__",
262];
263
264/// Directories that hold docs and examples.
265const DOCUMENTATION_DIRS: &[&str] = &["docs", "doc", "documentation", "examples", "example", "samples"];
266
267/// Directories of output that a build makes.
268const GENERATED_DIRS: &[&str] = &["dist", "__generated__", "generated", "__pycache__", ".next", ".nuxt", ".svelte-kit"];
269
270fn directories(path: &str) -> impl Iterator<Item = &str> {
271 let mut parts: Vec<&str> = path.split('/').collect();
272 parts.pop();
273 parts.into_iter()
274}
275
276/// Whether a file is someone else's code kept in the repository, or the
277/// configuration under a dot-directory (`.github`, `.vscode`).
278pub fn vendored(path: &str) -> bool {
279 let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
280 directories(path).any(|dir| dir.starts_with('.') || VENDORED_DIRS.iter().any(|vendored| dir.eq_ignore_ascii_case(vendored)))
281 || name.starts_with("jquery")
282 || name.starts_with("bootstrap") && (name.ends_with(".js") || name.ends_with(".css"))
283 || matches!(name.as_str(), "gradlew" | "gradlew.bat" | "mvnw" | "mvnw.cmd")
284}
285
286/// Whether a file was made by a tool rather than written: minified,
287/// compiled from a schema, or a lockfile.
288pub fn generated(path: &str) -> bool {
289 let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
290 directories(path).any(|dir| GENERATED_DIRS.iter().any(|generated| dir.eq_ignore_ascii_case(generated)))
291 || [".min.js", ".min.css", ".min.mjs", ".map", ".pb.go", "_pb2.py", "_pb2_grpc.py", ".pb.cc", ".pb.h", ".g.dart", ".freezed.dart", ".designer.cs", ".lock"]
292 .iter()
293 .any(|suffix| name.ends_with(suffix))
294 || name.contains(".generated.")
295 || matches!(name.as_str(), "package-lock.json" | "pnpm-lock.yaml" | "npm-shrinkwrap.json" | "go.sum" | "composer.lock")
296}
297
298/// Whether a file is documentation or an example.
299pub fn documentation(path: &str) -> bool {
300 directories(path).any(|dir| DOCUMENTATION_DIRS.iter().any(|docs| dir.eq_ignore_ascii_case(docs)))
301}
302
303/// What `.gitattributes` says of a path. Each is None when it says nothing.
304#[derive(Clone, Debug, Default, PartialEq, Eq)]
305pub struct Said {
306 pub vendored: Option<bool>,
307 pub generated: Option<bool>,
308 pub documentation: Option<bool>,
309 pub detectable: Option<bool>,
310 pub language: Option<String>,
311}
312
313/// The `linguist-*` lines of a `.gitattributes` file, in order: a later
314/// line that matches wins, as in git.
315#[derive(Clone, Debug, Default)]
316pub struct Attributes {
317 rules: Vec<(String, Said)>,
318}
319
320/// An attribute set (`attr`, `attr=true`), unset (`-attr`, `attr=false`) or
321/// not this one.
322fn switch(word: &str, attribute: &str) -> Option<bool> {
323 if let Some(rest) = word.strip_prefix('-') {
324 return (rest == attribute).then_some(false);
325 }
326 if word == attribute {
327 return Some(true);
328 }
329 let (name, value) = word.split_once('=')?;
330 (name == attribute).then_some(!matches!(value, "false" | "0"))
331}
332
333impl Attributes {
334 pub fn parse(text: &str) -> Self {
335 let mut rules = Vec::new();
336 for line in text.lines() {
337 let line = line.trim();
338 if line.is_empty() || line.starts_with('#') {
339 continue;
340 }
341 let mut words = line.split_whitespace();
342 let Some(pattern) = words.next() else { continue };
343 let mut said = Said::default();
344 for word in words {
345 if let Some(on) = switch(word, "linguist-vendored") {
346 said.vendored = Some(on);
347 } else if let Some(on) = switch(word, "linguist-generated") {
348 said.generated = Some(on);
349 } else if let Some(on) = switch(word, "linguist-documentation") {
350 said.documentation = Some(on);
351 } else if let Some(on) = switch(word, "linguist-detectable") {
352 said.detectable = Some(on);
353 } else if let Some(name) = word.strip_prefix("linguist-language=") {
354 said.language = Some(name.replace("-", " "));
355 }
356 }
357 if said != Said::default() {
358 rules.push((pattern.to_owned(), said));
359 }
360 }
361 Attributes { rules }
362 }
363
364 /// What the file's matching lines say, the later over the earlier.
365 pub fn of(&self, path: &str) -> Said {
366 let mut said = Said::default();
367 for (pattern, rule) in &self.rules {
368 if !g1t_rules::glob::path_matches(pattern, path) {
369 continue;
370 }
371 said.vendored = rule.vendored.or(said.vendored);
372 said.generated = rule.generated.or(said.generated);
373 said.documentation = rule.documentation.or(said.documentation);
374 said.detectable = rule.detectable.or(said.detectable);
375 if rule.language.is_some() {
376 said.language = rule.language.clone();
377 }
378 }
379 said
380 }
381}
382
383/// The language a file counts towards, or None when it does not count.
384pub fn counted(path: &str, attributes: &Attributes) -> Option<&'static Language> {
385 let said = attributes.of(path);
386 let language = match &said.language {
387 Some(name) => named(name).or_else(|| language_of(path))?,
388 None => language_of(path)?,
389 };
390 if said.vendored.unwrap_or_else(|| vendored(path))
391 || said.generated.unwrap_or_else(|| generated(path))
392 || said.documentation.unwrap_or_else(|| documentation(path))
393 {
394 return None;
395 }
396 let detectable = said.detectable.unwrap_or(matches!(language.kind, Kind::Programming | Kind::Markup));
397 detectable.then_some(language)
398}
399
400/// Each language's share of the bytes, largest first, to one decimal place.
401pub fn shares(bytes: impl IntoIterator<Item = (&'static Language, u64)>) -> Vec<LanguageShare> {
402 let mut by_language: Vec<(&'static Language, u64)> = Vec::new();
403 for (language, size) in bytes {
404 match by_language.iter_mut().find(|(known, _)| known.name == language.name) {
405 Some((_, total)) => *total += size,
406 None => by_language.push((language, size)),
407 }
408 }
409 let total: u64 = by_language.iter().map(|(_, size)| size).sum();
410 by_language.retain(|(_, size)| *size > 0);
411 by_language.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.name.cmp(b.0.name)));
412 by_language
413 .into_iter()
414 .map(|(language, size)| LanguageShare {
415 name: language.name.to_owned(),
416 color: language.color.map(str::to_owned),
417 bytes: size,
418 percent: if total == 0 { 0.0 } else { (size as f64 * 1000.0 / total as f64).round() / 10.0 },
419 })
420 .collect()
421}
422
423#[cfg(test)]
424mod tests {
425 use super::*;
426
427 fn name(path: &str) -> Option<&'static str> {
428 counted(path, &Attributes::default()).map(|language| language.name)
429 }
430
431 #[test]
432 fn files_are_known_by_extension_and_name() {
433 assert_eq!(name("src/main.rs"), Some("Rust"));
434 assert_eq!(name("app/routes/home.tsx"), Some("TSX"));
435 assert_eq!(name("lib/x.mts"), Some("TypeScript"));
436 assert_eq!(name("Dockerfile"), Some("Dockerfile"));
437 assert_eq!(name("deploy/Dockerfile.prod"), Some("Dockerfile"));
438 assert_eq!(name("Makefile"), Some("Makefile"));
439 assert_eq!(name("resources/views/home.blade.php"), Some("Blade"));
440 assert_eq!(name("SCRIPT.SH"), Some("Shell"));
441 assert_eq!(name(".bashrc"), None, "a dotfile has no extension");
442 assert_eq!(name("LICENSE"), None);
443 }
444
445 #[test]
446 fn data_prose_vendored_generated_and_docs_do_not_count() {
447 assert_eq!(name("package.json"), None);
448 assert_eq!(name("README.md"), None);
449 assert_eq!(name("config.yml"), None);
450 assert_eq!(name("node_modules/react/index.js"), None);
451 assert_eq!(name("web/vendor/lib.go"), None);
452 assert_eq!(name("static/jquery-3.7.1.js"), None);
453 assert_eq!(name("public/app.min.js"), None);
454 assert_eq!(name("dist/index.js"), None);
455 assert_eq!(name("api/v1/service.pb.go"), None);
456 assert_eq!(name("docs/src/plugin.ts"), None);
457 assert_eq!(name("examples/hello.rs"), None);
458 assert_eq!(name(".github/scripts/release.sh"), None);
459 assert_eq!(name("tests/it.rs"), Some("Rust"), "tests count");
460 }
461
462 #[test]
463 fn gitattributes_overrides_what_is_counted() {
464 let attributes = Attributes::parse(
465 "# ours\n\
466 vendor/** -linguist-vendored\n\
467 *.gen.ts linguist-generated=true\n\
468 docs/** linguist-documentation=false\n\
469 *.inc linguist-language=PHP\n\
470 *.sql linguist-detectable\n\
471 src/legacy/*.js -linguist-detectable\n",
472 );
473 let of = |path: &str| counted(path, &attributes).map(|language| language.name);
474 assert_eq!(of("vendor/ours.go"), Some("Go"));
475 assert_eq!(of("src/api.gen.ts"), None);
476 assert_eq!(of("docs/site.ts"), Some("TypeScript"));
477 assert_eq!(of("lib/header.inc"), Some("PHP"));
478 assert_eq!(of("db/schema.sql"), Some("SQL"));
479 assert_eq!(of("src/legacy/old.js"), None);
480 assert_eq!(of("src/new.js"), Some("JavaScript"));
481 }
482
483 #[test]
484 fn a_later_line_wins() {
485 let attributes = Attributes::parse("*.js linguist-vendored\nsrc/*.js -linguist-vendored\n");
486 assert_eq!(attributes.of("src/a.js").vendored, Some(false));
487 assert_eq!(attributes.of("lib/a.js").vendored, Some(true));
488 }
489
490 #[test]
491 fn shares_add_up_by_language() {
492 let rust = named("Rust").unwrap();
493 let ts = named("TypeScript").unwrap();
494 let shares = shares([(rust, 600), (ts, 300), (rust, 100), (ts, 0)]);
495 assert_eq!(shares.len(), 2);
496 assert_eq!(shares[0].name, "Rust");
497 assert_eq!(shares[0].bytes, 700);
498 assert_eq!(shares[0].percent, 70.0);
499 assert_eq!(shares[1].percent, 30.0);
500 assert_eq!(shares[0].color.as_deref(), Some("#dea584"));
501 assert!(super::shares(Vec::new()).is_empty());
502 }
503
504 #[test]
505 fn every_extension_names_a_known_language() {
506 let one_of_each = "asm astro bat c cs cpp clj cmake lisp cr css cu dart dockerfile ejs ex elm el erl fs f90 gleam glsl go graphql \
507 groovy hbs hs tf html java js json jsonnet jl ipynb kt less liquid lua mk md mdx nim nix m mm ml pl php pgsql ps1 pug py r \
508 rkt rb rs sass scala scm scss sh sol sql bzl styl svelte swift toml tsx twig ts vim vb vue wat xml yml zig";
509 for extension in one_of_each.split_whitespace() {
510 assert!(language_of(&format!("a.{extension}")).is_some(), "{extension}");
511 }
512 for name in ["Dockerfile", "Makefile", "CMakeLists.txt", "Gemfile", "Jenkinsfile", "BUILD.bazel", "x.blade.php"] {
513 assert!(language_of(name).is_some(), "{name}");
514 }
515 for path in ["a.asm", "a.tsx", "a.ipynb", "a.vue", "a.ex", "a.hcl", "a.sol", "a.zig", "a.wat", "a.bzl", "a.vb", "a.pgsql"] {
516 assert!(language_of(path).is_some(), "{path}");
517 }
518 let mut names: Vec<&str> = LANGUAGES.iter().map(|language| language.name).collect();
519 names.sort();
520 names.dedup();
521 assert_eq!(names.len(), LANGUAGES.len(), "each language once");
522 }
523}