| 1 | /** How alike two facts' words must be (shared over all, as sets) to be one fact. */ |
| 2 | const SAME_WORDS = 0.85; |
| 3 | /** The fewest distinct words a fact needs before it can be found inside another. */ |
| 4 | const CONTAINED_MIN_WORDS = 6; |
| 5 | |
| 6 | /** A fact's words, lowercase, one space apart: case, punctuation and spacing aside. */ |
| 7 | function folded(text: string): string { |
| 8 | return text |
| 9 | .toLowerCase() |
| 10 | .split(/[^\p{L}\p{N}]+/u) |
| 11 | .filter(Boolean) |
| 12 | .join(" "); |
| 13 | } |
| 14 | |
| 15 | /** |
| 16 | * Whether two facts say the same thing, as the work service decides it |
| 17 | * (`same_memory`): the same words; one's words, in order, inside the |
| 18 | * other's; all of one's words (at least six) among the other's; or most of |
| 19 | * their words shared. |
| 20 | */ |
| 21 | export function sameFact(a: string, b: string): boolean { |
| 22 | const [x, y] = [folded(a), folded(b)]; |
| 23 | if (!x || !y) return false; |
| 24 | if (x === y) return true; |
| 25 | const [short, long] = x.length <= y.length ? [x, y] : [y, x]; |
| 26 | if (short.split(" ").length >= 4 && ` ${long} `.includes(` ${short} `)) return true; |
| 27 | const shortWords = new Set(short.split(" ")); |
| 28 | const longWords = new Set(long.split(" ")); |
| 29 | let shared = 0; |
| 30 | for (const word of shortWords) if (longWords.has(word)) shared++; |
| 31 | if (shortWords.size >= CONTAINED_MIN_WORDS && shared === shortWords.size) return true; |
| 32 | const all = new Set([...shortWords, ...longWords]).size; |
| 33 | return all > 0 && shared / all >= SAME_WORDS; |
| 34 | } |
| 35 | |
| 36 | /** |
| 37 | * The same fact remembered twice (two runs that learned it, one saved |
| 38 | * again, or a sentence that grew since) is shown once: the first of them in |
| 39 | * the order given, so a pinned one or the newest wins. Facts match when |
| 40 | * their words do, whatever the case, spacing or punctuation, or when they |
| 41 | * are near enough to be one (see `sameFact`). |
| 42 | */ |
| 43 | export function distinctFacts<T extends { text: string }>(facts: T[]): T[] { |
| 44 | const shown: T[] = []; |
| 45 | for (const fact of facts) { |
| 46 | if (!shown.some((other) => sameFact(other.text, fact.text))) shown.push(fact); |
| 47 | } |
| 48 | return shown; |
| 49 | } |