From f417d886312f5211e40b52eae2f7962f1b62d9e0 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:19:25 -0100 Subject: [PATCH 1/9] Proofing options reach the Russian and Turkish checks too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit storytold/wordcraft#3966 made Ignore words in UPPERCASE and Ignore words that contain numbers reach six languages' own checks, one copy at a time, and Russian and Turkish were left out. lib.rs still called `ru::is_correct(word)` and `tr::is_correct(word)`, which accept words with digits (both) and short words in capitals (Russian) whatever the options say, so turning either option off changed nothing in those languages. Both now have `is_correct_with_options`, like the others. Russian's rule for a part in capitals (`ИИ-консультант`) follows the option too, as Indonesian's does, and its digits rule keeps counting superscripts (`м²`). Turkish had no test dictionary, so its two dictionary tests returned early wherever none was installed, CI included; it has a synthetic one now, also in the dictionaries-folder test. Tests: proofing_options_reach_the_russian_check and proofing_options_reach_the_turkish_check. Calling the old checks from lib.rs fails each, and so does ungating the Russian part rule or the Turkish hyphen recursion. --- crates/proof/src/lib.rs | 9 ++++--- crates/proof/src/ru/mod.rs | 45 +++++++++++++++++++++++++++----- crates/proof/src/tr/mod.rs | 39 ++++++++++++++++++++++----- crates/proof/src/tr/suggest.rs | 5 +--- crates/proof/src/tr/test_dict.rs | 22 ++++++++++++++++ 5 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 crates/proof/src/tr/test_dict.rs diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 8d0baba5..47216d72 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -778,8 +778,8 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts), ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts), ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts), - ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word), - ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), + ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct_with_options(word, opts), + ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct_with_options(word, opts), ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => { accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts) } @@ -1075,6 +1075,7 @@ mod tests { ("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC), ("it_IT", it::test_dict::AFF, it::test_dict::DIC), ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), + ("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC), ("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC), ("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC), ("id_ID", id::test_dict::AFF, id::test_dict::DIC), @@ -1094,9 +1095,9 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it. + // Turkish follows the four below; Russian, Czech, Indonesian and the Spanishes follow it. let found = installed_dictionaries(); - assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr)); + assert_eq!(found.get(4), Some(&(ProofLang::Tr, Some(dir.join("tr_TR.dic"))))); assert_eq!( found[7..], [ diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index c92e2712..d5035e6a 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -130,18 +130,30 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off, +/// short words in capitals are checked against the dictionary too, and with Ignore words that +/// contain numbers off, so are words with digits or superscripts. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary (an empty one accepts everything). pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // no Russian dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'); let w = w.trim_matches('-'); let n = w.chars().count(); - if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -161,7 +173,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { return true; } if w.contains('-') { - return hyphenated(d, w); + return hyphenated(d, w, opts); } solid_compound(d, w) } @@ -340,7 +352,7 @@ fn known(d: &AffixDict, w: &str) -> bool { /// A hyphenated word that isn't in the dictionary whole: its parts, with the particles, /// prefixes and linking vowels Russian joins with a hyphen. -fn hyphenated(d: &AffixDict, w: &str) -> bool { +fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let parts: Vec<&str> = w.split('-').collect(); if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 { return false; @@ -353,7 +365,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { && lp.iter().skip(k).all(|p| PARTICLES.contains(p)) { let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default(); - return known(d, &base) || (base.contains('-') && hyphenated(d, &base)); + return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts)); } match lp.as_slice() { // `кое-кто`, `кой-какой`. @@ -369,7 +381,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -497,4 +509,25 @@ mod tests { crate::add_word("Зкщфрусскоеслово"); assert!(is_correct("зкщфрусскоеслово")); } + + /// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks + /// those words against the dictionary instead of accepting them. + #[test] + fn proofing_options_reach_the_russian_check() { + test_dict::install(); + let lang = crate::ProofLang::Ru; + let defaults = crate::Options::default(); + let check_caps = crate::Options { ignore_uppercase: false, ..defaults }; + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default"); + assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps)); + assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults)); + assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too"); + assert!(crate::is_correct_in_with("превет2", lang, &defaults)); + assert!(!crate::is_correct_in_with("превет2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index bcab1546..ab3fe92d 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -15,6 +15,8 @@ pub mod dict; pub mod grammar; pub mod hyphen; pub mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; use std::sync::{Arc, RwLock}; @@ -87,15 +89,26 @@ pub fn is_correct(word: &str) -> bool { is_correct_with(&dictionary(), word) } +/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain +/// numbers off, words with digits are checked against the dictionary too. +pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool { + check(&dictionary(), word, opts) +} + /// [`is_correct`] against a given dictionary. pub fn is_correct_with(d: &TrDict, word: &str) -> bool { + check(d, word, &crate::Options::default()) +} + +/// [`is_correct_with_options`] against a given dictionary. +fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { if d.is_empty() { return true; // No Turkish dictionary installed: no spelling marks } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); let w = w.trim_matches(['\'', '’']); let n = w.chars().count(); - if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return true; } // Check user dictionary first @@ -120,7 +133,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool { } // Compound words separated by hyphens (e.g., `afyon-karahisar`) - if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) { + if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) { return true; } @@ -146,11 +159,8 @@ mod tests { } #[test] - fn installed_dictionary_checks_words_if_present() { - // Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check. - if !has_dictionary() { - return; - } + fn installed_dictionary_checks_words() { + test_dict::install(); assert!(is_correct("kitap")); assert!(is_correct("kitaplar")); assert!(is_correct("evde")); @@ -165,4 +175,19 @@ mod tests { crate::add_word("benimbilmedigimkelime"); assert!(is_correct("benimbilmedigimkelime")); } + + /// With Ignore words that contain numbers off, Turkish checks words with digits against the + /// dictionary instead of accepting them, hyphen parts included. + #[test] + fn proofing_options_reach_the_turkish_check() { + test_dict::install(); + let lang = crate::ProofLang::Tr; + let defaults = crate::Options::default(); + let check_digits = crate::Options { ignore_numbers: false, ..defaults }; + assert!(crate::is_correct_in_with("kitapp2", lang, &defaults)); + assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits)); + assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits)); + assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); + assert!(is_correct("kitapp2"), "direct checks keep the defaults"); + } } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index dedc1bb2..f91f8e3c 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -285,10 +285,7 @@ mod tests { #[test] fn typo_transposition_is_suggested() { - // Suggestions come from the installed Turkish dictionary; without one there are none. - if !super::super::has_dictionary() { - return; - } + crate::tr::test_dict::install(); let sugs = suggest("sorna", 3); assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}"); diff --git a/crates/proof/src/tr/test_dict.rs b/crates/proof/src/tr/test_dict.rs new file mode 100644 index 00000000..de4a1319 --- /dev/null +++ b/crates/proof/src/tr/test_dict.rs @@ -0,0 +1,22 @@ +//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// No affix rules: the forms the tests look up are listed whole. +pub const AFF: &str = "SET UTF-8\n"; + +/// The words the Turkish tests expect to be known. +pub const DIC: &str = "7 +kitap +kitaplar +ev +evde +olacaklardan +sonra +geldin +"; + +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +} From 0d953d717900a5d76f134ad9800ff6eb6ba69615 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:21:57 -0100 Subject: [PATCH 2/9] Croatian, Italian and Russian suggestions skip NOSUGGEST words storytold/wordcraft#1048 (c14e97428) taught affix.rs about NOSUGGEST, the dictionary flag for words that are correct but never offered (vulgarisms, rare forms), and patched the Polish and Portuguese copies of the suggestion engine; Czech's started with it. The Croatian, Italian and Russian copies never got it: they test candidates with `contains`, so a suggestion could be a word the dictionary says not to offer. They now use `contains_suggestible` and drop a suggestion with any part `can_suggest` refuses, as the others do. Tests: Croatian and Italian run the shared assert_nosuggest_is_honoured. Russian gets its own, since its MAP and REP tables are built in and the shared Latin-script check doesn't reach them. Each fails on the old code. --- crates/proof/src/hr/suggest.rs | 11 +++++++++-- crates/proof/src/it/suggest.rs | 12 ++++++++++-- crates/proof/src/ru/suggest.rs | 23 +++++++++++++++++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index 875bf98c..d6f8159a 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -52,11 +52,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -69,6 +69,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -348,6 +351,10 @@ fn scan(col: &mut Collector, chars: &[char]) { mod tests { use super::*; #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] { diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 500d1c09..6a043ff2 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -73,11 +73,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -90,6 +90,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -371,6 +374,11 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn italian_misspellings_get_the_right_suggestion() { crate::it::test_dict::install(); diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index cb4c3791..24eeb764 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -161,11 +161,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -178,6 +178,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -458,6 +461,22 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never + /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the + /// shared Latin-script check doesn't reach it. + #[test] + fn nosuggest_words_are_never_offered() { + let offered = |aff: &str, input: &str| { + let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap(); + suggest_in(&d, input, 16) + }; + assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared"); + for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] { + let s = offered("SET UTF-8\nNOSUGGEST N\n", input); + assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}"); + } + } + #[test] fn russian_misspellings_get_the_right_suggestion() { crate::ru::test_dict::install(); From 8f4616eecec38a9c2b768f69b1d424e31ce2ed14 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:33:17 -0100 Subject: [PATCH 3/9] Proofing: one dictionary slot for every language Nine languages each kept their own copy of the same code: the installed dictionary behind a lock, read on first use and logged, replaced by install_dictionary, forgotten by reload_dictionary, and its file reported by installed_path. The copies had drifted already. Every one documented that a dictionary given with install_dictionary has no file and isn't reported, but only Russian and Czech did that (storytold/wordcraft#3252); the other seven reported the disk file anyway. `DictSlot` (proof/src/slot.rs) is that code once. It is generic over the dictionary type, so Turkish's `TrDict` fits, and keeps the dictionary and its in-memory origin under one lock as storytold/wordcraft#3252 made it. Each language keeps its functions as one-line wrappers over its slot, so no caller changes, and the log lines are the same. installed_path now follows the documented rule in every language. Nothing outside tests installs a dictionary in memory, but tests do, concurrently, so the dictionaries-folder tests in proof and engine could race on the reported path of a language another test installs. They now accept the folder's file or none there, and check that the folder is searched first with hunspell::find, which no install affects. Tests: proof 279, engine 927, layout 501 (lib); slot.rs has a test of its own. --- crates/proof/src/cs/mod.rs | 60 +++------------ crates/proof/src/es/mod.rs | 71 ++++++----------- crates/proof/src/hr/mod.rs | 45 +++-------- crates/proof/src/id/mod.rs | 50 ++++-------- crates/proof/src/it/mod.rs | 44 +++-------- crates/proof/src/lib.rs | 50 ++++++------ crates/proof/src/pl/mod.rs | 52 ++++--------- crates/proof/src/pt/mod.rs | 44 +++-------- crates/proof/src/ru/mod.rs | 68 ++++------------ crates/proof/src/slot.rs | 150 ++++++++++++++++++++++++++++++++++++ crates/proof/src/tr/dict.rs | 15 ++++ crates/proof/src/tr/mod.rs | 57 ++++---------- 12 files changed, 325 insertions(+), 381 deletions(-) create mode 100644 crates/proof/src/slot.rs diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index fee7c561..55e21aef 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -7,83 +7,47 @@ pub(crate) mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Czech Hunspell dictionary. const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; +/// The Czech dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); + /// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Czech spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; + DICT.reload(); } -/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. +/// The `.dic` file of the installed Czech dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 94c6511a..253642c8 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -16,16 +16,19 @@ mod suggest; pub(crate) mod test_dict; pub mod voseo; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::ProofLang; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first -/// use, then the installed one (or an empty one). -static DICTS: [RwLock>>; 4] = [const { RwLock::new(None) }; 4]; +/// The dictionary of each variety, found on first use (see [`DictSlot`]). +static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); /// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the /// generic one RLA-ES publishes). @@ -40,79 +43,53 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e /// or generic one. const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"]; -/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's -/// file names; Spain for any other language. -fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) { +/// `lang`'s variety's dictionary; Spain's for any other language. +fn slot(lang: ProofLang) -> &'static DictSlot { match lang { - ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO), - ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA), - ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY), - _ => (0, "Spanish (Spain)", SPAIN), + ProofLang::EsMx => &MEXICO_DICT, + ProofLang::Es419 => &LATIN_AMERICA_DICT, + ProofLang::EsUy => &URUGUAY_DICT, + _ => &SPAIN_DICT, } } /// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`, /// `es_UY`): the first of its file names, used for the status-bar hint and load errors. pub fn dictionary_name(lang: ProofLang) -> &'static str { - variety(lang).2.first().copied().unwrap_or("es_ES") + slot(lang).name() } /// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given /// with [`install_dictionary`], or an empty one when none is available (spelling is off then; /// see [`has_dictionary`]). pub fn dictionary(lang: ProofLang) -> Arc { - let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) }; - if let Some(d) = slot.read().unwrap_or_else(|e| e.into_inner()).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed(lang)); - let mut slot = slot.write().unwrap_or_else(|e| e.into_inner()); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed(lang: ProofLang) -> AffixDict { - let (_, name, files) = variety(lang); - let key = dictionary_name(lang); - let Some((aff, dic)) = crate::hunspell::find(files) else { - crate::hunspell::clear_load_error(key); - log::info!("{name} spelling: no Hunspell {key} dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("{name} spelling: {e}"); - AffixDict::default() - }) + slot(lang).get() } /// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text, /// decoded to UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - if let Some(slot) = DICTS.get(variety(lang).0) { - *slot.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); - } - Ok(()) + slot(lang).install(aff, dic) } /// Forget every variety's loaded dictionary, so the next check looks for an installed one again /// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - for slot in &DICTS { - *slot.write().unwrap_or_else(|e| e.into_inner()) = None; + for slot in [&SPAIN_DICT, &MEXICO_DICT, &LATIN_AMERICA_DICT, &URUGUAY_DICT] { + slot.reload(); } } /// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder -/// or the system locations now (File › Options › Proofing shows it); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// or the system locations now (File › Options › Proofing shows it); `None` when there is none, +/// or when one given with [`install_dictionary`] is in use. pub fn installed_path(lang: ProofLang) -> Option { - crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic) + slot(lang).installed_path() } /// Is a dictionary available for `lang`'s variety (spelling is checked)? pub fn has_dictionary(lang: ProofLang) -> bool { - !dictionary(lang).is_empty() + slot(lang).has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? @@ -200,7 +177,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); for lang in VARIETIES { - let found = crate::hunspell::find_in(&dir, variety(lang).2); + let found = crate::hunspell::find_in(&dir, slot(lang).names()); assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}"); if let Some((aff, dic)) = found { assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang)); @@ -215,7 +192,7 @@ mod tests { std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); } for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] { - assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); + assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); assert_eq!(dictionary_name(lang), name); } std::fs::remove_dir_all(dir).unwrap(); diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index ee9d58e9..6372a47d 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -9,68 +9,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Croatian Hunspell dictionary. const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; +/// The Croatian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); + /// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed"); - return AffixDict::default(); - }; - - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Croatian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Croatian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Croatian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)? diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 012babc0..65b820df 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -9,74 +9,54 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by /// Java locales). const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; -/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number -/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. -const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; +/// The Indonesian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); /// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Indonesian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it); `None` when there is none. A -/// dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Indonesian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is an Indonesian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } +/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number +/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. +const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; + /// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)? fn is_latin_letter(c: char) -> bool { c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 027006a6..168883e1 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -18,67 +18,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Italian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of an Italian Hunspell dictionary (Italy first, then Switzerland). const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; +/// The Italian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); + /// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Italian spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Italian spelling: no Hunspell it_IT dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Italian spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Italian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is an Italian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)? diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 47216d72..7469c4d9 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -65,6 +65,7 @@ pub mod patterns; pub mod pl; pub mod pt; pub mod ru; +mod slot; pub mod tr; pub mod vi; @@ -1095,36 +1096,35 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish follows the four below; Russian, Czech, Indonesian and the Spanishes follow it. + // Every language is listed, Russian and Czech too (#3226: they were once missing, so + // neither showed up in File › Options › Proofing). A language reports the folder's file, + // or none while a dictionary given with install_dictionary() is in use (#3252 review); + // the languages' own tests install theirs concurrently, so an exact path would race them. + // The folder is searched first whatever is in use. let found = installed_dictionaries(); - assert_eq!(found.get(4), Some(&(ProofLang::Tr, Some(dir.join("tr_TR.dic"))))); + let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"]; assert_eq!( - found[7..], + found.iter().map(|(lang, _)| *lang).collect::>(), [ - (ProofLang::Id, Some(dir.join("id_ID.dic"))), - (ProofLang::Es, Some(dir.join("es_ES.dic"))), - (ProofLang::EsMx, Some(dir.join("es_MX.dic"))), - (ProofLang::Es419, Some(dir.join("es_419.dic"))), - (ProofLang::EsUy, Some(dir.join("es_UY.dic"))), + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy ] ); - assert_eq!( - found[..4], - [ - (ProofLang::Pl, Some(dir.join("pl_PL.dic"))), - (ProofLang::Pt, Some(dir.join("pt_BR.dic"))), - (ProofLang::It, Some(dir.join("it_IT.dic"))), - (ProofLang::Hr, Some(dir.join("hr_HR.dic"))), - ] - ); - // #3226: Russian and Czech were missing from installed_dictionaries() entirely (no - // installed_path() in either module), so neither ever showed up in File › Options › - // Proofing, installed or not. Their languages, not their exact path, is what belongs - // here: unlike the four above, installed_path() for Russian and Czech also reflects - // whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own - // test modules do, concurrently, in tests of their own — asserting an exact path here - // would race them for no reason this test needs to care about. - assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}"); + for ((lang, path), name) in found.iter().zip(names) { + let file = dir.join(format!("{name}.dic")); + assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}"); + assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}"); + } assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy)); assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 18c968fe..4d79986d 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -17,67 +17,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Polish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Polish Hunspell dictionary. const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; +/// The Polish dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); + /// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Polish spelling: no Hunspell pl_PL dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Polish spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Polish dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Polish dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index de419825..860cf457 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -16,67 +16,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Portuguese Hunspell dictionary. const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; +/// The Portuguese dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); + /// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() -} - -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Portuguese spelling: {e}"); - AffixDict::default() - }) + DICT.get() } /// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Portuguese dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Portuguese dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)? diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index d5035e6a..e46bf305 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -28,83 +28,47 @@ mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use crate::affix::AffixDict; +use crate::slot::DictSlot; pub use suggest::suggest; -/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on -/// disk (it then has no file of its own to report from [`installed_path`], even if a matching -/// file also happens to exist): one lock, so a reload and an install can't interleave their two -/// fields into an inconsistent combination. -static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); - -struct DictState { - dict: Option>, - in_memory: bool, -} - /// File names of a Russian Hunspell dictionary. const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; +/// The Russian dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); + /// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with -/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then; -/// see [`has_dictionary`]). +/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off +/// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.dict.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> AffixDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Russian spelling: no Hunspell ru_RU dictionary installed"); - return AffixDict::default(); - }; - crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { - log::warn!("Russian spelling: {e}"); - AffixDict::default() - }) -} - -/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), -/// for example one the user picked. On error the current dictionary stays. +/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = AffixDict::from_hunspell(aff, dic, &[])?; - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = Some(Arc::new(d)); - slot.in_memory = true; - Ok(()) + DICT.install(aff, dic) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.dict = None; - slot.in_memory = false; + DICT.reload(); } -/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and -/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a -/// matching file also happens to exist. +/// The `.dic` file of the installed Russian dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory { - return None; - } - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)? diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs new file mode 100644 index 00000000..3c0f2bd3 --- /dev/null +++ b/crates/proof/src/slot.rs @@ -0,0 +1,150 @@ +//! The installed Hunspell dictionary of a proofing language, in one shape for every language: +//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`]. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::affix::AffixDict; + +/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own. +pub(crate) trait Dictionary: Default + Send + Sync { + /// The installed `.aff` and `.dic` files, read. + fn load(aff: &Path, dic: &Path) -> Result; + /// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed. + fn parse(aff: &str, dic: &str) -> Result; + /// No words: the language has no spelling marks. + fn is_empty(&self) -> bool; +} + +impl Dictionary for AffixDict { + fn load(aff: &Path, dic: &Path) -> Result { + crate::hunspell::load(aff, dic) + } + + fn parse(aff: &str, dic: &str) -> Result { + AffixDict::from_hunspell(aff, dic, &[]) + } + + fn is_empty(&self) -> bool { + AffixDict::is_empty(self) + } +} + +/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on +/// disk (it then has no file to report, even if a matching file also happens to exist): one lock, +/// so a reload and an install can't interleave their two fields. +struct State { + dict: Option>, + in_memory: bool, +} + +/// One language's dictionary. +pub(crate) struct DictSlot { + /// File names the dictionary may have, the usual one first (also the key of its load error). + names: &'static [&'static str], + /// The language's English name, for the log. + english: &'static str, + state: RwLock>, +} + +impl DictSlot { + pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false }) } + } + + fn read(&self) -> RwLockReadGuard<'_, State> { + self.state.read().unwrap_or_else(PoisonError::into_inner) + } + + fn write(&self) -> RwLockWriteGuard<'_, State> { + self.state.write().unwrap_or_else(PoisonError::into_inner) + } + + /// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names. + pub(crate) fn name(&self) -> &'static str { + self.names.first().copied().unwrap_or_default() + } + + /// The file names the dictionary may have, the usual one first. + #[cfg(test)] + pub(crate) fn names(&self) -> &'static [&'static str] { + self.names + } + + /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or + /// an empty one when none is available (the language's spelling is off then). + pub(crate) fn get(&self) -> Arc { + if let Some(d) = self.read().dict.as_ref() { + return d.clone(); + } + let loaded = Arc::new(self.load_installed()); + // Another thread (or an install) may have filled it meanwhile: that one wins. + self.write().dict.get_or_insert(loaded).clone() + } + + fn load_installed(&self) -> D { + let name = self.name(); + let Some((aff, dic)) = crate::hunspell::find(self.names) else { + crate::hunspell::clear_load_error(name); + log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english); + return D::default(); + }; + crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("{} spelling: {e}", self.english); + D::default() + }) + } + + /// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for + /// example one the user picked. On error the current dictionary stays. + pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> { + let d = D::parse(aff, dic)?; + let mut slot = self.write(); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; + Ok(()) + } + + /// Forget the dictionary, so the next check looks for an installed one again (after + /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). + pub(crate) fn reload(&self) { + let mut slot = self.write(); + slot.dict = None; + slot.in_memory = false; + } + + /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the + /// system locations now (File › Options › Proofing shows it); `None` when there is none, or + /// when the dictionary in use was given with [`Self::install`] and so has no file. + pub(crate) fn installed_path(&self) -> Option { + if self.read().in_memory { + return None; + } + crate::hunspell::find(self.names).map(|(_, dic)| dic) + } + + /// Is a dictionary available (spelling is checked)? + pub(crate) fn has_dictionary(&self) -> bool { + !self.get().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static SLOT: DictSlot = DictSlot::new(&["zz_SLOT_TEST"], "Test"); + + #[test] + fn an_installed_dictionary_has_no_file_until_reloaded() { + assert!(!SLOT.has_dictionary(), "nothing installed under this name"); + assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(!SLOT.has_dictionary(), "a refused install keeps the current one"); + SLOT.install("", "1\nkot\n").unwrap(); + assert!(SLOT.has_dictionary() && SLOT.get().check("kot")); + assert_eq!(SLOT.installed_path(), None); + SLOT.reload(); + assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); + assert_eq!(SLOT.name(), "zz_SLOT_TEST"); + } +} diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index ed57cbca..b47ba7c1 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -26,6 +26,21 @@ pub struct TrDict { max_suffix_len: usize, } +impl crate::slot::Dictionary for TrDict { + fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let read = |p: &std::path::Path| std::fs::read_to_string(p).map_err(|e| e.to_string()); + TrDict::from_hunspell(&read(aff)?, &read(dic)?) + } + + fn parse(aff: &str, dic: &str) -> Result { + TrDict::from_hunspell(aff, dic) + } + + fn is_empty(&self) -> bool { + TrDict::is_empty(self) + } +} + impl TrDict { /// Returns true if empty (no dictionary loaded). pub fn is_empty(&self) -> bool { diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index ab3fe92d..eb5f0ea5 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -18,70 +18,45 @@ pub mod suggest; #[cfg(test)] pub(crate) mod test_dict; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; + +use crate::slot::DictSlot; pub use dict::TrDict; pub use suggest::suggest; -/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); - /// File names of a Turkish Hunspell dictionary. const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; +/// The Turkish dictionary, found on first use (see [`DictSlot`]). +static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); + /// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available. pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() { - return d.clone(); - } - let loaded = Arc::new(load_installed()); - let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner); - slot.get_or_insert(loaded).clone() + DICT.get() } -fn load_installed() -> TrDict { - let Some((aff, dic)) = crate::hunspell::find(NAMES) else { - crate::hunspell::clear_load_error(NAMES[0]); - log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); - return TrDict::default(); - }; - let read = |p: &std::path::Path| std::fs::read_to_string(p); - match (read(&aff), read(&dic)) { - (Ok(aff_text), Ok(dic_text)) => crate::hunspell::note_load(NAMES[0], &dic, TrDict::from_hunspell(&aff_text, &dic_text)).unwrap_or_else(|e| { - log::warn!("Turkish spelling: {e}"); - TrDict::default() - }), - (a, d) => { - let reason = a.err().or(d.err()).map_or_else(String::new, |e| e.to_string()); - let _ = crate::hunspell::note_load::<()>(NAMES[0], &dic, Err(reason)); - TrDict::default() - } - } -} - -/// Use this Turkish dictionary from now on. +/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the +/// current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { - let d = TrDict::from_hunspell(aff, dic)?; - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d)); - Ok(()) + DICT.install(aff, dic) } -/// Reload the dictionary on next check. +/// Forget the loaded dictionary, so the next check looks for an installed one again. pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None; + DICT.reload(); } -/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the -/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none. -/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// The `.dic` file of the installed Turkish dictionary (File › Options › Proofing shows it); +/// `None` when there is none, or when one given with [`install_dictionary`] is in use. pub fn installed_path() -> Option { - crate::hunspell::find(NAMES).map(|(_, dic)| dic) + DICT.installed_path() } /// Is a Turkish dictionary available? pub fn has_dictionary() -> bool { - !dictionary().is_empty() + DICT.has_dictionary() } /// Is `word` spelled correctly in Turkish? From 172c935fca54e5791777d3ad68bf23b796e1c8e0 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:48:47 -0100 Subject: [PATCH 4/9] Proof tests: the user-dictionary file test waits for the dictionary tests user_dictionary_persists_through_a_file saves the whole user dictionary to a file and loads it back. Run beside user_dictionary_and_ignore_all_match_both_normalization_forms, it could catch a word that test had added and bring it back after the test removed it, so the other test's last check failed now and then (once in about two dozen full runs here). It now holds DICTIONARY_TESTS, as the other test does. --- crates/proof/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 7469c4d9..ad516b29 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1059,7 +1059,9 @@ mod tests { use super::*; - /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test. + /// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land + /// mid-test, and by those that save the whole user dictionary and load it back, which would + /// bring back a word another test just removed. static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// #1063: the dictionaries folder is searched before the system locations; the dictionaries @@ -1579,6 +1581,7 @@ mod tests { /// #40: the user dictionary survives a save and a fresh load. #[test] fn user_dictionary_persists_through_a_file() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist")); let _ = std::fs::remove_dir_all(&dir); // The folder is created on the first save. From c19b1cae73bfd7dd78609f5ba4795795e77ae64c Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:49:22 -0100 Subject: [PATCH 5/9] Proofing: one suggestion engine with per-language tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Czech's dictionary-driven ranking (case, MAP, REP, missing spaces, KEY, TRY edits, a stem scan) had five older copies. Polish, Portuguese, Croatian, Italian and Russian each kept one: 300 to 400 lines of the same rounds with a few lines of language knowledge mixed in. Fixes reached some copies only. NOSUGGEST (storytold/wordcraft#1048) and the first-difference tie-break (storytold/wordcraft#797) never got to all of them, and none had the KEY round or the bound on TRY letters. affix_suggest.rs is that ranking once. A language's `Tuning` holds what is its own: the short words that run into their neighbours (Polish, Portuguese, Croatian, Italian), Russian's split rule, the letter groups and confusions its dictionaries lack (Croatian, Russian), letters to try for a dictionary without a TRY line, and cost hooks for Polish final devoicing and Russian е/ё. Czech, Indonesian and Spanish use the dictionary's tables alone. Turkish keeps its own suggester, which reads another dictionary type with rounds of its own. The language tables add to the dictionary's instead of replacing them, and TRY letters are tried in the dictionary's order, commonest first, as the copies did (the Czech engine sorted them). Old and new compared on the test dictionaries, with every one-letter typo of each word plus the tests' misspellings, top eight suggestions: Czech, Croatian, Italian and Portuguese give the same for all 7,903 inputs. Polish differs for 44 of 3,244 and Russian for 33 of 2,346, all orderings among candidates of equal cost, from the tie-break they now share (`ie` → `nie` before `i`). New tests pin the moved knowledge: the tuning mechanism, Croatian's groups without MAP lines, and Russian's costs and split rule. Turning each knob off fails one. Proof: 283 tests. --- crates/proof/src/affix_suggest.rs | 493 ++++++++++++++++++++++++++++++ crates/proof/src/cs/suggest.rs | 358 +--------------------- crates/proof/src/es/suggest.rs | 8 +- crates/proof/src/hr/suggest.rs | 351 ++------------------- crates/proof/src/id/suggest.rs | 9 +- crates/proof/src/it/suggest.rs | 350 +-------------------- crates/proof/src/lib.rs | 1 + crates/proof/src/pl/suggest.rs | 361 ++-------------------- crates/proof/src/pt/suggest.rs | 356 +-------------------- crates/proof/src/ru/suggest.rs | 391 ++++-------------------- 10 files changed, 625 insertions(+), 2053 deletions(-) create mode 100644 crates/proof/src/affix_suggest.rs diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs new file mode 100644 index 00000000..1da35559 --- /dev/null +++ b/crates/proof/src/affix_suggest.rs @@ -0,0 +1,493 @@ +//! Spelling suggestions from an affix dictionary, one engine for every language that reads one +//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come +//! in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`): +//! adding a missing accent is the commonest slip and ranks first, while taking away one the +//! writer typed ranks after the known confusions; +//! 3. the REP table of common confusions (`żeka` → `rzeka`); +//! 4. a missing space (`napewno` → `na pewno`); +//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with +//! the next or mistyped (TRY letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in +//! which a MAP slip costs half (two typos at most). +//! +//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may +//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every +//! round, so a call costs a few thousand dictionary lookups at most. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters. +pub(crate) const MAX_LETTERS: usize = 40; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +pub(crate) const COST_ADD_DIACRITIC: u32 = 4; +pub(crate) const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +pub(crate) const COST_SPLIT: u32 = 12; +/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]). +const COST_SPLIT_SHORT: u32 = 17; +/// A missing space between two longer halves with no short word beside it ([`Split::Near`]). +pub(crate) const COST_SPLIT_LONG: u32 = 18; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +pub(crate) const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +/// What a language adds to its dictionary's tables. +pub(crate) struct Tuning { + /// Where a missing space may go (round 4). + pub split: Split, + /// TRY letters for a dictionary without a TRY line. + pub alphabet: &'static str, + /// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`). + pub map: &'static [&'static str], + /// REP confusions the language adds to its dictionary's. + pub rep: &'static [(&'static str, &'static str)], + /// The cost of a REP confusion. + pub rep_cost: u32, + /// Most letters a MAP variant changes. + pub max_map_changes: usize, + /// The cost of changing letter `from` to its MAP relative `to`. + pub map_cost: fn(char, char) -> u32, + /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. + pub mistype_cost: fn(&[char], usize, char) -> u32, +} + +/// Where a missing space may go. +pub(crate) enum Split { + /// Anywhere, dearer when a half is shorter than three letters. + Anywhere, + /// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one + /// that runs into the word before (`kocham cię`), dearer between two longer halves, and never + /// leaving a stray short half (`że ka`). + Near { leading: &'static [&'static str], trailing: &'static [&'static str] }, + /// The language's own rule: the cost of the halves `a b`, or `None` to skip them. + Custom(fn(&str, &str) -> Option), +} + +impl Tuning { + /// The dictionary's tables alone (Czech, Indonesian, Spanish). + pub(crate) const DEFAULT: Tuning = + Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type is likelier right, +/// so taking one away costs more than a known confusion from the REP table. +pub(crate) fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Any mistyped letter costs the same. +fn mistype_cost(_: &[char], _: usize, _: char) -> u32 { + COST_MISTYPE +} + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + tested: HashSet, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + return; + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP groups: the dictionary's, then the language's. +fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec]> { + if t.map.is_empty() { + return Cow::Borrowed(d.map()); + } + Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect()) +} + +/// The other members of every MAP group of `c` (its likely confusions), each once. +fn alternatives(c: char, map: &[Vec]) -> Vec { + let mut out: Vec = Vec::new(); + for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) { + if x != c && !out.contains(&x) { + out.push(x); + } + } + out +} + +/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, +/// letter by letter), with its cost, each written after `buf`. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some((&c, rest)) = chars.split_first() else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); + buf.push(c); + map_variants(rest, later, t, changes, cost, buf, out); + buf.pop(); + if changes < t.max_map_changes { + for &a in here { + buf.push(a); + map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`. +pub(crate) fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered. +fn split_cost(split: &Split, a: &str, b: &str) -> Option { + let long = |part: &str| part.chars().count() >= 3; + match split { + Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }), + Split::Near { leading, trailing } => { + if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } + } + Split::Custom(cost) => cost(a, b), + } +} + +/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case. +pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + // 1. Letter case. + if d.contains(&input) && word != input { + col.keep(input.clone(), COST_CASE); + } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { + col.keep(capitalize(&input), COST_CASE); + } + // 2. Accents and other MAP slips. + let map = groups(d, t); + let alts: Vec> = chars.iter().map(|&c| alternatives(c, &map)).collect(); + let mut variants = Vec::new(); + map_variants(&chars, &alts, t, 0, 0, &mut Vec::with_capacity(n), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } + // 3. Common confusions. + let rep = d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } + // 4. A missing space; both halves must be dictionary words. + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(&t.split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } + // 5. Keyboard neighbours from the dictionary's KEY rows. + for row in d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } + // One typo, with the TRY letters (the language's when the dictionary has none), bounded and + // each once, in their order: a TRY line lists the commonest letters first, so equal-cost + // candidates come out in that order. + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.clone(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.clone(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.clone(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(&chars, i, c)); + } + } + } + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars, &map); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let t = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text, + }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { + let d = col.d; + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + #[test] + fn distance_weighs_map_slips_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY + /// line, and a split rule decides which missing spaces are offered. + #[test] + fn tuning_adds_to_the_dictionary_tables() { + let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap(); + let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT }; + assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into())); + assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group"); + assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair"); + assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters"); + let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT }; + assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into())); + assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half"); + assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into())); + } +} diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 4e5ecd17..05eba74a 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -1,168 +1,8 @@ -//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits -//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian -//! ([`crate::id`]) uses the same ranking with its own dictionary. +//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed +//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Longest word suggestions are looked for, in letters (Spanish shares the bound). -pub(crate) const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 17; -const COST_KEY: u32 = 13; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (a capitalized name). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (typing without diacritics); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; /// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -171,201 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space; both halves must be dictionary words. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |part: &str| part.chars().count() >= 3; - let cost = if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_RARE }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. Keyboard neighbours from the installed dictionary's KEY rows. - for row in d.key().split('|').take(32) { - let keys: Vec = row.chars().take(128).collect(); - for pair in keys.windows(2) { - let [a, b] = pair else { continue }; - for (i, &c) in chars.iter().enumerate() { - let replacement = if c == *a { - Some(*b) - } else if c == *b { - Some(*a) - } else { - None - }; - if let Some(replacement) = replacement { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = replacement; - } - col.offer(&v.iter().collect::(), COST_KEY); - } - } - } - } - // One-letter edits use only the runtime TRY alphabet, with bounded work. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).take(128).collect(); - letters.sort_unstable(); - letters.dedup(); - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (the first changed letter breaks ties). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) } #[cfg(test)] mod tests { use super::*; - #[test] - fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { - let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); - let suggestions = suggest_in(&d, "počítačz", 8); - assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); - assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); - } - #[test] fn suggestions_are_fast_enough() { crate::cs::test_dict::install(); diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs index b42efa2e..0568ae87 100644 --- a/crates/proof/src/es/suggest.rs +++ b/crates/proof/src/es/suggest.rs @@ -1,5 +1,5 @@ -//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to +//! Spanish suggestions. The ranking is the shared dictionary-driven one +//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to //! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, //! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. //! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first @@ -7,7 +7,7 @@ use crate::ProofLang; use crate::affix::{AffixDict, Casing, capitalize, casing}; -use crate::cs::suggest::MAX_LETTERS; +use crate::affix_suggest::{MAX_LETTERS, Tuning}; /// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's /// case. @@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec _ => v, }); } - for s in crate::cs::suggest::suggest_in(d, word, max) { + for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) { if !out.contains(&s) { out.push(s); } diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index d6f8159a..d8c19b99 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -1,12 +1,11 @@ -//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance, -//! missing spaces and a bounded search. Proper Croatian corrections depend on -//! the actual installed dictionary entries. +//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of +//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines. -use std::collections::{HashMap, HashSet}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Short words often run into the next one (`não é`, `da água`, `para isso`). +/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`). const LEADING_WORDS: &[&str] = &[ "a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će", "bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad", @@ -14,162 +13,13 @@ const LEADING_WORDS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Croatian adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiourntsjkvlmdpbgczčćšžđfh", + map: &["cčć", "dđ", "sš", "zž"], + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -178,173 +28,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']]; - let map = if d.map().is_empty() { &fallback_map } else { d.map() }; - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -354,6 +38,13 @@ mod tests { fn nosuggest_stems_and_forms_are_never_offered() { crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); } + /// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća` + /// first, before a typo that costs more (`kuha`). + #[test] + fn diacritics_without_map_lines() { + let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap(); + assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]); + } #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); diff --git a/crates/proof/src/id/suggest.rs b/crates/proof/src/id/suggest.rs index 562af9a8..b42efde9 100644 --- a/crates/proof/src/id/suggest.rs +++ b/crates/proof/src/id/suggest.rs @@ -1,7 +1,6 @@ -//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem -//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is -//! hard-coded here. +//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing +//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID` +//! dictionary, so nothing Indonesian is hard-coded here. use crate::affix::AffixDict; @@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - crate::cs::suggest::suggest_in(d, word, max) + crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT) } #[cfg(test)] diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 6a043ff2..55f86909 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -1,25 +1,9 @@ -//! Italian spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`roma` → `Roma`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`, -//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`); -//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`per me`, `a Milano`, `di Roma`). const LEADING_WORDS: &[&str] = &[ @@ -35,161 +19,12 @@ const TRAILING_WORDS: &[&str] = &[ "mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove", "è", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; +/// What Italian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `à` → `a`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} /// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -198,176 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent. - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index ad516b29..4bf4da78 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -50,6 +50,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod affix; +mod affix_suggest; pub mod bg; pub mod cs; pub mod dict; diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs index 0e32a1ea..7baeaa54 100644 --- a/crates/proof/src/pl/suggest.rs +++ b/crates/proof/src/pl/suggest.rs @@ -1,26 +1,9 @@ -//! Polish spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`warszawa` → `Warszawa`); -//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy` -//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most -//! common slip, so adding them ranks first, while taking away a diacritic the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` → -//! `huśtawka`, `pszyjaciel` → `przyjaciel`); -//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham -//! cię`), cheaper when one half is a short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which a diacritic slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand -//! dictionary lookups at most. +//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`), +//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning}; /// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede /// wszystkim`). @@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[ /// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). const TRAILING_WORDS: &[&str] = &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; +/// What Polish adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx", + mistype_cost, + ..Tuning::DEFAULT +}; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moze` → `może`, then - // `Mozę`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the -/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the -/// writer did type is likelier right, so taking one away costs more than a known confusion from -/// the REP table (`żeka` → `rzeka` before `zeka`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) +/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it +/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo. +fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 { + let last = i + 1 == chars.len(); + if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE } } /// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. @@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Diacritics and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half (`że ka`). - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - // Final devoicing can make a voiced consonant sound voiceless. - let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g')); - let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }; - col.offer(&s(&v), cost); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs index 36f1c7b0..f8f33bff 100644 --- a/crates/proof/src/pt/suggest.rs +++ b/crates/proof/src/pt/suggest.rs @@ -1,26 +1,9 @@ -//! Portuguese spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`brasília` → `Brasília`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`, -//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` → -//! `exceção`, `previlégio` → `privilégio`); -//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`não é`, `da água`, `para isso`). const LEADING_WORDS: &[&str] = &[ @@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[ "me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si", "eu", "tu", "que", "de", "em", "é", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Portuguese adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw", + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index 24eeb764..135f4152 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -1,30 +1,14 @@ -//! Russian spelling suggestions, best first. -//! -//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the -//! common confusions are defined here. Candidates come in rounds of rising cost, each checked -//! against the dictionary: -//! 1. letter case (`москва` → `Москва`); -//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`, -//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to -//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`); -//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or -//! doubled consonant, `здраствуй` → `здравствуй`); -//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped; -//! 6. when that finds fewer than asked for: the stems that start like the word and share the -//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within -//! a letter group costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round. +//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP` +//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` → +//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule +//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning}; /// Letters confused with each other: a slip inside a group is cheaper than another typo. const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"]; - /// Common confusions: a written sequence and what was meant. const REP: &[(&str, &str)] = &[ ("жы", "жи"), @@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[ ("ц", "тс"), ("тс", "ц"), ]; - /// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`). const LEADING_WORDS: &[&str] = &[ "не", @@ -123,161 +106,42 @@ const PREPOSITIONS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"]; /// The Russian alphabet, when the dictionary has no `TRY` line. const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё"; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 3; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; +/// `е` written for `ё`, or the other way round, is barely a slip. const COST_YO: u32 = 3; -const COST_MAP: u32 = 7; -const COST_REP: u32 = 9; -const COST_SPLIT: u32 = 12; +/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`). const COST_SPLIT_PREPOSITION: u32 = 17; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} +/// What Russian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT }; -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The other members of each [`MAP`] group holding `c`. -fn alternatives(c: char) -> Vec { - let mut out: Vec = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect(); - out.sort_unstable(); - out.dedup(); - out -} - -/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip -/// (`ё` is optional in ordinary writing). +/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest. fn map_cost(from: char, to: char) -> u32 { - if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP } + if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP } } -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; +/// A missing space: cheap after a short leading word or before a short trailing one, dearer after +/// a short preposition or between two longer words, never leaving a stray short half. +fn split(a: &str, b: &str) -> Option { + let long = |p: &str| p.chars().count() >= 3; + // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for + // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. + let preposition = PREPOSITIONS.contains(&a); + if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { + return None; } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } + if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { + Some(COST_SPLIT_PREPOSITION) + } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None } } -/// A letter's group representative (the first of its first group), lower case: `ё` → `е`, -/// `з` → `з`, `с` → `з`. -fn fold(c: char) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l) -} - -/// Optimal string alignment distance where a slip within a letter group costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if fold(ai) == fold(bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} - /// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { suggest_in(&super::dictionary(), word, max) @@ -285,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Slips within a letter group. - let alts: Vec> = chars.iter().map(|&c| alternatives(c)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in REP { - for (i, _) in input.match_indices(from) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for - // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. - let preposition = PREPOSITIONS.contains(&a); - if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { - continue; - } - let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { - COST_SPLIT_PREPOSITION - } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = ALPHABET.chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a group relative of it, -/// either case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(fold).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -461,6 +161,20 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip + /// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip; + /// a preposition is never split off a verb, and after a one-letter one a typo is likelier. + #[test] + fn russian_costs_rank_its_slips() { + let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap(); + assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]); + assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]); + assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into())); + let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4); + assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}"); + assert!(s.contains(&"в доме".into()), "{s:?}"); + } + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the /// shared Latin-script check doesn't reach it. @@ -518,12 +232,15 @@ mod tests { #[test] fn distance_weighs_letter_groups_half() { + use crate::affix_suggest::{fold, weighted_distance}; + let map: Vec> = MAP.iter().map(|g| g.chars().collect()).collect(); + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); let c = |s: &str| s.chars().collect::>(); - assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1); - assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1); - assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2); - assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6); - assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3); + assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1); + assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1); + assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2); + assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6); + assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3); } #[test] From a388c4787c09c33385de55f4e23897f6b75861ef Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:55:43 -0100 Subject: [PATCH 6/9] Proofing: one language table instead of nine lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib.rs named every dictionary language in nine places: reloading for a new folder, the installed list, load errors, the missing-dictionary hint, the loaded check, the spelling message, spelling, suggestions and grammar. The lists had drifted. dictionary_load_error had no Russian or Czech entry, so a Russian or Czech dictionary that failed to load showed no reason in File › Options › Proofing, though the status bar, which reads another list, said why. LANGUAGES holds one row per language: its dictionary slot, its spelling message, and its spelling, suggestion and grammar functions. The nine functions read the row, so a language now joins with its module, its ProofLang and one row. The per-language reload_dictionary and installed_path wrappers had no callers left, inside the crate or out, and are gone. Tests: every_dictionary_language_reports_its_load_error, which fails for Russian on the old code, and the_language_table_covers_the_proofed_languages (each ProofLang::PROOFED language but English has exactly one row). Proof 285, engine 927 and layout 501 (lib). --- crates/proof/src/cs/mod.rs | 14 +- crates/proof/src/es/mod.rs | 23 +--- crates/proof/src/hr/mod.rs | 14 +- crates/proof/src/id/mod.rs | 14 +- crates/proof/src/it/mod.rs | 14 +- crates/proof/src/lib.rs | 274 ++++++++++++++++++++++--------------- crates/proof/src/pl/mod.rs | 14 +- crates/proof/src/pt/mod.rs | 14 +- crates/proof/src/ru/mod.rs | 14 +- crates/proof/src/slot.rs | 31 +++++ crates/proof/src/tr/mod.rs | 13 +- 11 files changed, 208 insertions(+), 231 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 55e21aef..1d72dc91 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -18,7 +18,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; /// The Czech dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Czech"); /// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off @@ -33,18 +33,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Czech dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 253642c8..e16e9d6b 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -25,10 +25,10 @@ use crate::slot::DictSlot; pub use suggest::suggest; /// The dictionary of each variety, found on first use (see [`DictSlot`]). -static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); -static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); -static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); -static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); +pub(crate) static SPAIN_DICT: DictSlot = DictSlot::new(SPAIN, "Spanish (Spain)"); +pub(crate) static MEXICO_DICT: DictSlot = DictSlot::new(MEXICO, "Spanish (Mexico)"); +pub(crate) static LATIN_AMERICA_DICT: DictSlot = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)"); +pub(crate) static URUGUAY_DICT: DictSlot = DictSlot::new(URUGUAY, "Spanish (Uruguay)"); /// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the /// generic one RLA-ES publishes). @@ -72,21 +72,6 @@ pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), S slot(lang).install(aff, dic) } -/// Forget every variety's loaded dictionary, so the next check looks for an installed one again -/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - for slot in [&SPAIN_DICT, &MEXICO_DICT, &LATIN_AMERICA_DICT, &URUGUAY_DICT] { - slot.reload(); - } -} - -/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder -/// or the system locations now (File › Options › Proofing shows it); `None` when there is none, -/// or when one given with [`install_dictionary`] is in use. -pub fn installed_path(lang: ProofLang) -> Option { - slot(lang).installed_path() -} - /// Is a dictionary available for `lang`'s variety (spelling is checked)? pub fn has_dictionary(lang: ProofLang) -> bool { slot(lang).has_dictionary() diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 6372a47d..036b7235 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -20,7 +20,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"]; /// The Croatian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Croatian"); /// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off @@ -35,18 +35,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Croatian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Croatian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 65b820df..f1e44df4 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -21,7 +21,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"]; /// The Indonesian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Indonesian"); /// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off @@ -36,18 +36,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Indonesian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is an Indonesian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 168883e1..1e606ad7 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -29,7 +29,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"]; /// The Italian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Italian"); /// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Italian spelling is off @@ -44,18 +44,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Italian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is an Italian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 4bf4da78..0c51e236 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -159,57 +159,150 @@ pub fn ignore_word(w: &str) { } } +/// A language proofed against an installed Hunspell dictionary: all the functions below need to +/// know about it, so a language joins with one row in [`LANGUAGES`]. +struct Language { + lang: ProofLang, + slot: &'static dyn slot::Slot, + /// The spelling mark's message, in the language. + message: &'static str, + is_correct: fn(&str, ProofLang, &Options) -> bool, + suggest: fn(&str, usize, ProofLang) -> Vec, + /// Grammar issues; the flag says whether the first word must have a capital (not in a list + /// item). + grammar: fn(&str, ProofLang, bool) -> Vec, +} + +/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing +/// lists them. English has a built-in word list and is handled apart. +static LANGUAGES: [Language; 12] = [ + Language { + lang: ProofLang::Pl, + slot: &pl::DICT, + message: "Możliwy błąd pisowni", + is_correct: |w, _, o| pl::is_correct_with_options(w, o), + suggest: |w, n, _| pl::suggest(w, n), + grammar: |t, _, c| pl::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Pt, + slot: &pt::DICT, + message: "Possível erro de ortografia", + is_correct: |w, _, o| pt::is_correct_with_options(w, o), + suggest: |w, n, _| pt::suggest(w, n), + grammar: |t, _, c| pt::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::It, + slot: &it::DICT, + message: "Possibile errore di ortografia", + is_correct: |w, _, o| it::is_correct_with_options(w, o), + suggest: |w, n, _| it::suggest(w, n), + grammar: |t, _, c| it::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Hr, + slot: &hr::DICT, + message: "Moguća pravopisna pogreška", + is_correct: |w, _, o| hr::is_correct_with_options(w, o), + suggest: |w, n, _| hr::suggest(w, n), + grammar: |t, _, c| hr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Tr, + slot: &tr::DICT, + message: "Olası yazım hatası", + is_correct: |w, _, o| tr::is_correct_with_options(w, o), + suggest: |w, n, _| tr::suggest(w, n), + grammar: |t, _, c| tr::grammar::check_with(t, c), + }, + Language { + lang: ProofLang::Ru, + slot: &ru::DICT, + message: "Возможна орфографическая ошибка", + is_correct: |w, _, o| ru::is_correct_with_options(w, o), + suggest: |w, n, _| ru::suggest(w, n), + grammar: |t, _, c| ru::grammar::check_with(t, c), + }, + // Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no + // exception. + Language { + lang: ProofLang::Cs, + slot: &cs::DICT, + message: "Možná pravopisná chyba", + is_correct: |w, _, o| cs::is_correct_with_options(w, o), + suggest: |w, n, _| cs::suggest(w, n), + grammar: |t, _, _| cs::grammar::check(t), + }, + Language { + lang: ProofLang::Id, + slot: &id::DICT, + message: "Kemungkinan salah eja", + is_correct: |w, _, o| id::is_correct_with_options(w, o), + suggest: |w, n, _| id::suggest(w, n), + grammar: |t, _, _| id::grammar::check(t), + }, + Language { + lang: ProofLang::Es, + slot: &es::SPAIN_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsMx, + slot: &es::MEXICO_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::Es419, + slot: &es::LATIN_AMERICA_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, + Language { + lang: ProofLang::EsUy, + slot: &es::URUGUAY_DICT, + message: "Posible error ortográfico", + is_correct: es::is_correct_with_options, + suggest: es::suggest, + grammar: |t, l, _| es::grammar::check(t, l), + }, +]; + +/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof. +fn language(lang: ProofLang) -> Option<&'static Language> { + LANGUAGES.iter().find(|l| l.lang == lang) +} + /// Search `dir` for Hunspell dictionaries before the system locations (File › Options › /// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from /// the new search path, and text is checked again. pub fn set_dictionaries_folder(dir: Option) { hunspell::set_dictionary_dir(dir); - pl::reload_dictionary(); - pt::reload_dictionary(); - it::reload_dictionary(); - hr::reload_dictionary(); - tr::reload_dictionary(); - ru::reload_dictionary(); - cs::reload_dictionary(); - id::reload_dictionary(); - es::reload_dictionary(); + for l in &LANGUAGES { + l.slot.reload(); + } DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } /// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it /// now (`None`: not installed, so the language has no spelling marks). pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> { - vec![ - (ProofLang::Pl, pl::installed_path()), - (ProofLang::Pt, pt::installed_path()), - (ProofLang::It, it::installed_path()), - (ProofLang::Hr, hr::installed_path()), - (ProofLang::Tr, tr::installed_path()), - (ProofLang::Ru, ru::installed_path()), - (ProofLang::Cs, cs::installed_path()), - (ProofLang::Id, id::installed_path()), - (ProofLang::Es, es::installed_path(ProofLang::Es)), - (ProofLang::EsMx, es::installed_path(ProofLang::EsMx)), - (ProofLang::Es419, es::installed_path(ProofLang::Es419)), - (ProofLang::EsUy, es::installed_path(ProofLang::EsUy)), - ] + LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect() } /// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse /// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has /// checked yet reports `None` (#2830). pub fn dictionary_load_error(lang: ProofLang) -> Option { - let key = match lang { - ProofLang::Pl => "pl_PL", - ProofLang::Pt => "pt_BR", - ProofLang::It => "it_IT", - ProofLang::Hr => "hr_HR", - ProofLang::Tr => "tr_TR", - ProofLang::Id => "id_ID", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang), - _ => return None, - }; - hunspell::load_error(key).map(|(_, reason)| reason) + hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason) } /// A number that changes whenever a word is added to or removed from the user dictionary, or the @@ -440,36 +533,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p /// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English). fn dictionary_loaded(lang: ProofLang) -> Option { - Some(match lang { - ProofLang::Pl => pl::has_dictionary(), - ProofLang::Pt => pt::has_dictionary(), - ProofLang::It => it::has_dictionary(), - ProofLang::Hr => hr::has_dictionary(), - ProofLang::Tr => tr::has_dictionary(), - ProofLang::Ru => ru::has_dictionary(), - ProofLang::Cs => cs::has_dictionary(), - ProofLang::Id => id::has_dictionary(), - l if l.is_spanish() => es::has_dictionary(l), - _ => return None, - }) + language(lang).map(|l| l.slot.has_dictionary()) } /// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for; /// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`). fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> { - match lang { - _ if installed => None, - ProofLang::Pl => Some("pl_PL"), - ProofLang::Pt => Some("pt_BR"), - ProofLang::It => Some("it_IT"), - ProofLang::Hr => Some("hr_HR"), - ProofLang::Tr => Some("tr_TR"), - ProofLang::Ru => Some("ru_RU"), - ProofLang::Cs => Some("cs_CZ"), - ProofLang::Id => Some("id_ID"), - l if l.is_spanish() => Some(es::dictionary_name(l)), - _ => None, - } + language(lang).filter(|_| !installed).map(|l| l.slot.name()) } /// Is `word` spelled correctly in the run's proofing language? A language without a dictionary @@ -739,18 +809,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { return None; } - let message = match lang { - ProofLang::Pl => "Możliwy błąd pisowni", - ProofLang::Hr => "Moguća pravopisna pogreška", - ProofLang::Pt => "Possível erro de ortografia", - ProofLang::It => "Possibile errore di ortografia", - ProofLang::Tr => "Olası yazım hatası", - ProofLang::Ru => "Возможна орфографическая ошибка", - ProofLang::Cs => "Možná pravopisná chyba", - ProofLang::Id => "Kemungkinan salah eja", - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico", - _ => "Possible spelling mistake", - }; + let message = language(lang).map_or("Possible spelling mistake", |l| l.message); Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) } @@ -774,40 +833,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> { /// checked as its composed form ([`composed`]). pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts), - ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts), - ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts), - ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts), - ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts), - ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct_with_options(word, opts), - ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct_with_options(word, opts), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => { - accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts) - } - ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts), - ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts), - ProofLang::Bg | ProofLang::De | ProofLang::Other => true, + if lang.is_english() { + return is_correct_english(word, lang, opts); } + // A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks. + language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts)) } /// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed /// ([`composed`]) like the dictionaries' words. pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { let word = &*composed(word); - match lang { - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang), - ProofLang::Pl => pl::suggest(word, max), - ProofLang::Hr => hr::suggest(word, max), - ProofLang::Pt => pt::suggest(word, max), - ProofLang::It => it::suggest(word, max), - ProofLang::Tr => tr::suggest(word, max), - ProofLang::Ru => ru::suggest(word, max), - ProofLang::Cs => cs::suggest(word, max), - ProofLang::Id => id::suggest(word, max), - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + if lang.is_english() { + return suggest_english(word, max, lang); } + language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang)) } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones @@ -824,18 +864,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), - ProofLang::Pl => pl::grammar::check_with(text, capital_first), - ProofLang::Hr => hr::grammar::check_with(text, capital_first), - ProofLang::Pt => pt::grammar::check_with(text, capital_first), - ProofLang::It => it::grammar::check_with(text, capital_first), - ProofLang::Tr => tr::grammar::check_with(text, capital_first), - ProofLang::Ru => ru::grammar::check_with(text, capital_first), - // Czech has no sentence-capital rule, so a list item needs no exception. - ProofLang::Cs => cs::grammar::check(text), - ProofLang::Id => id::grammar::check(text), - // Spanish has no sentence-capital rule either. - ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang), - ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), + _ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)), } } @@ -1133,6 +1162,33 @@ mod tests { assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); } + /// A dictionary that fails to load says why in File › Options › Proofing in every language + /// that reads one; Russian and Czech were missing from the list that named their dictionaries + /// there. + #[test] + fn every_dictionary_language_reports_its_load_error() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for (lang, _) in installed_dictionaries() { + let name = dictionary_needed(lang, false).unwrap(); + let reason = format!("zq unreadable {name}"); + let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone())); + assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}"); + hunspell::clear_load_error(name); + } + } + + /// Every proofed language other than English has a row in the language table, and only those. + #[test] + fn the_language_table_covers_the_proofed_languages() { + for l in ProofLang::PROOFED { + assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}"); + } + for (i, row) in LANGUAGES.iter().enumerate() { + assert!(row.lang.is_proofed(), "{:?}", row.lang); + assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang); + } + } + /// #1741: a list item may be a fragment, so its first word needs no capital in any language /// with the rule; a plain paragraph's does, and later sentences in the item are still checked. #[test] diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 4d79986d..30d6d966 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -28,7 +28,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; /// The Polish dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Polish"); /// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Polish spelling is off @@ -43,18 +43,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Polish dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Polish dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index 860cf457..0110a8c1 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -27,7 +27,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"]; /// The Portuguese dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Portuguese"); /// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off @@ -42,18 +42,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Portuguese dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Portuguese dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index e46bf305..67be7085 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -39,7 +39,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; /// The Russian dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Russian"); /// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available (Russian spelling is off @@ -54,18 +54,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again (after -/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Russian dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { DICT.has_dictionary() diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs index 3c0f2bd3..8b90a94a 100644 --- a/crates/proof/src/slot.rs +++ b/crates/proof/src/slot.rs @@ -30,6 +30,37 @@ impl Dictionary for AffixDict { } } +/// What every language's slot answers, whatever its dictionary type (the language table in +/// lib.rs holds them side by side). +pub(crate) trait Slot: Sync { + /// See [`DictSlot::name`]. + fn name(&self) -> &'static str; + /// See [`DictSlot::installed_path`]. + fn installed_path(&self) -> Option; + /// See [`DictSlot::has_dictionary`]. + fn has_dictionary(&self) -> bool; + /// See [`DictSlot::reload`]. + fn reload(&self); +} + +impl Slot for DictSlot { + fn name(&self) -> &'static str { + DictSlot::name(self) + } + + fn installed_path(&self) -> Option { + DictSlot::installed_path(self) + } + + fn has_dictionary(&self) -> bool { + DictSlot::has_dictionary(self) + } + + fn reload(&self) { + DictSlot::reload(self); + } +} + /// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on /// disk (it then has no file to report, even if a matching file also happens to exist): one lock, /// so a reload and an install can't interleave their two fields. diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index eb5f0ea5..f247336b 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -29,7 +29,7 @@ pub use suggest::suggest; const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; /// The Turkish dictionary, found on first use (see [`DictSlot`]). -static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); +pub(crate) static DICT: DictSlot = DictSlot::new(NAMES, "Turkish"); /// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with /// [`install_dictionary`], or an empty one when none is available. @@ -43,17 +43,6 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { DICT.install(aff, dic) } -/// Forget the loaded dictionary, so the next check looks for an installed one again. -pub fn reload_dictionary() { - DICT.reload(); -} - -/// The `.dic` file of the installed Turkish dictionary (File › Options › Proofing shows it); -/// `None` when there is none, or when one given with [`install_dictionary`] is in use. -pub fn installed_path() -> Option { - DICT.installed_path() -} - /// Is a Turkish dictionary available? pub fn has_dictionary() -> bool { DICT.has_dictionary() From 140c7041e1c9cd4fe1459f339c7565f9dc1df52f Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 19:58:30 -0100 Subject: [PATCH 7/9] Proofing: the spelling options are read in one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Czech, Croatian, Indonesian, Italian, Polish, Portuguese and Spanish began their spelling checks with the same thirteen lines, byte for byte: no dictionary means no marks, single letters and words without letters pass, words with digits and short words in capitals pass when the File › Options › Proofing choices say so, and words in another script pass. Each also had its own copy of the Latin-script test. These were the lines storytold/wordcraft#3966 had to fix in six copies one by one, and where the Russian and Turkish copies stayed wrong until the first commit here. spelling.rs has them once (`word_to_check`), and each language goes on from there with its own rules (Polish apostrophe endings, Italian elisions, Indonesian clitics, Uruguayan `vos` forms). Tests: words_needing_no_lookup. Ignoring the uppercase option there fails it and all seven languages' proofing-options tests. Proof: 286 tests. --- crates/proof/src/cs/mod.rs | 21 ++------------- crates/proof/src/es/mod.rs | 21 ++------------- crates/proof/src/hr/mod.rs | 21 ++------------- crates/proof/src/id/mod.rs | 21 ++------------- crates/proof/src/it/mod.rs | 21 ++------------- crates/proof/src/lib.rs | 1 + crates/proof/src/pl/mod.rs | 21 ++------------- crates/proof/src/pt/mod.rs | 21 ++------------- crates/proof/src/spelling.rs | 52 ++++++++++++++++++++++++++++++++++++ 9 files changed, 67 insertions(+), 133 deletions(-) create mode 100644 crates/proof/src/spelling.rs diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 1d72dc91..9f8bbd0a 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -38,11 +38,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals /// (acronyms: `ČR`, `EU`), words with digits and words in other scripts are accepted, and so /// is every word when no Czech dictionary is installed. @@ -64,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Czech dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index e16e9d6b..5263bd1d 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -77,11 +77,6 @@ pub fn has_dictionary(lang: ProofLang) -> bool { slot(lang).has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in /// capitals (acronyms: `ONU`, `CDMX`), words with digits and words in other scripts are accepted, /// and so is every word when no dictionary is installed for the variety. Hyphenated compounds @@ -105,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Spanish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang)); } diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 036b7235..53edf067 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -40,11 +40,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals /// (acronyms: `HRT`, `RH`), words with digits and words in other scripts are accepted, and so /// is every word when no Croatian dictionary is installed. @@ -66,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Croatian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index f1e44df4..04248d63 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -45,11 +45,6 @@ pub fn has_dictionary() -> bool { /// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own. const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"]; -/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals /// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so /// is every word when no Indonesian dictionary is installed. A hyphenated word is correct when @@ -73,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Indonesian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return known(d, w) || w.split('-').all(|p| { diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index 1e606ad7..ee5aaaee 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -49,11 +49,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the /// apostrophe is what gets checked (`l'amico`, `un'idea`, `c'è`, `m'hai`, `dov'è`, `anch'io`). const PROCLITICS: &[&str] = &[ @@ -93,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Italian dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let lower = w.to_lowercase(); if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) { return true; diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 0c51e236..365229d7 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -67,6 +67,7 @@ pub mod pl; pub mod pt; pub mod ru; mod slot; +mod spelling; pub mod tr; pub mod vi; diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 30d6d966..7cde9d0f 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -52,11 +52,6 @@ pub fn has_dictionary() -> bool { /// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`). const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"]; -/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals /// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so /// is every word when no Polish dictionary is installed. @@ -78,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Polish dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; let w = w.replace('’', "'"); if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index 0110a8c1..ea91dbdb 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -47,11 +47,6 @@ pub fn has_dictionary() -> bool { DICT.has_dictionary() } -/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)? -fn is_latin_letter(c: char) -> bool { - c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) -} - /// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals /// (acronyms: `USP`, `PT`), words with digits and words in other scripts are accepted, and so /// is every word when no Portuguese dictionary is installed. @@ -73,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { /// [`is_correct_with_options`] against a given dictionary. fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { - if d.is_empty() { - return true; // no Portuguese dictionary installed: no spelling marks - } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); - let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { - return true; - } - if !w.chars().any(is_latin_letter) { - return true; // other scripts: no dictionary - } + }; if w.contains('-') { return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs new file mode 100644 index 00000000..ab5e520e --- /dev/null +++ b/crates/proof/src/spelling.rs @@ -0,0 +1,52 @@ +//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian, +//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the +//! File › Options › Proofing choices. Each language goes on from there with its own rules. + +use crate::Options; +use crate::affix::AffixDict; + +/// Is `c` a letter of the Latin script (the only one these dictionaries cover)? +fn latin(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it +/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single +/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the +/// options skip those, or it is in another script. +pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> { + if d.is_empty() { + return None; + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { + return None; + } + if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return None; + } + w.chars().any(latin).then_some(w) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn words_needing_no_lookup() { + let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); + let defaults = Options::default(); + assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary"); + for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] { + assert_eq!(word_to_check(&d, w, &defaults), None, "{w}"); + } + assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot")); + assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot")); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO")); + assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B")); + assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms"); + } +} From b06aece4eb18257578418a75b6b688651f4c0aeb Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 20:57:35 -0100 Subject: [PATCH 8/9] Proofing: a candidate found again at a lower cost ranks at that cost The suggestion rounds don't run in rising cost: a MAP diacritic drop (11) comes before a REP confusion (10, 9 for Russian). The engine checked each candidate once, at the first cost it was offered, so the same word found again more cheaply kept the dearer cost and could rank one place too low. Czech's engine always did this, and the merged engine took it over. The candidate set is now a map to the cheapest cost offered so far, and an offer that is no cheaper is skipped as before. Found by CodeRabbit on storytold/wordcraft#4191. Tests: a_cheaper_later_offer_wins; skipping every repeat offer again fails it. Proof: 288 tests. --- crates/proof/src/affix_suggest.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 1da35559..ade16edd 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -102,7 +102,8 @@ struct Collector<'a> { d: &'a AffixDict, /// The word, lower case. input: String, - tested: HashSet, + /// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost). + tested: HashMap, /// Suggestion → (cost, order found). found: HashMap, seq: usize, @@ -112,9 +113,16 @@ impl Collector<'_> { /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every /// word is in the dictionary, as written or capitalized (names). fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + if cand.is_empty() || cand == self.input { return; } + match self.tested.get_mut(cand) { + Some(prev) if *prev <= cost => return, + Some(prev) => *prev = cost, + None => { + self.tested.insert(cand.to_string(), cost); + } + } let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { @@ -270,7 +278,7 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let case = casing(word); let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; // 1. Letter case. if d.contains(&input) && word != input { col.keep(input.clone(), COST_CASE); @@ -463,6 +471,18 @@ mod tests { assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); } + /// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop) + /// ranks at the lower one. + #[test] + fn a_cheaper_later_offer_wins() { + let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); + let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + col.offer("cafe", COST_DROP_DIACRITIC); + col.offer("cafe", COST_REP); + col.offer("cafe", COST_SCAN); + assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP)); + } + #[test] fn distance_weighs_map_slips_half() { let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; From cb77c398b9e0ef2840ef8f44d7023e386f3c3e0b Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sat, 24 Oct 2026 20:57:37 -0100 Subject: [PATCH 9/9] Proofing: a dictionary load from before a folder change is not kept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dictionary is read outside the lock. If the dictionaries folder changed while it was being read (File › Options › Proofing), the reload emptied the slot first and the late load then filled it with the old folder's dictionary. It stayed until the next reload. Every language's copy of the loader had this window; the shared slot now closes it in one place. The slot counts reloads. A load that started before a reload still answers the check that asked for it, but it isn't kept: the next check reads the new folder. Found by CodeRabbit on storytold/wordcraft#4191. Tests: a_load_from_before_a_reload_is_not_kept; not counting reloads fails it. Proof: 288 tests. --- crates/proof/src/slot.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs index 8b90a94a..6255ae5a 100644 --- a/crates/proof/src/slot.rs +++ b/crates/proof/src/slot.rs @@ -67,6 +67,8 @@ impl Slot for DictSlot { struct State { dict: Option>, in_memory: bool, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, } /// One language's dictionary. @@ -80,7 +82,7 @@ pub(crate) struct DictSlot { impl DictSlot { pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self { - DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false }) } + DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) } } fn read(&self) -> RwLockReadGuard<'_, State> { @@ -105,12 +107,25 @@ impl DictSlot { /// The dictionary: the installed one (read on first use), one given with [`Self::install`], or /// an empty one when none is available (the language's spelling is off then). pub(crate) fn get(&self) -> Arc { - if let Some(d) = self.read().dict.as_ref() { - return d.clone(); + let reloads = { + let s = self.read(); + if let Some(d) = s.dict.as_ref() { + return d.clone(); + } + s.reloads + }; + self.keep(reloads, Arc::new(self.load_installed())) + } + + /// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the + /// load may have read the old folder: it serves this one check and the next one loads again. + fn keep(&self, reloads: u64, loaded: Arc) -> Arc { + let mut s = self.write(); + if s.reloads != reloads { + return s.dict.clone().unwrap_or(loaded); } - let loaded = Arc::new(self.load_installed()); // Another thread (or an install) may have filled it meanwhile: that one wins. - self.write().dict.get_or_insert(loaded).clone() + s.dict.get_or_insert(loaded).clone() } fn load_installed(&self) -> D { @@ -142,6 +157,7 @@ impl DictSlot { let mut slot = self.write(); slot.dict = None; slot.in_memory = false; + slot.reloads = slot.reloads.wrapping_add(1); } /// The `.dic` file of the installed dictionary, searching the dictionaries folder and the @@ -178,4 +194,17 @@ mod tests { assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none"); assert_eq!(SLOT.name(), "zz_SLOT_TEST"); } + + /// A load that started before a dictionaries-folder change is not kept after its reload. + #[test] + fn a_load_from_before_a_reload_is_not_kept() { + let slot: DictSlot = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test"); + let before = slot.read().reloads; + slot.reload(); + let _ = slot.keep(before, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_none(), "the next check loads from the new folder"); + let now = slot.read().reloads; + let _ = slot.keep(now, Arc::new(AffixDict::default())); + assert!(slot.read().dict.is_some(), "a load since the reload is kept"); + } }