diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs new file mode 100644 index 00000000..1da35559 --- /dev/null +++ b/crates/proof/src/affix_suggest.rs @@ -0,0 +1,493 @@ +//! Spelling suggestions from an affix dictionary, one engine for every language that reads one +//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come +//! in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`): +//! adding a missing accent is the commonest slip and ranks first, while taking away one the +//! writer typed ranks after the known confusions; +//! 3. the REP table of common confusions (`żeka` → `rzeka`); +//! 4. a missing space (`napewno` → `na pewno`); +//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with +//! the next or mistyped (TRY letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in +//! which a MAP slip costs half (two typos at most). +//! +//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may +//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every +//! round, so a call costs a few thousand dictionary lookups at most. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters. +pub(crate) const MAX_LETTERS: usize = 40; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +pub(crate) const COST_ADD_DIACRITIC: u32 = 4; +pub(crate) const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +pub(crate) const COST_SPLIT: u32 = 12; +/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]). +const COST_SPLIT_SHORT: u32 = 17; +/// A missing space between two longer halves with no short word beside it ([`Split::Near`]). +pub(crate) const COST_SPLIT_LONG: u32 = 18; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +pub(crate) const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +/// What a language adds to its dictionary's tables. +pub(crate) struct Tuning { + /// Where a missing space may go (round 4). + pub split: Split, + /// TRY letters for a dictionary without a TRY line. + pub alphabet: &'static str, + /// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`). + pub map: &'static [&'static str], + /// REP confusions the language adds to its dictionary's. + pub rep: &'static [(&'static str, &'static str)], + /// The cost of a REP confusion. + pub rep_cost: u32, + /// Most letters a MAP variant changes. + pub max_map_changes: usize, + /// The cost of changing letter `from` to its MAP relative `to`. + pub map_cost: fn(char, char) -> u32, + /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. + pub mistype_cost: fn(&[char], usize, char) -> u32, +} + +/// Where a missing space may go. +pub(crate) enum Split { + /// Anywhere, dearer when a half is shorter than three letters. + Anywhere, + /// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one + /// that runs into the word before (`kocham cię`), dearer between two longer halves, and never + /// leaving a stray short half (`że ka`). + Near { leading: &'static [&'static str], trailing: &'static [&'static str] }, + /// The language's own rule: the cost of the halves `a b`, or `None` to skip them. + Custom(fn(&str, &str) -> Option), +} + +impl Tuning { + /// The dictionary's tables alone (Czech, Indonesian, Spanish). + pub(crate) const DEFAULT: Tuning = + Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type is likelier right, +/// so taking one away costs more than a known confusion from the REP table. +pub(crate) fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Any mistyped letter costs the same. +fn mistype_cost(_: &[char], _: usize, _: char) -> u32 { + COST_MISTYPE +} + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + tested: HashSet, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + return; + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP groups: the dictionary's, then the language's. +fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec]> { + if t.map.is_empty() { + return Cow::Borrowed(d.map()); + } + Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect()) +} + +/// The other members of every MAP group of `c` (its likely confusions), each once. +fn alternatives(c: char, map: &[Vec]) -> Vec { + let mut out: Vec = Vec::new(); + for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) { + if x != c && !out.contains(&x) { + out.push(x); + } + } + out +} + +/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, +/// letter by letter), with its cost, each written after `buf`. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some((&c, rest)) = chars.split_first() else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); + buf.push(c); + map_variants(rest, later, t, changes, cost, buf, out); + buf.pop(); + if changes < t.max_map_changes { + for &a in here { + buf.push(a); + map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`. +pub(crate) fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered. +fn split_cost(split: &Split, a: &str, b: &str) -> Option { + let long = |part: &str| part.chars().count() >= 3; + match split { + Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }), + Split::Near { leading, trailing } => { + if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None + } + } + Split::Custom(cost) => cost(a, b), + } +} + +/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case. +pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + // 1. Letter case. + if d.contains(&input) && word != input { + col.keep(input.clone(), COST_CASE); + } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { + col.keep(capitalize(&input), COST_CASE); + } + // 2. Accents and other MAP slips. + let map = groups(d, t); + let alts: Vec> = chars.iter().map(|&c| alternatives(c, &map)).collect(); + let mut variants = Vec::new(); + map_variants(&chars, &alts, t, 0, 0, &mut Vec::with_capacity(n), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } + // 3. Common confusions. + let rep = d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } + // 4. A missing space; both halves must be dictionary words. + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(&t.split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } + // 5. Keyboard neighbours from the dictionary's KEY rows. + for row in d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } + // One typo, with the TRY letters (the language's when the dictionary has none), bounded and + // each once, in their order: a TRY line lists the commonest letters first, so equal-cost + // candidates come out in that order. + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.clone(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.clone(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.clone(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(&chars, i, c)); + } + } + } + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars, &map); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let t = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text, + }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { + let d = col.d; + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + #[test] + fn distance_weighs_map_slips_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY + /// line, and a split rule decides which missing spaces are offered. + #[test] + fn tuning_adds_to_the_dictionary_tables() { + let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap(); + let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT }; + assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into())); + assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group"); + assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair"); + assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters"); + let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT }; + assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into())); + assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half"); + assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into())); + } +} diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 4e5ecd17..05eba74a 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -1,168 +1,8 @@ -//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits -//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian -//! ([`crate::id`]) uses the same ranking with its own dictionary. +//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed +//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Longest word suggestions are looked for, in letters (Spanish shares the bound). -pub(crate) const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 17; -const COST_KEY: u32 = 13; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (a capitalized name). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (typing without diacritics); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; /// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -171,201 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space; both halves must be dictionary words. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |part: &str| part.chars().count() >= 3; - let cost = if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_RARE }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. Keyboard neighbours from the installed dictionary's KEY rows. - for row in d.key().split('|').take(32) { - let keys: Vec = row.chars().take(128).collect(); - for pair in keys.windows(2) { - let [a, b] = pair else { continue }; - for (i, &c) in chars.iter().enumerate() { - let replacement = if c == *a { - Some(*b) - } else if c == *b { - Some(*a) - } else { - None - }; - if let Some(replacement) = replacement { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = replacement; - } - col.offer(&v.iter().collect::(), COST_KEY); - } - } - } - } - // One-letter edits use only the runtime TRY alphabet, with bounded work. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).take(128).collect(); - letters.sort_unstable(); - letters.dedup(); - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (the first changed letter breaks ties). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) } #[cfg(test)] mod tests { use super::*; - #[test] - fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { - let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); - let suggestions = suggest_in(&d, "počítačz", 8); - assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); - assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); - } - #[test] fn suggestions_are_fast_enough() { crate::cs::test_dict::install(); diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs index b42efa2e..0568ae87 100644 --- a/crates/proof/src/es/suggest.rs +++ b/crates/proof/src/es/suggest.rs @@ -1,5 +1,5 @@ -//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to +//! Spanish suggestions. The ranking is the shared dictionary-driven one +//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to //! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, //! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. //! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first @@ -7,7 +7,7 @@ use crate::ProofLang; use crate::affix::{AffixDict, Casing, capitalize, casing}; -use crate::cs::suggest::MAX_LETTERS; +use crate::affix_suggest::{MAX_LETTERS, Tuning}; /// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's /// case. @@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec _ => v, }); } - for s in crate::cs::suggest::suggest_in(d, word, max) { + for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) { if !out.contains(&s) { out.push(s); } diff --git a/crates/proof/src/hr/suggest.rs b/crates/proof/src/hr/suggest.rs index d6f8159a..d8c19b99 100644 --- a/crates/proof/src/hr/suggest.rs +++ b/crates/proof/src/hr/suggest.rs @@ -1,12 +1,11 @@ -//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance, -//! missing spaces and a bounded search. Proper Croatian corrections depend on -//! the actual installed dictionary entries. +//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of +//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines. -use std::collections::{HashMap, HashSet}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; -use crate::affix::{AffixDict, Casing, capitalize, casing}; - -/// Short words often run into the next one (`não é`, `da água`, `para isso`). +/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`). const LEADING_WORDS: &[&str] = &[ "a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će", "bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad", @@ -14,162 +13,13 @@ const LEADING_WORDS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; - -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Croatian adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiourntsjkvlmdpbgczčćšžđfh", + map: &["cčć", "dđ", "sš", "zž"], + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -178,173 +28,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']]; - let map = if d.map().is_empty() { &fallback_map } else { d.map() }; - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -354,6 +38,13 @@ mod tests { fn nosuggest_stems_and_forms_are_never_offered() { crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); } + /// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća` + /// first, before a typo that costs more (`kuha`). + #[test] + fn diacritics_without_map_lines() { + let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap(); + assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]); + } #[test] fn croatian_suggestions() { crate::hr::test_dict::install(); diff --git a/crates/proof/src/id/suggest.rs b/crates/proof/src/id/suggest.rs index 562af9a8..b42efde9 100644 --- a/crates/proof/src/id/suggest.rs +++ b/crates/proof/src/id/suggest.rs @@ -1,7 +1,6 @@ -//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses -//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem -//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is -//! hard-coded here. +//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing +//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID` +//! dictionary, so nothing Indonesian is hard-coded here. use crate::affix::AffixDict; @@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - crate::cs::suggest::suggest_in(d, word, max) + crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT) } #[cfg(test)] diff --git a/crates/proof/src/it/suggest.rs b/crates/proof/src/it/suggest.rs index 6a043ff2..55f86909 100644 --- a/crates/proof/src/it/suggest.rs +++ b/crates/proof/src/it/suggest.rs @@ -1,25 +1,9 @@ -//! Italian spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`roma` → `Roma`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`, -//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`); -//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`per me`, `a Milano`, `di Roma`). const LEADING_WORDS: &[&str] = &[ @@ -35,161 +19,12 @@ const TRAILING_WORDS: &[&str] = &[ "mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove", "è", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table. -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; +/// What Italian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `à` → `a`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} /// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -198,176 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent. - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index ad516b29..4bf4da78 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -50,6 +50,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod affix; +mod affix_suggest; pub mod bg; pub mod cs; pub mod dict; diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs index 0e32a1ea..7baeaa54 100644 --- a/crates/proof/src/pl/suggest.rs +++ b/crates/proof/src/pl/suggest.rs @@ -1,26 +1,9 @@ -//! Polish spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`warszawa` → `Warszawa`); -//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy` -//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most -//! common slip, so adding them ranks first, while taking away a diacritic the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` → -//! `huśtawka`, `pszyjaciel` → `przyjaciel`); -//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham -//! cię`), cheaper when one half is a short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which a diacritic slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand -//! dictionary lookups at most. +//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the +//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`), +//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning}; /// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede /// wszystkim`). @@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[ /// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). const TRAILING_WORDS: &[&str] = &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; +/// What Polish adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx", + mistype_cost, + ..Tuning::DEFAULT +}; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moze` → `może`, then - // `Mozę`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the -/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the -/// writer did type is likelier right, so taking one away costs more than a known confusion from -/// the REP table (`żeka` → `rzeka` before `zeka`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) +/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it +/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo. +fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 { + let last = i + 1 == chars.len(); + if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE } } /// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. @@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Diacritics and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half (`że ka`). - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - // Final devoicing can make a voiced consonant sound voiceless. - let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g')); - let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }; - col.offer(&s(&v), cost); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs index 36f1c7b0..f8f33bff 100644 --- a/crates/proof/src/pt/suggest.rs +++ b/crates/proof/src/pt/suggest.rs @@ -1,26 +1,9 @@ -//! Portuguese spelling suggestions, best first. -//! -//! Candidates come in rounds of rising cost, each checked against the dictionary: -//! 1. letter case (`brasília` → `Brasília`); -//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`, -//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the -//! most common slip, so adding one ranks first, while taking away an accent the writer typed -//! ranks after the known confusions; -//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` → -//! `exceção`, `previlégio` → `privilégio`); -//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); -//! 6. when that finds fewer than asked for: the stems that start like the word (accents -//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance -//! in which an accent slip costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand -//! dictionary lookups at most. +//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and +//! letters to try for a dictionary without a TRY line. -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; /// Short words often run into the next one (`não é`, `da água`, `para isso`). const LEADING_WORDS: &[&str] = &[ @@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[ "me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si", "eu", "tu", "que", "de", "em", "é", ]; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 4; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; -const COST_ADD_DIACRITIC: u32 = 4; -const COST_OTHER_MAP: u32 = 7; -const COST_DROP_DIACRITIC: u32 = 11; -const COST_REP: u32 = 10; -const COST_SPLIT: u32 = 12; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; - -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} - -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance (`moça`, then `Moça`). - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The MAP group of each letter (its other members are its likely confusions). -fn alternatives(c: char, map: &[Vec]) -> Vec { - map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() -} - -/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the -/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type -/// is likelier right, so taking one away costs more than a known confusion from the REP table -/// (`seje` → `seja` before `sejé`). -fn map_cost(from: char, to: char) -> u32 { - match (from.is_ascii(), to.is_ascii()) { - (true, false) => COST_ADD_DIACRITIC, - (false, true) => COST_DROP_DIACRITIC, - _ => COST_OTHER_MAP, - } -} - -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; - } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } - } -} - -/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`. -fn fold(c: char, map: &[Vec]) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) -} - -/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if related(ai, bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} +/// What Portuguese adds to its dictionary's tables. +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw", + ..Tuning::DEFAULT +}; /// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { @@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. - let map = d.map(); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in d.rep() { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from.as_str()) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - // Among equal costs, the change nearest the word's start is the likelier intent - // (`saúde` before `saudê`: both add an accent, but at the first difference). - let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); - found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either -/// case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let map = d.map(); - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let related = |a: char, b: char| fold(a, map) == fold(b, map); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, &related, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] mod tests { use super::*; + use crate::affix_suggest::{fold, weighted_distance}; fn has(word: &str, want: &str) { let s = suggest(word, 8); diff --git a/crates/proof/src/ru/suggest.rs b/crates/proof/src/ru/suggest.rs index 24eeb764..135f4152 100644 --- a/crates/proof/src/ru/suggest.rs +++ b/crates/proof/src/ru/suggest.rs @@ -1,30 +1,14 @@ -//! Russian spelling suggestions, best first. -//! -//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the -//! common confusions are defined here. Candidates come in rounds of rising cost, each checked -//! against the dictionary: -//! 1. letter case (`москва` → `Москва`); -//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`, -//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to -//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`); -//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or -//! doubled consonant, `здраствуй` → `здравствуй`); -//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a -//! short function word; -//! 5. one typo: a letter left out, added, swapped with the next or mistyped; -//! 6. when that finds fewer than asked for: the stems that start like the word and share the -//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within -//! a letter group costs half (two typos at most). -//! -//! The input is bounded (40 letters), and so is every round. +//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with +//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP` +//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` → +//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule +//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`). -use std::collections::{HashMap, HashSet}; - -use crate::affix::{AffixDict, Casing, capitalize, casing}; +use crate::affix::AffixDict; +use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning}; /// Letters confused with each other: a slip inside a group is cheaper than another typo. const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"]; - /// Common confusions: a written sequence and what was meant. const REP: &[(&str, &str)] = &[ ("жы", "жи"), @@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[ ("ц", "тс"), ("тс", "ц"), ]; - /// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`). const LEADING_WORDS: &[&str] = &[ "не", @@ -123,161 +106,42 @@ const PREPOSITIONS: &[&str] = &[ const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"]; /// The Russian alphabet, when the dictionary has no `TRY` line. const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё"; -const MAX_LETTERS: usize = 40; -const MAX_MAP_CHANGES: usize = 3; -const MAX_MAP_VARIANTS: usize = 4096; -/// Stems the scan of round 6 expands. -const SCAN_STEMS: usize = 48; -const COST_CASE: u32 = 0; +/// `е` written for `ё`, or the other way round, is barely a slip. const COST_YO: u32 = 3; -const COST_MAP: u32 = 7; -const COST_REP: u32 = 9; -const COST_SPLIT: u32 = 12; +/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`). const COST_SPLIT_PREPOSITION: u32 = 17; -const COST_SPLIT_RARE: u32 = 18; -const COST_SWAP: u32 = 14; -const COST_DROP_OR_ADD: u32 = 15; -const COST_MISTYPE: u32 = 16; -const COST_SCAN: u32 = 20; -struct Collector<'a> { - d: &'a AffixDict, - /// The word, lower case. - input: String, - tested: HashSet, - /// Suggestion → (cost, order found). - found: HashMap, - seq: usize, -} +/// What Russian adds to its dictionary's tables. +const TUNING: Tuning = + Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT }; -impl Collector<'_> { - /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every - /// word is in the dictionary, as written or capitalized (names). - fn offer(&mut self, cand: &str, cost: u32) { - if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { - return; - } - let mut words = Vec::new(); - let mut cost = cost; - for part in cand.split(' ') { - if self.d.contains_suggestible(part) { - words.push(part.to_string()); - } else { - let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; - } - words.push(c); - // A name ranks after a common word at the same distance. - cost += 1; - } - } - self.keep(words.join(" "), cost); - } - - /// Keep a suggestion known to be correct. - fn keep(&mut self, text: String, cost: u32) { - if !text.split(' ').all(|part| self.d.can_suggest(part)) { - return; - } - self.seq += 1; - let seq = self.seq; - let e = self.found.entry(text).or_insert((cost, seq)); - if cost < e.0 { - *e = (cost, seq); - } - } - - fn count_within(&self, cost: u32) -> usize { - self.found.values().filter(|(c, _)| *c <= cost).count() - } -} - -/// The other members of each [`MAP`] group holding `c`. -fn alternatives(c: char) -> Vec { - let mut out: Vec = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect(); - out.sort_unstable(); - out.dedup(); - out -} - -/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip -/// (`ё` is optional in ordinary writing). +/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest. fn map_cost(from: char, to: char) -> u32 { - if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP } + if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP } } -/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost. -fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { - return; +/// A missing space: cheap after a short leading word or before a short trailing one, dearer after +/// a short preposition or between two longer words, never leaving a stray short half. +fn split(a: &str, b: &str) -> Option { + let long = |p: &str| p.chars().count() >= 3; + // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for + // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. + let preposition = PREPOSITIONS.contains(&a); + if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { + return None; } - let Some(&c) = chars.get(i) else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } - return; - }; - buf.push(c); - map_variants(chars, alts, i + 1, changes, cost, buf, out); - buf.pop(); - if changes < MAX_MAP_CHANGES { - for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { - buf.push(a); - map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); - buf.pop(); - } + if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { + Some(COST_SPLIT_PREPOSITION) + } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + Some(COST_SPLIT) + } else if long(a) && long(b) { + Some(COST_SPLIT_LONG) + } else { + None } } -/// A letter's group representative (the first of its first group), lower case: `ё` → `е`, -/// `з` → `з`, `с` → `з`. -fn fold(c: char) -> char { - let l = c.to_lowercase().next().unwrap_or(c); - MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l) -} - -/// Optimal string alignment distance where a slip within a letter group costs 1 and any other -/// edit 2; `limit + 1` once it is certainly above `limit`. -fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 { - let m = b.len(); - let over = limit + 1; - let mut prev2 = vec![0u32; m + 1]; - let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); - let mut cur = vec![0u32; m + 1]; - for (i, &ai) in a.iter().enumerate() { - if let Some(c) = cur.get_mut(0) { - *c = (i as u32 + 1) * 2; - } - let mut row_min = u32::MAX; - for (j, &bj) in b.iter().enumerate() { - let sub = if ai == bj { - 0 - } else if fold(ai) == fold(bj) { - 1 - } else { - 2 - }; - let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); - let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); - if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { - v = v.min(at(&prev2, j - 1) + 2); - } - if let Some(c) = cur.get_mut(j + 1) { - *c = v; - } - row_min = row_min.min(v); - } - if m > 0 && row_min > limit { - return over; - } - std::mem::swap(&mut prev2, &mut prev); - std::mem::swap(&mut prev, &mut cur); - } - prev.get(m).copied().unwrap_or(over).min(over) -} - /// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case. pub fn suggest(word: &str, max: usize) -> Vec { suggest_in(&super::dictionary(), word, max) @@ -285,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// [`suggest`] against a given dictionary. pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { - let word = word.trim(); - let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { - return Vec::new(); - } - let case = casing(word); - let input = word.to_lowercase(); - let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Slips within a letter group. - let alts: Vec> = chars.iter().map(|&c| alternatives(c)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - for (from, to) in REP { - for (i, _) in input.match_indices(from) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), COST_REP); - } - } - } - // 4. A missing space: cheap after a short leading word or before a short trailing one, - // dearer between two longer words, never leaving a stray short half. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - let long = |p: &str| p.chars().count() >= 3; - // A preposition takes a noun, not a verb (`с генерировать` isn't a fix for - // `сгенерировать`); after one or two letters a split is likelier wrong than a typo. - let preposition = PREPOSITIONS.contains(&a); - if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) { - continue; - } - let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) { - COST_SPLIT_PREPOSITION - } else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { - COST_SPLIT - } else if long(a) && long(b) { - COST_SPLIT_RARE - } else { - continue; - }; - col.offer(&format!("{a} {b}"), cost); - } - // 5. One typo. - let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); - if letters.is_empty() { - letters = ALPHABET.chars().collect(); - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), COST_MISTYPE); - } - } - } - // 6. Further off: scan the stems that start like the word. - if col.count_within(COST_MISTYPE) < max { - scan(&mut col, &chars); - } - let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); - found.sort_by_key(|(_, k)| *k); - let mut out: Vec = Vec::new(); - for (text, _) in found { - let t = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text, - }; - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; - } - } - out -} - -/// Round 6: rank the stems starting with the word's first letter (or a group relative of it, -/// either case) by shared letter pairs, then keep the forms of the best within two typos. -fn scan(col: &mut Collector, chars: &[char]) { - let d = col.d; - let Some(&first) = chars.first() else { return }; - let folded: Vec = chars.iter().map(|&c| fold(c)).collect(); - let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); - let mut firsts: Vec = std::iter::once(first).chain(alternatives(first)).collect(); - let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); - firsts.extend(upper); - firsts.sort_unstable(); - firsts.dedup(); - let n = folded.len(); - let mut best: Vec<(u32, &str)> = Vec::new(); - for f in firsts { - let prefix = f.to_string(); - for stem in d.stems_with_prefix(&prefix) { - let sf: Vec = stem.chars().map(fold).collect(); - let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); - if shared < 2 { - continue; - } - // Shared pairs, less the stem's letters the word doesn't explain, less a length gap - // beyond what an ending explains. - let extra = sf.len().saturating_sub(shared); - let gap = sf.len().abs_diff(n).saturating_sub(4); - let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; - if score == 0 { - continue; - } - if best.len() < SCAN_STEMS { - best.push((score, stem)); - } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) - && score > min.0 - { - *min = (score, stem); - } - } - } - best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); - let limit = 4; - for (_, stem) in best { - for form in d.forms(stem) { - let lf: Vec = form.to_lowercase().chars().collect(); - if lf.len().abs_diff(chars.len()) > 2 { - continue; - } - let dist = weighted_distance(&lf, chars, limit); - if dist <= limit && lf.as_slice() != chars { - col.keep(form, COST_SCAN + dist * 3); - } - } - } + crate::affix_suggest::suggest_in(d, word, max, &TUNING) } #[cfg(test)] @@ -461,6 +161,20 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + /// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip + /// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip; + /// a preposition is never split off a verb, and after a one-letter one a typo is likelier. + #[test] + fn russian_costs_rank_its_slips() { + let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap(); + assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]); + assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]); + assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into())); + let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4); + assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}"); + assert!(s.contains(&"в доме".into()), "{s:?}"); + } + /// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never /// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the /// shared Latin-script check doesn't reach it. @@ -518,12 +232,15 @@ mod tests { #[test] fn distance_weighs_letter_groups_half() { + use crate::affix_suggest::{fold, weighted_distance}; + let map: Vec> = MAP.iter().map(|g| g.chars().collect()).collect(); + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); let c = |s: &str| s.chars().collect::>(); - assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1); - assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1); - assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2); - assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6); - assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3); + assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1); + assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1); + assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2); + assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6); + assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3); } #[test]