This commit is contained in:
Martin Ball
2026-10-30 21:16:12 +00:00
24 changed files with 1372 additions and 2784 deletions
+539
View File
@@ -0,0 +1,539 @@
//! Spelling suggestions from an affix dictionary, one engine for every language that reads one
//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come
//! in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`warszawa` → `Warszawa`);
//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`):
//! adding a missing accent is the commonest slip and ranks first, while taking away one the
//! writer typed ranks after the known confusions;
//! 3. the REP table of common confusions (`żeka` → `rzeka`);
//! 4. a missing space (`napewno` → `na pewno`);
//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with
//! the next or mistyped (TRY letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in
//! which a MAP slip costs half (two typos at most).
//!
//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may
//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every
//! round, so a call costs a few thousand dictionary lookups at most.
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters.
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
pub(crate) const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
pub(crate) const COST_SPLIT: u32 = 12;
/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]).
const COST_SPLIT_SHORT: u32 = 17;
/// A missing space between two longer halves with no short word beside it ([`Split::Near`]).
pub(crate) const COST_SPLIT_LONG: u32 = 18;
const COST_KEY: u32 = 13;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
pub(crate) const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
/// What a language adds to its dictionary's tables.
pub(crate) struct Tuning {
/// Where a missing space may go (round 4).
pub split: Split,
/// TRY letters for a dictionary without a TRY line.
pub alphabet: &'static str,
/// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`).
pub map: &'static [&'static str],
/// REP confusions the language adds to its dictionary's.
pub rep: &'static [(&'static str, &'static str)],
/// The cost of a REP confusion.
pub rep_cost: u32,
/// Most letters a MAP variant changes.
pub max_map_changes: usize,
/// The cost of changing letter `from` to its MAP relative `to`.
pub map_cost: fn(char, char) -> u32,
/// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead.
pub mistype_cost: fn(&[char], usize, char) -> u32,
}
/// Where a missing space may go.
pub(crate) enum Split {
/// Anywhere, dearer when a half is shorter than three letters.
Anywhere,
/// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one
/// that runs into the word before (`kocham cię`), dearer between two longer halves, and never
/// leaving a stray short half (`że ka`).
Near { leading: &'static [&'static str], trailing: &'static [&'static str] },
/// The language's own rule: the cost of the halves `a b`, or `None` to skip them.
Custom(fn(&str, &str) -> Option<u32>),
}
impl Tuning {
/// The dictionary's tables alone (Czech, Indonesian, Spanish).
pub(crate) const DEFAULT: Tuning =
Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost };
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (typing without diacritics); an accent the writer did type is likelier right,
/// so taking one away costs more than a known confusion from the REP table.
pub(crate) fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Any mistyped letter costs the same.
fn mistype_cost(_: &[char], _: usize, _: char) -> u32 {
COST_MISTYPE
}
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
/// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost).
tested: HashMap<String, u32>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input {
return;
}
match self.tested.get_mut(cand) {
Some(prev) if *prev <= cost => return,
Some(prev) => *prev = cost,
None => {
self.tested.insert(cand.to_string(), cost);
}
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
// `Mozę`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP groups: the dictionary's, then the language's.
fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec<char>]> {
if t.map.is_empty() {
return Cow::Borrowed(d.map());
}
Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect())
}
/// The other members of every MAP group of `c` (its likely confusions), each once.
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
let mut out: Vec<char> = Vec::new();
for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) {
if x != c && !out.contains(&x) {
out.push(x);
}
}
out
}
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
/// letter by letter), with its cost, each written after `buf`.
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some((&c, rest)) = chars.split_first() else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
buf.push(c);
map_variants(rest, later, t, changes, cost, buf, out);
buf.pop();
if changes < t.max_map_changes {
for &a in here {
buf.push(a);
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`.
pub(crate) fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered.
fn split_cost(split: &Split, a: &str, b: &str) -> Option<u32> {
let long = |part: &str| part.chars().count() >= 3;
match split {
Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }),
Split::Near { leading, trailing } => {
if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
Split::Custom(cost) => cost(a, b),
}
}
/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case.
pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
add_case(&mut col, word, &input);
let map = groups(d, t);
add_map_slips(&mut col, &chars, &map, t);
add_confusions(&mut col, &input, t);
add_splits(&mut col, &input, &t.split);
add_key_neighbours(&mut col, &chars);
add_typos(&mut col, &chars, t);
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars, &map);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (the first changed letter breaks ties).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 1: letter case.
fn add_case(col: &mut Collector, word: &str, input: &str) {
if col.d.contains(input) && word != input {
col.keep(input.to_string(), COST_CASE);
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
col.keep(capitalize(input), COST_CASE);
}
}
/// Round 2: accents and other MAP slips.
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
}
/// Round 3: common confusions.
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
for (from, to) in rep {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
}
}
}
}
/// Round 4: a missing space; both halves must be dictionary words.
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
if let Some(cost) = split_cost(split, a, b) {
col.offer(&format!("{a} {b}"), cost);
}
}
}
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
for row in col.d.key().split('|').take(32) {
let keys: Vec<char> = row.chars().take(128).collect();
for pair in keys.windows(2) {
let [a, b] = pair else { continue };
for (i, &c) in chars.iter().enumerate() {
let replacement = if c == *a {
Some(*b)
} else if c == *b {
Some(*a)
} else {
None
};
if let Some(replacement) = replacement {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = replacement;
}
col.offer(&v.iter().collect::<String>(), COST_KEY);
}
}
}
}
}
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
/// equal-cost candidates come out in that order.
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
let d = col.d;
let n = chars.len();
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
let mut letters: Vec<char> = Vec::new();
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
if !letters.contains(&c) {
letters.push(c);
}
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.to_vec();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.to_vec();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.to_vec();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.to_vec();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
}
}
}
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
let d = col.d;
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT);
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
}
/// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop)
/// ranks at the lower one.
#[test]
fn a_cheaper_later_offer_wins() {
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
col.offer("cafe", COST_DROP_DIACRITIC);
col.offer("cafe", COST_REP);
col.offer("cafe", COST_SCAN);
assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP));
}
#[test]
fn distance_weighs_map_slips_half() {
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
}
/// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY
/// line, and a split rule decides which missing spaces are offered.
#[test]
fn tuning_adds_to_the_dictionary_tables() {
let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap();
let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT };
assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into()));
assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group");
assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair");
assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters");
let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT };
assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into()));
assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half");
assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into()));
}
}
+10 -75
View File
@@ -7,88 +7,35 @@ pub(crate) mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Czech Hunspell dictionary.
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
/// The Czech dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Czech");
/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.dict.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Czech spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Czech dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals
@@ -112,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Czech dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+5 -389
View File
@@ -1,168 +1,8 @@
//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits
//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian
//! ([`crate::id`]) uses the same ranking with its own dictionary.
//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed
//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters (Spanish shares the bound).
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 17;
const COST_KEY: u32 = 13;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (a capitalized name).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (typing without diacritics); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
use crate::affix::AffixDict;
use crate::affix_suggest::Tuning;
/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -171,237 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
let map = d.map();
add_case_suggestions(&mut col, &input, word);
add_map_suggestions(&mut col, &chars, map);
add_replacement_suggestions(&mut col, &input, d.rep());
add_split_suggestions(&mut col, &input);
add_keyboard_suggestions(&mut col, &chars, d.key());
add_typing_suggestions(&mut col, &chars, d.try_chars());
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (the first changed letter breaks ties).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
fn add_case_suggestions(col: &mut Collector, input: &str, word: &str) {
if col.d.contains(input) && word != input {
col.keep(input.to_string(), COST_CASE);
} else {
let capitalized = capitalize(input);
if col.d.contains(&capitalized) && word != capitalized {
col.keep(capitalized, COST_CASE);
}
}
}
fn add_map_suggestions(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
let alternatives: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(chars, &alternatives, 0, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
variants.sort_by_key(|variant| variant.1);
for (variant, cost) in variants {
col.offer(&variant, cost);
}
}
fn add_replacement_suggestions(col: &mut Collector, input: &str, replacements: &[(String, String)]) {
for (from, to) in replacements {
if from.is_empty() {
continue;
}
for (index, _) in input.match_indices(from) {
if let (Some(prefix), Some(suffix)) = (input.get(..index), input.get(index + from.len()..)) {
col.offer(&format!("{prefix}{to}{suffix}"), COST_REP);
}
}
}
}
fn add_split_suggestions(col: &mut Collector, input: &str) {
for (index, _) in input.char_indices().skip(1) {
let (Some(first), Some(second)) = (input.get(..index), input.get(index..)) else { continue };
let cost = if first.chars().count() >= 3 && second.chars().count() >= 3 { COST_SPLIT } else { COST_SPLIT_RARE };
col.offer(&format!("{first} {second}"), cost);
}
}
fn add_keyboard_suggestions(col: &mut Collector, chars: &[char], key_rows: &str) {
for row in key_rows.split('|').take(32) {
let keys: Vec<char> = row.chars().take(128).collect();
for pair in keys.windows(2) {
let [first, second] = pair else { continue };
for (index, &character) in chars.iter().enumerate() {
let replacement = if character == *first {
Some(*second)
} else if character == *second {
Some(*first)
} else {
None
};
if let Some(replacement) = replacement {
let mut candidate = chars.to_vec();
if let Some(character) = candidate.get_mut(index) {
*character = replacement;
}
col.offer(&candidate.iter().collect::<String>(), COST_KEY);
}
}
}
}
}
fn add_typing_suggestions(col: &mut Collector, chars: &[char], try_chars: &str) {
let mut letters: Vec<char> = try_chars.chars().filter(|character| character.is_lowercase()).take(128).collect();
letters.sort_unstable();
letters.dedup();
let word = |candidate: &[char]| candidate.iter().collect::<String>();
add_swap_suggestions(col, chars, &word);
add_drop_suggestions(col, chars, &word);
add_insert_suggestions(col, chars, &letters, &word);
add_mistype_suggestions(col, chars, &letters, &word);
}
fn add_swap_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len().saturating_sub(1) {
if chars.get(index) != chars.get(index + 1) {
let mut candidate = chars.to_vec();
candidate.swap(index, index + 1);
col.offer(&word(&candidate), COST_SWAP);
}
}
}
fn add_drop_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len() {
let mut candidate = chars.to_vec();
candidate.remove(index);
col.offer(&word(&candidate), COST_DROP_OR_ADD);
}
}
fn add_insert_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..=chars.len() {
for &letter in letters {
let mut candidate = chars.to_vec();
candidate.insert(index, letter);
col.offer(&word(&candidate), COST_DROP_OR_ADD);
}
}
}
fn add_mistype_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
for index in 0..chars.len() {
for &letter in letters {
if chars.get(index) != Some(&letter) {
let mut candidate = chars.to_vec();
if let Some(character) = candidate.get_mut(index) {
*character = letter;
}
col.offer(&word(&candidate), COST_MISTYPE);
}
}
}
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
let suggestions = suggest_in(&d, "počítačz", 8);
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
}
#[test]
fn suggestions_are_fast_enough() {
crate::cs::test_dict::install();
+21 -76
View File
@@ -16,16 +16,19 @@ mod suggest;
pub(crate) mod test_dict;
pub mod voseo;
use std::sync::{Arc, PoisonError, RwLock};
use std::sync::Arc;
use crate::ProofLang;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first
/// use, then the installed one (or an empty one).
static DICTS: [RwLock<Option<Arc<AffixDict>>>; 4] = [const { RwLock::new(None) }; 4];
/// The dictionary of each variety, found on first use (see [`DictSlot`]).
pub(crate) static SPAIN_DICT: DictSlot<AffixDict> = DictSlot::new(SPAIN, "Spanish (Spain)");
pub(crate) static MEXICO_DICT: DictSlot<AffixDict> = DictSlot::new(MEXICO, "Spanish (Mexico)");
pub(crate) static LATIN_AMERICA_DICT: DictSlot<AffixDict> = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)");
pub(crate) static URUGUAY_DICT: DictSlot<AffixDict> = DictSlot::new(URUGUAY, "Spanish (Uruguay)");
/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the
/// generic one RLA-ES publishes).
@@ -40,84 +43,38 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e
/// or generic one.
const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"];
/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's
/// file names; Spain for any other language.
fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) {
/// `lang`'s variety's dictionary; Spain's for any other language.
fn slot(lang: ProofLang) -> &'static DictSlot<AffixDict> {
match lang {
ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO),
ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA),
ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY),
_ => (0, "Spanish (Spain)", SPAIN),
ProofLang::EsMx => &MEXICO_DICT,
ProofLang::Es419 => &LATIN_AMERICA_DICT,
ProofLang::EsUy => &URUGUAY_DICT,
_ => &SPAIN_DICT,
}
}
/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`,
/// `es_UY`): the first of its file names, used for the status-bar hint and load errors.
pub fn dictionary_name(lang: ProofLang) -> &'static str {
variety(lang).2.first().copied().unwrap_or("es_ES")
slot(lang).name()
}
/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given
/// with [`install_dictionary`], or an empty one when none is available (spelling is off then;
/// see [`has_dictionary`]).
pub fn dictionary(lang: ProofLang) -> Arc<AffixDict> {
let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) };
if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed(lang));
let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed(lang: ProofLang) -> AffixDict {
let (_, name, files) = variety(lang);
let key = dictionary_name(lang);
let Some((aff, dic)) = crate::hunspell::find(files) else {
crate::hunspell::clear_load_error(key);
log::info!("{name} spelling: no Hunspell {key} dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("{name} spelling: {e}");
AffixDict::default()
})
slot(lang).get()
}
/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text,
/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
if let Some(slot) = DICTS.get(variety(lang).0) {
*slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d));
}
Ok(())
}
/// Forget every variety's loaded dictionary, so the next check looks for an installed one again
/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
for slot in &DICTS {
*slot.write().unwrap_or_else(PoisonError::into_inner) = None;
}
}
/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder
/// or the system locations now (File › Options › Proofing shows it); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path(lang: ProofLang) -> Option<std::path::PathBuf> {
crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic)
slot(lang).install(aff, dic)
}
/// Is a dictionary available for `lang`'s variety (spelling is checked)?
pub fn has_dictionary(lang: ProofLang) -> bool {
!dictionary(lang).is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
slot(lang).has_dictionary()
}
/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in
@@ -143,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Spanish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang));
}
@@ -200,7 +145,7 @@ mod tests {
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
for lang in VARIETIES {
let found = crate::hunspell::find_in(&dir, variety(lang).2);
let found = crate::hunspell::find_in(&dir, slot(lang).names());
assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}");
if let Some((aff, dic)) = found {
assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang));
@@ -215,7 +160,7 @@ mod tests {
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
}
for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] {
assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
assert_eq!(dictionary_name(lang), name);
}
std::fs::remove_dir_all(dir).unwrap();
+4 -4
View File
@@ -1,5 +1,5 @@
//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses
//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to
//! Spanish suggestions. The ranking is the shared dictionary-driven one
//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to
//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces,
//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary.
//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first
@@ -7,7 +7,7 @@
use crate::ProofLang;
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::cs::suggest::MAX_LETTERS;
use crate::affix_suggest::{MAX_LETTERS, Tuning};
/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's
/// case.
@@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec
_ => v,
});
}
for s in crate::cs::suggest::suggest_in(d, word, max) {
for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) {
if !out.contains(&s) {
out.push(s);
}
+10 -60
View File
@@ -9,73 +9,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Croatian Hunspell dictionary.
const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"];
/// The Croatian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Croatian");
/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Croatian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Croatian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals
@@ -99,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Croatian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+25 -327
View File
@@ -1,12 +1,11 @@
//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance,
//! missing spaces and a bounded search. Proper Croatian corrections depend on
//! the actual installed dictionary entries.
//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of
//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines.
use std::collections::{HashMap, HashSet};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`).
const LEADING_WORDS: &[&str] = &[
"a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će",
"bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad",
@@ -14,159 +13,13 @@ const LEADING_WORDS: &[&str] = &[
const TRAILING_WORDS: &[&str] =
&["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table
/// (`seje` → `seja` before `sejé`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// What Croatian adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiourntsjkvlmdpbgczčćšžđfh",
map: &["cčć", "dđ", "sš", "zž"],
..Tuning::DEFAULT
};
/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -175,179 +28,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']];
let map = if d.map().is_empty() { &fallback_map } else { d.map() };
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (`saúde` before `saudê`: both add an accent, but at the first difference).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
/// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća`
/// first, before a typo that costs more (`kuha`).
#[test]
fn diacritics_without_map_lines() {
let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap();
assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]);
}
#[test]
fn croatian_suggestions() {
crate::hr::test_dict::install();
for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] {
+12 -61
View File
@@ -9,78 +9,41 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
/// Java locales).
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
/// The Indonesian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Indonesian");
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Indonesian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it); `None` when there is none. A
/// dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is an Indonesian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Indonesian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return known(d, w)
|| w.split('-').all(|p| {
+4 -5
View File
@@ -1,7 +1,6 @@
//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses
//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem
//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is
//! hard-coded here.
//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing
//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID`
//! dictionary, so nothing Indonesian is hard-coded here.
use crate::affix::AffixDict;
@@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
crate::cs::suggest::suggest_in(d, word, max)
crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT)
}
#[cfg(test)]
+10 -59
View File
@@ -18,72 +18,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Italian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
/// The Italian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Italian");
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Italian spelling: no Hunspell it_IT dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Italian spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is an Italian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
@@ -125,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Italian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
let lower = w.to_lowercase();
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
return true;
+16 -336
View File
@@ -1,25 +1,9 @@
//! Italian spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`roma` → `Roma`);
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`,
//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`);
//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which an accent slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
//! dictionary lookups at most.
//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and
//! letters to try for a dictionary without a TRY line.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
const LEADING_WORDS: &[&str] = &[
@@ -35,158 +19,12 @@ const TRAILING_WORDS: &[&str] = &[
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
"è",
];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance.
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
/// What Italian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `à` → `a`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -195,182 +33,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent.
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
#[test]
fn nosuggest_stems_and_forms_are_never_offered() {
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
}
#[test]
fn italian_misspellings_get_the_right_suggestion() {
crate::it::test_dict::install();
+197 -135
View File
@@ -50,6 +50,7 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
pub mod affix;
mod affix_suggest;
pub mod bg;
pub mod cs;
pub mod dict;
@@ -65,6 +66,8 @@ pub mod patterns;
pub mod pl;
pub mod pt;
pub mod ru;
mod slot;
mod spelling;
pub mod tr;
pub mod vi;
@@ -157,57 +160,150 @@ pub fn ignore_word(w: &str) {
}
}
/// A language proofed against an installed Hunspell dictionary: all the functions below need to
/// know about it, so a language joins with one row in [`LANGUAGES`].
struct Language {
lang: ProofLang,
slot: &'static dyn slot::Slot,
/// The spelling mark's message, in the language.
message: &'static str,
is_correct: fn(&str, ProofLang, &Options) -> bool,
suggest: fn(&str, usize, ProofLang) -> Vec<String>,
/// Grammar issues; the flag says whether the first word must have a capital (not in a list
/// item).
grammar: fn(&str, ProofLang, bool) -> Vec<Issue>,
}
/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing
/// lists them. English has a built-in word list and is handled apart.
static LANGUAGES: [Language; 12] = [
Language {
lang: ProofLang::Pl,
slot: &pl::DICT,
message: "Możliwy błąd pisowni",
is_correct: |w, _, o| pl::is_correct_with_options(w, o),
suggest: |w, n, _| pl::suggest(w, n),
grammar: |t, _, c| pl::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Pt,
slot: &pt::DICT,
message: "Possível erro de ortografia",
is_correct: |w, _, o| pt::is_correct_with_options(w, o),
suggest: |w, n, _| pt::suggest(w, n),
grammar: |t, _, c| pt::grammar::check_with(t, c),
},
Language {
lang: ProofLang::It,
slot: &it::DICT,
message: "Possibile errore di ortografia",
is_correct: |w, _, o| it::is_correct_with_options(w, o),
suggest: |w, n, _| it::suggest(w, n),
grammar: |t, _, c| it::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Hr,
slot: &hr::DICT,
message: "Moguća pravopisna pogreška",
is_correct: |w, _, o| hr::is_correct_with_options(w, o),
suggest: |w, n, _| hr::suggest(w, n),
grammar: |t, _, c| hr::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Tr,
slot: &tr::DICT,
message: "Olası yazım hatası",
is_correct: |w, _, o| tr::is_correct_with_options(w, o),
suggest: |w, n, _| tr::suggest(w, n),
grammar: |t, _, c| tr::grammar::check_with(t, c),
},
Language {
lang: ProofLang::Ru,
slot: &ru::DICT,
message: "Возможна орфографическая ошибка",
is_correct: |w, _, o| ru::is_correct_with_options(w, o),
suggest: |w, n, _| ru::suggest(w, n),
grammar: |t, _, c| ru::grammar::check_with(t, c),
},
// Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no
// exception.
Language {
lang: ProofLang::Cs,
slot: &cs::DICT,
message: "Možná pravopisná chyba",
is_correct: |w, _, o| cs::is_correct_with_options(w, o),
suggest: |w, n, _| cs::suggest(w, n),
grammar: |t, _, _| cs::grammar::check(t),
},
Language {
lang: ProofLang::Id,
slot: &id::DICT,
message: "Kemungkinan salah eja",
is_correct: |w, _, o| id::is_correct_with_options(w, o),
suggest: |w, n, _| id::suggest(w, n),
grammar: |t, _, _| id::grammar::check(t),
},
Language {
lang: ProofLang::Es,
slot: &es::SPAIN_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::EsMx,
slot: &es::MEXICO_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::Es419,
slot: &es::LATIN_AMERICA_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
Language {
lang: ProofLang::EsUy,
slot: &es::URUGUAY_DICT,
message: "Posible error ortográfico",
is_correct: es::is_correct_with_options,
suggest: es::suggest,
grammar: |t, l, _| es::grammar::check(t, l),
},
];
/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof.
fn language(lang: ProofLang) -> Option<&'static Language> {
LANGUAGES.iter().find(|l| l.lang == lang)
}
/// Search `dir` for Hunspell dictionaries before the system locations (File › Options ›
/// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from
/// the new search path, and text is checked again.
pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
hunspell::set_dictionary_dir(dir);
pl::reload_dictionary();
pt::reload_dictionary();
it::reload_dictionary();
hr::reload_dictionary();
tr::reload_dictionary();
ru::reload_dictionary();
cs::reload_dictionary();
id::reload_dictionary();
es::reload_dictionary();
for l in &LANGUAGES {
l.slot.reload();
}
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
}
/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it
/// now (`None`: not installed, so the language has no spelling marks).
pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)> {
vec![
(ProofLang::Pl, pl::installed_path()),
(ProofLang::Pt, pt::installed_path()),
(ProofLang::It, it::installed_path()),
(ProofLang::Hr, hr::installed_path()),
(ProofLang::Tr, tr::installed_path()),
(ProofLang::Ru, ru::installed_path()),
(ProofLang::Cs, cs::installed_path()),
(ProofLang::Id, id::installed_path()),
(ProofLang::Es, es::installed_path(ProofLang::Es)),
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
(ProofLang::Es419, es::installed_path(ProofLang::Es419)),
(ProofLang::EsUy, es::installed_path(ProofLang::EsUy)),
]
LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect()
}
/// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse
/// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has
/// checked yet reports `None` (#2830).
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
let key = match lang {
ProofLang::Pl => "pl_PL",
ProofLang::Pt => "pt_BR",
ProofLang::It => "it_IT",
ProofLang::Hr => "hr_HR",
ProofLang::Tr => "tr_TR",
ProofLang::Id => "id_ID",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang),
_ => return None,
};
hunspell::load_error(key).map(|(_, reason)| reason)
hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason)
}
/// A number that changes whenever a word is added to or removed from the user dictionary, or the
@@ -438,36 +534,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p
/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English).
fn dictionary_loaded(lang: ProofLang) -> Option<bool> {
Some(match lang {
ProofLang::Pl => pl::has_dictionary(),
ProofLang::Pt => pt::has_dictionary(),
ProofLang::It => it::has_dictionary(),
ProofLang::Hr => hr::has_dictionary(),
ProofLang::Tr => tr::has_dictionary(),
ProofLang::Ru => ru::has_dictionary(),
ProofLang::Cs => cs::has_dictionary(),
ProofLang::Id => id::has_dictionary(),
l if l.is_spanish() => es::has_dictionary(l),
_ => return None,
})
language(lang).map(|l| l.slot.has_dictionary())
}
/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for;
/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`).
fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> {
match lang {
_ if installed => None,
ProofLang::Pl => Some("pl_PL"),
ProofLang::Pt => Some("pt_BR"),
ProofLang::It => Some("it_IT"),
ProofLang::Hr => Some("hr_HR"),
ProofLang::Tr => Some("tr_TR"),
ProofLang::Ru => Some("ru_RU"),
ProofLang::Cs => Some("cs_CZ"),
ProofLang::Id => Some("id_ID"),
l if l.is_spanish() => Some(es::dictionary_name(l)),
_ => None,
}
language(lang).filter(|_| !installed).map(|l| l.slot.name())
}
/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary
@@ -739,18 +812,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) {
return None;
}
let message = match lang {
ProofLang::Pl => "Możliwy błąd pisowni",
ProofLang::Hr => "Moguća pravopisna pogreška",
ProofLang::Pt => "Possível erro de ortografia",
ProofLang::It => "Possibile errore di ortografia",
ProofLang::Tr => "Olası yazım hatası",
ProofLang::Ru => "Возможна орфографическая ошибка",
ProofLang::Cs => "Možná pravopisná chyba",
ProofLang::Id => "Kemungkinan salah eja",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico",
_ => "Possible spelling mistake",
};
let message = language(lang).map_or("Possible spelling mistake", |l| l.message);
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
}
@@ -774,40 +836,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> {
/// checked as its composed form ([`composed`]).
pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
let word = &*composed(word);
match lang {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts),
ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts),
ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts),
ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts),
ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts),
ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word),
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => {
accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts)
}
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts),
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts),
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
if lang.is_english() {
return is_correct_english(word, lang, opts);
}
// A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks.
language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts))
}
/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed
/// ([`composed`]) like the dictionaries' words.
pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
let word = &*composed(word);
match lang {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang),
ProofLang::Pl => pl::suggest(word, max),
ProofLang::Hr => hr::suggest(word, max),
ProofLang::Pt => pt::suggest(word, max),
ProofLang::It => it::suggest(word, max),
ProofLang::Tr => tr::suggest(word, max),
ProofLang::Ru => ru::suggest(word, max),
ProofLang::Cs => cs::suggest(word, max),
ProofLang::Id => id::suggest(word, max),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
if lang.is_english() {
return suggest_english(word, max, lang);
}
language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang))
}
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
@@ -824,18 +867,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
match lang {
ProofLang::En => grammar_en(text, capital_first),
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
ProofLang::Pl => pl::grammar::check_with(text, capital_first),
ProofLang::Hr => hr::grammar::check_with(text, capital_first),
ProofLang::Pt => pt::grammar::check_with(text, capital_first),
ProofLang::It => it::grammar::check_with(text, capital_first),
ProofLang::Tr => tr::grammar::check_with(text, capital_first),
ProofLang::Ru => ru::grammar::check_with(text, capital_first),
// Czech has no sentence-capital rule, so a list item needs no exception.
ProofLang::Cs => cs::grammar::check(text),
ProofLang::Id => id::grammar::check(text),
// Spanish has no sentence-capital rule either.
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
_ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)),
}
}
@@ -1060,7 +1092,9 @@ mod tests {
use super::*;
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test.
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land
/// mid-test, and by those that save the whole user dictionary and load it back, which would
/// bring back a word another test just removed.
static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// #1063: the dictionaries folder is searched before the system locations; the dictionaries
@@ -1077,6 +1111,7 @@ mod tests {
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC),
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
@@ -1096,41 +1131,67 @@ mod tests {
let before = dictionary_generation();
set_dictionaries_folder(Some(dir.clone()));
assert!(dictionary_generation() > before, "cached checks are dropped");
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
// Every language is listed, Russian and Czech too (#3226: they were once missing, so
// neither showed up in File › Options › Proofing). A language reports the folder's file,
// or none while a dictionary given with install_dictionary() is in use (#3252 review);
// the languages' own tests install theirs concurrently, so an exact path would race them.
// The folder is searched first whatever is in use.
let found = installed_dictionaries();
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"];
assert_eq!(
found[7..],
found.iter().map(|(lang, _)| *lang).collect::<Vec<_>>(),
[
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
(ProofLang::EsMx, Some(dir.join("es_MX.dic"))),
(ProofLang::Es419, Some(dir.join("es_419.dic"))),
(ProofLang::EsUy, Some(dir.join("es_UY.dic"))),
ProofLang::Pl,
ProofLang::Pt,
ProofLang::It,
ProofLang::Hr,
ProofLang::Tr,
ProofLang::Ru,
ProofLang::Cs,
ProofLang::Id,
ProofLang::Es,
ProofLang::EsMx,
ProofLang::Es419,
ProofLang::EsUy
]
);
assert_eq!(
found[..4],
[
(ProofLang::Pl, Some(dir.join("pl_PL.dic"))),
(ProofLang::Pt, Some(dir.join("pt_BR.dic"))),
(ProofLang::It, Some(dir.join("it_IT.dic"))),
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
]
);
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
// installed_path() in either module), so neither ever showed up in File › Options ›
// Proofing, installed or not. Their languages, not their exact path, is what belongs
// here: unlike the four above, installed_path() for Russian and Czech also reflects
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
// test modules do, concurrently, in tests of their own — asserting an exact path here
// would race them for no reason this test needs to care about.
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
for ((lang, path), name) in found.iter().zip(names) {
let file = dir.join(format!("{name}.dic"));
assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}");
assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}");
}
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
}
/// A dictionary that fails to load says why in File › Options › Proofing in every language
/// that reads one; Russian and Czech were missing from the list that named their dictionaries
/// there.
#[test]
fn every_dictionary_language_reports_its_load_error() {
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
for (lang, _) in installed_dictionaries() {
let name = dictionary_needed(lang, false).unwrap();
let reason = format!("zq unreadable {name}");
let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone()));
assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}");
hunspell::clear_load_error(name);
}
}
/// Every proofed language other than English has a row in the language table, and only those.
#[test]
fn the_language_table_covers_the_proofed_languages() {
for l in ProofLang::PROOFED {
assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}");
}
for (i, row) in LANGUAGES.iter().enumerate() {
assert!(row.lang.is_proofed(), "{:?}", row.lang);
assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang);
}
}
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
/// with the rule; a plain paragraph's does, and later sentences in the item are still checked.
#[test]
@@ -1592,6 +1653,7 @@ mod tests {
/// #40: the user dictionary survives a save and a fresh load.
#[test]
fn user_dictionary_persists_through_a_file() {
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist"));
let _ = std::fs::remove_dir_all(&dir);
// The folder is created on the first save.
+14 -63
View File
@@ -17,78 +17,41 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Polish dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Polish Hunspell dictionary.
const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"];
/// The Polish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Polish");
/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then;
/// see [`has_dictionary`]).
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
DICT.get()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Polish spelling: no Hunspell pl_PL dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Polish spelling: {e}");
AffixDict::default()
})
}
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
/// for example one the user picked. On error the current dictionary stays.
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Polish dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced
/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`).
const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"];
/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals
/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so
/// is every word when no Polish dictionary is installed.
@@ -110,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Polish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
let w = w.replace('’', "'");
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
+19 -342
View File
@@ -1,26 +1,9 @@
//! Polish spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`warszawa` → `Warszawa`);
//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy`
//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most
//! common slip, so adding them ranks first, while taking away a diacritic the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` →
//! `huśtawka`, `pszyjaciel` → `przyjaciel`);
//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham
//! cię`), cheaper when one half is a short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which a diacritic slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand
//! dictionary lookups at most.
//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`),
//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`).
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning};
/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede
/// wszystkim`).
@@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[
/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`).
const TRAILING_WORDS: &[&str] =
&["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
/// What Polish adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx",
mistype_cost,
..Tuning::DEFAULT
};
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
// `Mozę`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the
/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the
/// writer did type is likelier right, so taking one away costs more than a known confusion from
/// the REP table (`żeka` → `rzeka` before `zeka`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it
/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo.
fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 {
let last = i + 1 == chars.len();
if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }
}
/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case.
@@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Diacritics and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half (`że ka`).
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
// Final devoicing can make a voiced consonant sound voiceless.
let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g'));
let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE };
col.offer(&s(&v), cost);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
found.sort_by_key(|(_, k)| *k);
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
+10 -59
View File
@@ -16,72 +16,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// File names of a Portuguese Hunspell dictionary.
const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"];
/// The Portuguese dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Portuguese");
/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Portuguese spelling: {e}");
AffixDict::default()
})
DICT.get()
}
/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Portuguese dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
}
/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)?
fn is_latin_letter(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Portuguese dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return true;
}
if !w.chars().any(is_latin_letter) {
return true; // other scripts: no dictionary
}
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
+13 -343
View File
@@ -1,26 +1,9 @@
//! Portuguese spelling suggestions, best first.
//!
//! Candidates come in rounds of rising cost, each checked against the dictionary:
//! 1. letter case (`brasília` → `Brasília`);
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`,
//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
//! ranks after the known confusions;
//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` →
//! `exceção`, `previlégio` → `privilégio`);
//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
//! in which an accent slip costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
//! dictionary lookups at most.
//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and
//! letters to try for a dictionary without a TRY line.
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{Split, Tuning};
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
const LEADING_WORDS: &[&str] = &[
@@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
"eu", "tu", "que", "de", "em", "é",
];
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 4;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
const COST_ADD_DIACRITIC: u32 = 4;
const COST_OTHER_MAP: u32 = 7;
const COST_DROP_DIACRITIC: u32 = 11;
const COST_REP: u32 = 10;
const COST_SPLIT: u32 = 12;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
return;
}
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The MAP group of each letter (its other members are its likely confusions).
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
}
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type
/// is likelier right, so taking one away costs more than a known confusion from the REP table
/// (`seje` → `seja` before `sejé`).
fn map_cost(from: char, to: char) -> u32 {
match (from.is_ascii(), to.is_ascii()) {
(true, false) => COST_ADD_DIACRITIC,
(false, true) => COST_DROP_DIACRITIC,
_ => COST_OTHER_MAP,
}
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
}
}
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
fn fold(c: char, map: &[Vec<char>]) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
}
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if related(ai, bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// What Portuguese adds to its dictionary's tables.
const TUNING: Tuning = Tuning {
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw",
..Tuning::DEFAULT
};
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
@@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Accents and other MAP slips.
let map = d.map();
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in d.rep() {
if from.is_empty() {
continue;
}
for (i, _) in input.match_indices(from.as_str()) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
// (`saúde` before `saudê`: both add an accent, but at the first difference).
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
/// case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let map = d.map();
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, &related, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::affix_suggest::{fold, weighted_distance};
fn has(word: &str, want: &str) {
let s = suggest(word, 8);
+51 -66
View File
@@ -28,83 +28,35 @@ mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::affix::AffixDict;
use crate::slot::DictSlot;
pub use suggest::suggest;
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Russian Hunspell dictionary.
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
/// The Russian dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Russian");
/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
/// see [`has_dictionary`]).
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.dict.get_or_insert(loaded).clone()
DICT.get()
}
fn load_installed() -> AffixDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Russian spelling: no Hunspell ru_RU dictionary installed");
return AffixDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Russian spelling: {e}");
AffixDict::default()
})
}
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
/// for example one the user picked. On error the current dictionary stays.
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Russian dictionary available (spelling is checked)?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)?
@@ -130,18 +82,30 @@ pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
/// short words in capitals are checked against the dictionary too, and with Ignore words that
/// contain numbers off, so are words with digits or superscripts.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // no Russian dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
let w = w.trim_matches('-');
let n = w.chars().count();
if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) {
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
return true;
}
if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
return true;
}
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
@@ -161,7 +125,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
return true;
}
if w.contains('-') {
return hyphenated(d, w);
return hyphenated(d, w, opts);
}
solid_compound(d, w)
}
@@ -340,7 +304,7 @@ fn known(d: &AffixDict, w: &str) -> bool {
/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles,
/// prefixes and linking vowels Russian joins with a hyphen.
fn hyphenated(d: &AffixDict, w: &str) -> bool {
fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
let parts: Vec<&str> = w.split('-').collect();
if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 {
return false;
@@ -353,7 +317,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
&& lp.iter().skip(k).all(|p| PARTICLES.contains(p))
{
let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default();
return known(d, &base) || (base.contains('-') && hyphenated(d, &base));
return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts));
}
match lp.as_slice() {
// `кое-кто`, `кой-какой`.
@@ -369,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
let last = parts.len() - 1;
parts.iter().enumerate().all(|(i, p)| {
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
return true;
}
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
@@ -497,4 +461,25 @@ mod tests {
crate::add_word("Зкщфрусскоеслово");
assert!(is_correct("зкщфрусскоеслово"));
}
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks
/// those words against the dictionary instead of accepting them.
#[test]
fn proofing_options_reach_the_russian_check() {
test_dict::install();
let lang = crate::ProofLang::Ru;
let defaults = crate::Options::default();
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default");
assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps));
assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults));
assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too");
assert!(crate::is_correct_in_with("превет2", lang, &defaults));
assert!(!crate::is_correct_in_with("превет2", lang, &check_digits));
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
}
}
+70 -334
View File
@@ -1,30 +1,14 @@
//! Russian spelling suggestions, best first.
//!
//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the
//! common confusions are defined here. Candidates come in rounds of rising cost, each checked
//! against the dictionary:
//! 1. letter case (`москва` → `Москва`);
//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`,
//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to
//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`);
//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or
//! doubled consonant, `здраствуй` → `здравствуй`);
//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a
//! short function word;
//! 5. one typo: a letter left out, added, swapped with the next or mistyped;
//! 6. when that finds fewer than asked for: the stems that start like the word and share the
//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within
//! a letter group costs half (two typos at most).
//!
//! The input is bounded (40 letters), and so is every round.
//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP`
//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` →
//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule
//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`).
use std::collections::{HashMap, HashSet};
use crate::affix::{AffixDict, Casing, capitalize, casing};
use crate::affix::AffixDict;
use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning};
/// Letters confused with each other: a slip inside a group is cheaper than another typo.
const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"];
/// Common confusions: a written sequence and what was meant.
const REP: &[(&str, &str)] = &[
("жы", "жи"),
@@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[
("ц", "тс"),
("тс", "ц"),
];
/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`).
const LEADING_WORDS: &[&str] = &[
"не",
@@ -123,158 +106,42 @@ const PREPOSITIONS: &[&str] = &[
const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"];
/// The Russian alphabet, when the dictionary has no `TRY` line.
const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё";
const MAX_LETTERS: usize = 40;
const MAX_MAP_CHANGES: usize = 3;
const MAX_MAP_VARIANTS: usize = 4096;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
const COST_CASE: u32 = 0;
/// `е` written for `ё`, or the other way round, is barely a slip.
const COST_YO: u32 = 3;
const COST_MAP: u32 = 7;
const COST_REP: u32 = 9;
const COST_SPLIT: u32 = 12;
/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`).
const COST_SPLIT_PREPOSITION: u32 = 17;
const COST_SPLIT_RARE: u32 = 18;
const COST_SWAP: u32 = 14;
const COST_DROP_OR_ADD: u32 = 15;
const COST_MISTYPE: u32 = 16;
const COST_SCAN: u32 = 20;
struct Collector<'a> {
d: &'a AffixDict,
/// The word, lower case.
input: String,
tested: HashSet<String>,
/// Suggestion → (cost, order found).
found: HashMap<String, (u32, usize)>,
seq: usize,
}
/// What Russian adds to its dictionary's tables.
const TUNING: Tuning =
Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT };
impl Collector<'_> {
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
/// word is in the dictionary, as written or capitalized (names).
fn offer(&mut self, cand: &str, cost: u32) {
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
return;
}
let mut words = Vec::new();
let mut cost = cost;
for part in cand.split(' ') {
if self.d.contains(part) {
words.push(part.to_string());
} else {
let c = capitalize(part);
if !self.d.contains(&c) {
return;
}
words.push(c);
// A name ranks after a common word at the same distance.
cost += 1;
}
}
self.keep(words.join(" "), cost);
}
/// Keep a suggestion known to be correct.
fn keep(&mut self, text: String, cost: u32) {
self.seq += 1;
let seq = self.seq;
let e = self.found.entry(text).or_insert((cost, seq));
if cost < e.0 {
*e = (cost, seq);
}
}
fn count_within(&self, cost: u32) -> usize {
self.found.values().filter(|(c, _)| *c <= cost).count()
}
}
/// The other members of each [`MAP`] group holding `c`.
fn alternatives(c: char) -> Vec<char> {
let mut out: Vec<char> = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect();
out.sort_unstable();
out.dedup();
out
}
/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip
/// (`ё` is optional in ordinary writing).
/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest.
fn map_cost(from: char, to: char) -> u32 {
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP }
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP }
}
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost.
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
if out.len() >= MAX_MAP_VARIANTS {
return;
/// A missing space: cheap after a short leading word or before a short trailing one, dearer after
/// a short preposition or between two longer words, never leaving a stray short half.
fn split(a: &str, b: &str) -> Option<u32> {
let long = |p: &str| p.chars().count() >= 3;
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
let preposition = PREPOSITIONS.contains(&a);
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
return None;
}
let Some(&c) = chars.get(i) else {
if changes > 0 {
out.push((buf.iter().collect(), cost));
}
return;
};
buf.push(c);
map_variants(chars, alts, i + 1, changes, cost, buf, out);
buf.pop();
if changes < MAX_MAP_CHANGES {
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
buf.push(a);
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
buf.pop();
}
if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
Some(COST_SPLIT_PREPOSITION)
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
Some(COST_SPLIT)
} else if long(a) && long(b) {
Some(COST_SPLIT_LONG)
} else {
None
}
}
/// A letter's group representative (the first of its first group), lower case: `ё` → `е`,
/// `з` → `з`, `с` → `з`.
fn fold(c: char) -> char {
let l = c.to_lowercase().next().unwrap_or(c);
MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l)
}
/// Optimal string alignment distance where a slip within a letter group costs 1 and any other
/// edit 2; `limit + 1` once it is certainly above `limit`.
fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 {
let m = b.len();
let over = limit + 1;
let mut prev2 = vec![0u32; m + 1];
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
let mut cur = vec![0u32; m + 1];
for (i, &ai) in a.iter().enumerate() {
if let Some(c) = cur.get_mut(0) {
*c = (i as u32 + 1) * 2;
}
let mut row_min = u32::MAX;
for (j, &bj) in b.iter().enumerate() {
let sub = if ai == bj {
0
} else if fold(ai) == fold(bj) {
1
} else {
2
};
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
v = v.min(at(&prev2, j - 1) + 2);
}
if let Some(c) = cur.get_mut(j + 1) {
*c = v;
}
row_min = row_min.min(v);
}
if m > 0 && row_min > limit {
return over;
}
std::mem::swap(&mut prev2, &mut prev);
std::mem::swap(&mut prev, &mut cur);
}
prev.get(m).copied().unwrap_or(over).min(over)
}
/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
suggest_in(&super::dictionary(), word, max)
@@ -282,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// [`suggest`] against a given dictionary.
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
let word = word.trim();
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
// 1. Letter case.
if d.contains(&input) && word != input {
col.keep(input.clone(), COST_CASE);
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
col.keep(capitalize(&input), COST_CASE);
}
// 2. Slips within a letter group.
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c)).collect();
let mut variants = Vec::new();
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
variants.sort_by_key(|v| v.1);
for (v, cost) in &variants {
col.offer(v, *cost);
}
// 3. Common confusions.
for (from, to) in REP {
for (i, _) in input.match_indices(from) {
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
col.offer(&format!("{a}{to}{b}"), COST_REP);
}
}
}
// 4. A missing space: cheap after a short leading word or before a short trailing one,
// dearer between two longer words, never leaving a stray short half.
for (i, _) in input.char_indices().skip(1) {
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
let long = |p: &str| p.chars().count() >= 3;
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
let preposition = PREPOSITIONS.contains(&a);
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
continue;
}
let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
COST_SPLIT_PREPOSITION
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
COST_SPLIT
} else if long(a) && long(b) {
COST_SPLIT_RARE
} else {
continue;
};
col.offer(&format!("{a} {b}"), cost);
}
// 5. One typo.
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
if letters.is_empty() {
letters = ALPHABET.chars().collect();
}
let s = |v: &[char]| v.iter().collect::<String>();
for i in 0..n.saturating_sub(1) {
if chars.get(i) != chars.get(i + 1) {
let mut v = chars.clone();
v.swap(i, i + 1);
col.offer(&s(&v), COST_SWAP);
}
}
for i in 0..n {
let mut v = chars.clone();
v.remove(i);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
for i in 0..=n {
for &c in &letters {
let mut v = chars.clone();
v.insert(i, c);
col.offer(&s(&v), COST_DROP_OR_ADD);
}
}
for i in 0..n {
for &c in &letters {
if chars.get(i) != Some(&c) {
let mut v = chars.clone();
if let Some(x) = v.get_mut(i) {
*x = c;
}
col.offer(&s(&v), COST_MISTYPE);
}
}
}
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars);
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
found.sort_by_key(|(_, k)| *k);
let mut out: Vec<String> = Vec::new();
for (text, _) in found {
let t = match case {
Casing::Initial => capitalize(&text),
Casing::Upper if n > 1 => text.to_uppercase(),
_ => text,
};
if t != word && !out.contains(&t) {
out.push(t);
}
if out.len() >= max {
break;
}
}
out
}
/// Round 6: rank the stems starting with the word's first letter (or a group relative of it,
/// either case) by shared letter pairs, then keep the forms of the best within two typos.
fn scan(col: &mut Collector, chars: &[char]) {
let d = col.d;
let Some(&first) = chars.first() else { return };
let folded: Vec<char> = chars.iter().map(|&c| fold(c)).collect();
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first)).collect();
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
firsts.extend(upper);
firsts.sort_unstable();
firsts.dedup();
let n = folded.len();
let mut best: Vec<(u32, &str)> = Vec::new();
for f in firsts {
let prefix = f.to_string();
for stem in d.stems_with_prefix(&prefix) {
let sf: Vec<char> = stem.chars().map(fold).collect();
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
if shared < 2 {
continue;
}
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
// beyond what an ending explains.
let extra = sf.len().saturating_sub(shared);
let gap = sf.len().abs_diff(n).saturating_sub(4);
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
if score == 0 {
continue;
}
if best.len() < SCAN_STEMS {
best.push((score, stem));
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
&& score > min.0
{
*min = (score, stem);
}
}
}
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let limit = 4;
for (_, stem) in best {
for form in d.forms(stem) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
}
let dist = weighted_distance(&lf, chars, limit);
if dist <= limit && lf.as_slice() != chars {
col.keep(form, COST_SCAN + dist * 3);
}
}
}
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
}
#[cfg(test)]
@@ -458,6 +161,36 @@ mod tests {
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
}
/// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip
/// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip;
/// a preposition is never split off a verb, and after a one-letter one a typo is likelier.
#[test]
fn russian_costs_rank_its_slips() {
let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap();
assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]);
assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]);
assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into()));
let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4);
assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}");
assert!(s.contains(&"в доме".into()), "{s:?}");
}
/// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never
/// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the
/// shared Latin-script check doesn't reach it.
#[test]
fn nosuggest_words_are_never_offered() {
let offered = |aff: &str, input: &str| {
let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap();
suggest_in(&d, input, 16)
};
assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared");
for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] {
let s = offered("SET UTF-8\nNOSUGGEST N\n", input);
assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}");
}
}
#[test]
fn russian_misspellings_get_the_right_suggestion() {
crate::ru::test_dict::install();
@@ -499,12 +232,15 @@ mod tests {
#[test]
fn distance_weighs_letter_groups_half() {
use crate::affix_suggest::{fold, weighted_distance};
let map: Vec<Vec<char>> = MAP.iter().map(|g| g.chars().collect()).collect();
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
let c = |s: &str| s.chars().collect::<Vec<_>>();
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1);
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1);
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2);
assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6);
assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3);
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1);
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1);
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2);
assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6);
assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3);
}
#[test]
+210
View File
@@ -0,0 +1,210 @@
//! The installed Hunspell dictionary of a proofing language, in one shape for every language:
//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`].
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use crate::affix::AffixDict;
/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own.
pub(crate) trait Dictionary: Default + Send + Sync {
/// The installed `.aff` and `.dic` files, read.
fn load(aff: &Path, dic: &Path) -> Result<Self, String>;
/// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed.
fn parse(aff: &str, dic: &str) -> Result<Self, String>;
/// No words: the language has no spelling marks.
fn is_empty(&self) -> bool;
}
impl Dictionary for AffixDict {
fn load(aff: &Path, dic: &Path) -> Result<Self, String> {
crate::hunspell::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
AffixDict::from_hunspell(aff, dic, &[])
}
fn is_empty(&self) -> bool {
AffixDict::is_empty(self)
}
}
/// What every language's slot answers, whatever its dictionary type (the language table in
/// lib.rs holds them side by side).
pub(crate) trait Slot: Sync {
/// See [`DictSlot::name`].
fn name(&self) -> &'static str;
/// See [`DictSlot::installed_path`].
fn installed_path(&self) -> Option<PathBuf>;
/// See [`DictSlot::has_dictionary`].
fn has_dictionary(&self) -> bool;
/// See [`DictSlot::reload`].
fn reload(&self);
}
impl<D: Dictionary> Slot for DictSlot<D> {
fn name(&self) -> &'static str {
DictSlot::name(self)
}
fn installed_path(&self) -> Option<PathBuf> {
DictSlot::installed_path(self)
}
fn has_dictionary(&self) -> bool {
DictSlot::has_dictionary(self)
}
fn reload(&self) {
DictSlot::reload(self);
}
}
/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on
/// disk (it then has no file to report, even if a matching file also happens to exist): one lock,
/// so a reload and an install can't interleave their two fields.
struct State<D> {
dict: Option<Arc<D>>,
in_memory: bool,
/// Counts reloads, so a load that started before one is not kept.
reloads: u64,
}
/// One language's dictionary.
pub(crate) struct DictSlot<D> {
/// File names the dictionary may have, the usual one first (also the key of its load error).
names: &'static [&'static str],
/// The language's English name, for the log.
english: &'static str,
state: RwLock<State<D>>,
}
impl<D: Dictionary> DictSlot<D> {
pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self {
DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) }
}
fn read(&self) -> RwLockReadGuard<'_, State<D>> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State<D>> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
/// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names.
pub(crate) fn name(&self) -> &'static str {
self.names.first().copied().unwrap_or_default()
}
/// The file names the dictionary may have, the usual one first.
#[cfg(test)]
pub(crate) fn names(&self) -> &'static [&'static str] {
self.names
}
/// The dictionary: the installed one (read on first use), one given with [`Self::install`], or
/// an empty one when none is available (the language's spelling is off then).
pub(crate) fn get(&self) -> Arc<D> {
let reloads = {
let s = self.read();
if let Some(d) = s.dict.as_ref() {
return d.clone();
}
s.reloads
};
self.keep(reloads, Arc::new(self.load_installed()))
}
/// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the
/// load may have read the old folder: it serves this one check and the next one loads again.
fn keep(&self, reloads: u64, loaded: Arc<D>) -> Arc<D> {
let mut s = self.write();
if s.reloads != reloads {
return s.dict.clone().unwrap_or(loaded);
}
// Another thread (or an install) may have filled it meanwhile: that one wins.
s.dict.get_or_insert(loaded).clone()
}
fn load_installed(&self) -> D {
let name = self.name();
let Some((aff, dic)) = crate::hunspell::find(self.names) else {
crate::hunspell::clear_load_error(name);
log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english);
return D::default();
};
crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("{} spelling: {e}", self.english);
D::default()
})
}
/// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for
/// example one the user picked. On error the current dictionary stays.
pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> {
let d = D::parse(aff, dic)?;
let mut slot = self.write();
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub(crate) fn reload(&self) {
let mut slot = self.write();
slot.dict = None;
slot.in_memory = false;
slot.reloads = slot.reloads.wrapping_add(1);
}
/// The `.dic` file of the installed dictionary, searching the dictionaries folder and the
/// system locations now (File › Options › Proofing shows it); `None` when there is none, or
/// when the dictionary in use was given with [`Self::install`] and so has no file.
pub(crate) fn installed_path(&self) -> Option<PathBuf> {
if self.read().in_memory {
return None;
}
crate::hunspell::find(self.names).map(|(_, dic)| dic)
}
/// Is a dictionary available (spelling is checked)?
pub(crate) fn has_dictionary(&self) -> bool {
!self.get().is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
static SLOT: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_TEST"], "Test");
#[test]
fn an_installed_dictionary_has_no_file_until_reloaded() {
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
SLOT.install("", "1\nkot\n").unwrap();
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
assert_eq!(SLOT.installed_path(), None);
SLOT.reload();
assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none");
assert_eq!(SLOT.name(), "zz_SLOT_TEST");
}
/// A load that started before a dictionaries-folder change is not kept after its reload.
#[test]
fn a_load_from_before_a_reload_is_not_kept() {
let slot: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test");
let before = slot.read().reloads;
slot.reload();
let _ = slot.keep(before, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_none(), "the next check loads from the new folder");
let now = slot.read().reloads;
let _ = slot.keep(now, Arc::new(AffixDict::default()));
assert!(slot.read().dict.is_some(), "a load since the reload is kept");
}
}
+52
View File
@@ -0,0 +1,52 @@
//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian,
//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the
//! File › Options › Proofing choices. Each language goes on from there with its own rules.
use crate::Options;
use crate::affix::AffixDict;
/// Is `c` a letter of the Latin script (the only one these dictionaries cover)?
fn latin(c: char) -> bool {
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
}
/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it
/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single
/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the
/// options skip those, or it is in another script.
pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> {
if d.is_empty() {
return None;
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return None;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
return None;
}
w.chars().any(latin).then_some(w)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn words_needing_no_lookup() {
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
let defaults = Options::default();
assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary");
for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] {
assert_eq!(word_to_check(&d, w, &defaults), None, "{w}");
}
assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot"));
assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot"));
let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults };
assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO"));
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
}
}
+14
View File
@@ -26,6 +26,20 @@ pub struct TrDict {
max_suffix_len: usize,
}
impl crate::slot::Dictionary for TrDict {
fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
super::load(aff, dic)
}
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
TrDict::from_hunspell(aff, dic)
}
fn is_empty(&self) -> bool {
TrDict::is_empty(self)
}
}
impl TrDict {
/// Returns true if empty (no dictionary loaded).
pub fn is_empty(&self) -> bool {
+43 -46
View File
@@ -15,39 +15,26 @@ pub mod dict;
pub mod grammar;
pub mod hyphen;
pub mod suggest;
#[cfg(test)]
pub(crate) mod test_dict;
use std::sync::{Arc, RwLock};
use std::sync::Arc;
use crate::slot::DictSlot;
pub use dict::TrDict;
pub use suggest::suggest;
/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<TrDict>>> = RwLock::new(None);
/// File names of a Turkish Hunspell dictionary.
const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"];
/// The Turkish dictionary, found on first use (see [`DictSlot`]).
pub(crate) static DICT: DictSlot<TrDict> = DictSlot::new(NAMES, "Turkish");
/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with
/// [`install_dictionary`], or an empty one when none is available.
pub fn dictionary() -> Arc<TrDict> {
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
slot.get_or_insert(loaded).clone()
}
fn load_installed() -> TrDict {
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
crate::hunspell::clear_load_error(NAMES[0]);
log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed");
return TrDict::default();
};
crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Turkish spelling: {e}");
TrDict::default()
})
DICT.get()
}
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
@@ -57,28 +44,15 @@ pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDic
TrDict::from_hunspell(&aff_text, &dic_text)
}
/// Use this Turkish dictionary from now on.
/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
/// current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = TrDict::from_hunspell(aff, dic)?;
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
Ok(())
}
/// Reload the dictionary on next check.
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
pub fn installed_path() -> Option<std::path::PathBuf> {
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
DICT.install(aff, dic)
}
/// Is a Turkish dictionary available?
pub fn has_dictionary() -> bool {
!dictionary().is_empty()
DICT.has_dictionary()
}
/// Is `word` spelled correctly in Turkish?
@@ -86,15 +60,26 @@ pub fn is_correct(word: &str) -> bool {
is_correct_with(&dictionary(), word)
}
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain
/// numbers off, words with digits are checked against the dictionary too.
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
check(&dictionary(), word, opts)
}
/// [`is_correct`] against a given dictionary.
pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
check(d, word, &crate::Options::default())
}
/// [`is_correct_with_options`] against a given dictionary.
fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
if d.is_empty() {
return true; // No Turkish dictionary installed: no spelling marks
}
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
let w = w.trim_matches(['\'', '’']);
let n = w.chars().count();
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return true;
}
// Check user dictionary first
@@ -119,7 +104,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
}
// Compound words separated by hyphens (e.g., `afyon-karahisar`)
if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) {
if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) {
return true;
}
@@ -145,11 +130,8 @@ mod tests {
}
#[test]
fn installed_dictionary_checks_words_if_present() {
// Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check.
if !has_dictionary() {
return;
}
fn installed_dictionary_checks_words() {
test_dict::install();
assert!(is_correct("kitap"));
assert!(is_correct("kitaplar"));
assert!(is_correct("evde"));
@@ -164,4 +146,19 @@ mod tests {
crate::add_word("benimbilmedigimkelime");
assert!(is_correct("benimbilmedigimkelime"));
}
/// With Ignore words that contain numbers off, Turkish checks words with digits against the
/// dictionary instead of accepting them, hyphen parts included.
#[test]
fn proofing_options_reach_the_turkish_check() {
test_dict::install();
let lang = crate::ProofLang::Tr;
let defaults = crate::Options::default();
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
assert!(crate::is_correct_in_with("kitapp2", lang, &defaults));
assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits));
assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits));
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
}
}
+1 -4
View File
@@ -318,10 +318,7 @@ mod tests {
#[test]
fn typo_transposition_is_suggested() {
// Suggestions come from the installed Turkish dictionary; without one there are none.
if !super::super::has_dictionary() {
return;
}
crate::tr::test_dict::install();
let sugs = suggest("sorna", 3);
assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}");
+22
View File
@@ -0,0 +1,22 @@
//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
/// No affix rules: the forms the tests look up are listed whole.
pub const AFF: &str = "SET UTF-8\n";
/// The words the Turkish tests expect to be known.
pub const DIC: &str = "7
kitap
kitaplar
ev
evde
olacaklardan
sonra
geldin
";
pub fn install() {
if let Err(e) = super::install_dictionary(AFF, DIC) {
panic!("test dictionary: {e}");
}
}