mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-11 23:32:20 +00:00
Merge storytold/wordcraft#4191
This commit is contained in:
@@ -0,0 +1,539 @@
|
||||
//! Spelling suggestions from an affix dictionary, one engine for every language that reads one
|
||||
//! (Czech, Croatian, Indonesian, Italian, Polish, Portuguese, Russian, Spanish). Candidates come
|
||||
//! in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`warszawa` → `Warszawa`);
|
||||
//! 2. missing or wrong diacritics and other MAP slips, a few letters changed (`zolw` → `żółw`):
|
||||
//! adding a missing accent is the commonest slip and ranks first, while taking away one the
|
||||
//! writer typed ranks after the known confusions;
|
||||
//! 3. the REP table of common confusions (`żeka` → `rzeka`);
|
||||
//! 4. a missing space (`napewno` → `na pewno`);
|
||||
//! 5. keyboard neighbours from the KEY rows, then one typo: a letter left out, added, swapped with
|
||||
//! the next or mistyped (TRY letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (MAP relatives
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance in
|
||||
//! which a MAP slip costs half (two typos at most).
|
||||
//!
|
||||
//! The tables come from the dictionary; a language's [`Tuning`] adds the ones its dictionaries may
|
||||
//! lack and the slips no table can say. The input is bounded ([`MAX_LETTERS`]), and so is every
|
||||
//! round, so a call costs a few thousand dictionary lookups at most.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Longest word suggestions are looked for, in letters.
|
||||
pub(crate) const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
|
||||
pub(crate) const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
pub(crate) const COST_SPLIT: u32 = 12;
|
||||
/// A missing space leaving a half shorter than three letters ([`Split::Anywhere`]).
|
||||
const COST_SPLIT_SHORT: u32 = 17;
|
||||
/// A missing space between two longer halves with no short word beside it ([`Split::Near`]).
|
||||
pub(crate) const COST_SPLIT_LONG: u32 = 18;
|
||||
const COST_KEY: u32 = 13;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
pub(crate) const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
/// What a language adds to its dictionary's tables.
|
||||
pub(crate) struct Tuning {
|
||||
/// Where a missing space may go (round 4).
|
||||
pub split: Split,
|
||||
/// TRY letters for a dictionary without a TRY line.
|
||||
pub alphabet: &'static str,
|
||||
/// MAP groups the language adds to its dictionary's (each string is one group: `"cčć"`).
|
||||
pub map: &'static [&'static str],
|
||||
/// REP confusions the language adds to its dictionary's.
|
||||
pub rep: &'static [(&'static str, &'static str)],
|
||||
/// The cost of a REP confusion.
|
||||
pub rep_cost: u32,
|
||||
/// Most letters a MAP variant changes.
|
||||
pub max_map_changes: usize,
|
||||
/// The cost of changing letter `from` to its MAP relative `to`.
|
||||
pub map_cost: fn(char, char) -> u32,
|
||||
/// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead.
|
||||
pub mistype_cost: fn(&[char], usize, char) -> u32,
|
||||
}
|
||||
|
||||
/// Where a missing space may go.
|
||||
pub(crate) enum Split {
|
||||
/// Anywhere, dearer when a half is shorter than three letters.
|
||||
Anywhere,
|
||||
/// After a short word that often runs into the next one (`na pewno`, `w ogóle`) or before one
|
||||
/// that runs into the word before (`kocham cię`), dearer between two longer halves, and never
|
||||
/// leaving a stray short half (`że ka`).
|
||||
Near { leading: &'static [&'static str], trailing: &'static [&'static str] },
|
||||
/// The language's own rule: the cost of the halves `a b`, or `None` to skip them.
|
||||
Custom(fn(&str, &str) -> Option<u32>),
|
||||
}
|
||||
|
||||
impl Tuning {
|
||||
/// The dictionary's tables alone (Czech, Indonesian, Spanish).
|
||||
pub(crate) const DEFAULT: Tuning =
|
||||
Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost };
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (typing without diacritics); an accent the writer did type is likelier right,
|
||||
/// so taking one away costs more than a known confusion from the REP table.
|
||||
pub(crate) fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Any mistyped letter costs the same.
|
||||
fn mistype_cost(_: &[char], _: usize, _: char) -> u32 {
|
||||
COST_MISTYPE
|
||||
}
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
/// Candidate → the cheapest cost it was offered at (rounds don't run in rising cost).
|
||||
tested: HashMap<String, u32>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input {
|
||||
return;
|
||||
}
|
||||
match self.tested.get_mut(cand) {
|
||||
Some(prev) if *prev <= cost => return,
|
||||
Some(prev) => *prev = cost,
|
||||
None => {
|
||||
self.tested.insert(cand.to_string(), cost);
|
||||
}
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moze` → `może`, then
|
||||
// `Mozę`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP groups: the dictionary's, then the language's.
|
||||
fn groups<'a>(d: &'a AffixDict, t: &Tuning) -> Cow<'a, [Vec<char>]> {
|
||||
if t.map.is_empty() {
|
||||
return Cow::Borrowed(d.map());
|
||||
}
|
||||
Cow::Owned(d.map().iter().cloned().chain(t.map.iter().map(|g| g.chars().collect())).collect())
|
||||
}
|
||||
|
||||
/// The other members of every MAP group of `c` (its likely confusions), each once.
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
let mut out: Vec<char> = Vec::new();
|
||||
for x in map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()) {
|
||||
if x != c && !out.contains(&x) {
|
||||
out.push(x);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
|
||||
/// letter by letter), with its cost, each written after `buf`.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some((&c, rest)) = chars.split_first() else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
|
||||
buf.push(c);
|
||||
map_variants(rest, later, t, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < t.max_map_changes {
|
||||
for &a in here {
|
||||
buf.push(a);
|
||||
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its first group), lower case: `ż` → `z`, `ó` → `o`.
|
||||
pub(crate) fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
pub(crate) fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// The cost of a missing space between `a` and `b`, or `None` when the split isn't offered.
|
||||
fn split_cost(split: &Split, a: &str, b: &str) -> Option<u32> {
|
||||
let long = |part: &str| part.chars().count() >= 3;
|
||||
match split {
|
||||
Split::Anywhere => Some(if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_SHORT }),
|
||||
Split::Near { leading, trailing } => {
|
||||
if (leading.contains(&a) && long(b)) || (long(a) && trailing.contains(&b)) {
|
||||
Some(COST_SPLIT)
|
||||
} else if long(a) && long(b) {
|
||||
Some(COST_SPLIT_LONG)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Split::Custom(cost) => cost(a, b),
|
||||
}
|
||||
}
|
||||
|
||||
/// Spelling suggestions for `word` from `d`, best first, at most `max`, in the word's letter case.
|
||||
pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
|
||||
add_case(&mut col, word, &input);
|
||||
let map = groups(d, t);
|
||||
add_map_slips(&mut col, &chars, &map, t);
|
||||
add_confusions(&mut col, &input, t);
|
||||
add_splits(&mut col, &input, &t.split);
|
||||
add_key_neighbours(&mut col, &chars);
|
||||
add_typos(&mut col, &chars, t);
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars, &map);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (the first changed letter breaks ties).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 1: letter case.
|
||||
fn add_case(col: &mut Collector, word: &str, input: &str) {
|
||||
if col.d.contains(input) && word != input {
|
||||
col.keep(input.to_string(), COST_CASE);
|
||||
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
|
||||
col.keep(capitalize(input), COST_CASE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 2: accents and other MAP slips.
|
||||
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 3: common confusions.
|
||||
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
|
||||
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
|
||||
for (from, to) in rep {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 4: a missing space; both halves must be dictionary words.
|
||||
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
if let Some(cost) = split_cost(split, a, b) {
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
|
||||
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
|
||||
for row in col.d.key().split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [a, b] = pair else { continue };
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let replacement = if c == *a {
|
||||
Some(*b)
|
||||
} else if c == *b {
|
||||
Some(*a)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = replacement;
|
||||
}
|
||||
col.offer(&v.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
|
||||
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
|
||||
/// equal-cost candidates come out in that order.
|
||||
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
|
||||
let d = col.d;
|
||||
let n = chars.len();
|
||||
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
|
||||
let mut letters: Vec<char> = Vec::new();
|
||||
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
|
||||
if !letters.contains(&c) {
|
||||
letters.push(c);
|
||||
}
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.to_vec();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.to_vec();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.to_vec();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
let d = col.d;
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
|
||||
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
|
||||
let suggestions = suggest_in(&d, "počítačz", 8, &Tuning::DEFAULT);
|
||||
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
|
||||
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
|
||||
}
|
||||
|
||||
/// A candidate offered again at a lower cost (a REP confusion after a MAP diacritic drop)
|
||||
/// ranks at the lower one.
|
||||
#[test]
|
||||
fn a_cheaper_later_offer_wins() {
|
||||
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
|
||||
let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
|
||||
col.offer("cafe", COST_DROP_DIACRITIC);
|
||||
col.offer("cafe", COST_REP);
|
||||
col.offer("cafe", COST_SCAN);
|
||||
assert_eq!(col.found.get("cafe").map(|e| e.0), Some(COST_REP));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distance_weighs_map_slips_half() {
|
||||
let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']];
|
||||
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
||||
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
||||
assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4);
|
||||
assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3);
|
||||
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
|
||||
}
|
||||
|
||||
/// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY
|
||||
/// line, and a split rule decides which missing spaces are offered.
|
||||
#[test]
|
||||
fn tuning_adds_to_the_dictionary_tables() {
|
||||
let d = AffixDict::from_hunspell("", "3\nčaj\nkot\nna\n", &[]).unwrap();
|
||||
let tuned = Tuning { map: &["cč"], alphabet: "t", rep: &[("xy", "kot")], ..Tuning::DEFAULT };
|
||||
assert!(!suggest_in(&d, "caj", 4, &Tuning::DEFAULT).contains(&"čaj".into()));
|
||||
assert!(suggest_in(&d, "caj", 4, &tuned).contains(&"čaj".into()), "the language's MAP group");
|
||||
assert!(suggest_in(&d, "xy", 4, &tuned).contains(&"kot".into()), "the language's REP pair");
|
||||
assert!(suggest_in(&d, "ko", 4, &tuned).contains(&"kot".into()), "the language's TRY letters");
|
||||
let near = Tuning { split: Split::Near { leading: &["na"], trailing: &[] }, ..Tuning::DEFAULT };
|
||||
assert!(suggest_in(&d, "nakot", 4, &near).contains(&"na kot".into()));
|
||||
assert!(!suggest_in(&d, "kotna", 4, &near).contains(&"kot na".into()), "no stray short half");
|
||||
assert!(suggest_in(&d, "kotna", 4, &Tuning::DEFAULT).contains(&"kot na".into()));
|
||||
}
|
||||
}
|
||||
+10
-75
@@ -7,88 +7,35 @@ pub(crate) mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Czech Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
|
||||
|
||||
/// The Czech dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Czech");
|
||||
|
||||
/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Czech spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Czech dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals
|
||||
@@ -112,21 +59,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Czech dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
@@ -1,168 +1,8 @@
|
||||
//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits
|
||||
//! and a stem scan. All linguistic tables are read from the installed dictionary, so Indonesian
|
||||
//! ([`crate::id`]) uses the same ranking with its own dictionary.
|
||||
//! Czech suggestions: the shared ranking ([`crate::affix_suggest`]) with the installed
|
||||
//! dictionary's own MAP/REP/TRY/KEY tables and nothing Czech hard-coded.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Longest word suggestions are looked for, in letters (Spanish shares the bound).
|
||||
pub(crate) const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 17;
|
||||
const COST_KEY: u32 = 13;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (a capitalized name).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (typing without diacritics); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::Tuning;
|
||||
|
||||
/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -171,237 +11,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
let map = d.map();
|
||||
add_case_suggestions(&mut col, &input, word);
|
||||
add_map_suggestions(&mut col, &chars, map);
|
||||
add_replacement_suggestions(&mut col, &input, d.rep());
|
||||
add_split_suggestions(&mut col, &input);
|
||||
add_keyboard_suggestions(&mut col, &chars, d.key());
|
||||
add_typing_suggestions(&mut col, &chars, d.try_chars());
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (the first changed letter breaks ties).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn add_case_suggestions(col: &mut Collector, input: &str, word: &str) {
|
||||
if col.d.contains(input) && word != input {
|
||||
col.keep(input.to_string(), COST_CASE);
|
||||
} else {
|
||||
let capitalized = capitalize(input);
|
||||
if col.d.contains(&capitalized) && word != capitalized {
|
||||
col.keep(capitalized, COST_CASE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_map_suggestions(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
let alternatives: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alternatives, 0, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
variants.sort_by_key(|variant| variant.1);
|
||||
for (variant, cost) in variants {
|
||||
col.offer(&variant, cost);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_replacement_suggestions(col: &mut Collector, input: &str, replacements: &[(String, String)]) {
|
||||
for (from, to) in replacements {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (index, _) in input.match_indices(from) {
|
||||
if let (Some(prefix), Some(suffix)) = (input.get(..index), input.get(index + from.len()..)) {
|
||||
col.offer(&format!("{prefix}{to}{suffix}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_split_suggestions(col: &mut Collector, input: &str) {
|
||||
for (index, _) in input.char_indices().skip(1) {
|
||||
let (Some(first), Some(second)) = (input.get(..index), input.get(index..)) else { continue };
|
||||
let cost = if first.chars().count() >= 3 && second.chars().count() >= 3 { COST_SPLIT } else { COST_SPLIT_RARE };
|
||||
col.offer(&format!("{first} {second}"), cost);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_keyboard_suggestions(col: &mut Collector, chars: &[char], key_rows: &str) {
|
||||
for row in key_rows.split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [first, second] = pair else { continue };
|
||||
for (index, &character) in chars.iter().enumerate() {
|
||||
let replacement = if character == *first {
|
||||
Some(*second)
|
||||
} else if character == *second {
|
||||
Some(*first)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut candidate = chars.to_vec();
|
||||
if let Some(character) = candidate.get_mut(index) {
|
||||
*character = replacement;
|
||||
}
|
||||
col.offer(&candidate.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_typing_suggestions(col: &mut Collector, chars: &[char], try_chars: &str) {
|
||||
let mut letters: Vec<char> = try_chars.chars().filter(|character| character.is_lowercase()).take(128).collect();
|
||||
letters.sort_unstable();
|
||||
letters.dedup();
|
||||
let word = |candidate: &[char]| candidate.iter().collect::<String>();
|
||||
add_swap_suggestions(col, chars, &word);
|
||||
add_drop_suggestions(col, chars, &word);
|
||||
add_insert_suggestions(col, chars, &letters, &word);
|
||||
add_mistype_suggestions(col, chars, &letters, &word);
|
||||
}
|
||||
|
||||
fn add_swap_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len().saturating_sub(1) {
|
||||
if chars.get(index) != chars.get(index + 1) {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.swap(index, index + 1);
|
||||
col.offer(&word(&candidate), COST_SWAP);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_drop_suggestions(col: &mut Collector, chars: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len() {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.remove(index);
|
||||
col.offer(&word(&candidate), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
|
||||
fn add_insert_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..=chars.len() {
|
||||
for &letter in letters {
|
||||
let mut candidate = chars.to_vec();
|
||||
candidate.insert(index, letter);
|
||||
col.offer(&word(&candidate), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_mistype_suggestions(col: &mut Collector, chars: &[char], letters: &[char], word: &impl Fn(&[char]) -> String) {
|
||||
for index in 0..chars.len() {
|
||||
for &letter in letters {
|
||||
if chars.get(index) != Some(&letter) {
|
||||
let mut candidate = chars.to_vec();
|
||||
if let Some(character) = candidate.get_mut(index) {
|
||||
*character = letter;
|
||||
}
|
||||
col.offer(&word(&candidate), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
|
||||
let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();
|
||||
let suggestions = suggest_in(&d, "počítačz", 8);
|
||||
assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}");
|
||||
assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggestions_are_fast_enough() {
|
||||
crate::cs::test_dict::install();
|
||||
|
||||
+21
-76
@@ -16,16 +16,19 @@ mod suggest;
|
||||
pub(crate) mod test_dict;
|
||||
pub mod voseo;
|
||||
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::ProofLang;
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first
|
||||
/// use, then the installed one (or an empty one).
|
||||
static DICTS: [RwLock<Option<Arc<AffixDict>>>; 4] = [const { RwLock::new(None) }; 4];
|
||||
/// The dictionary of each variety, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static SPAIN_DICT: DictSlot<AffixDict> = DictSlot::new(SPAIN, "Spanish (Spain)");
|
||||
pub(crate) static MEXICO_DICT: DictSlot<AffixDict> = DictSlot::new(MEXICO, "Spanish (Mexico)");
|
||||
pub(crate) static LATIN_AMERICA_DICT: DictSlot<AffixDict> = DictSlot::new(LATIN_AMERICA, "Spanish (Latin America)");
|
||||
pub(crate) static URUGUAY_DICT: DictSlot<AffixDict> = DictSlot::new(URUGUAY, "Spanish (Uruguay)");
|
||||
|
||||
/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the
|
||||
/// generic one RLA-ES publishes).
|
||||
@@ -40,84 +43,38 @@ const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "e
|
||||
/// or generic one.
|
||||
const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"];
|
||||
|
||||
/// `lang`'s variety: its slot in [`DICTS`], its English name (for the log) and its dictionary's
|
||||
/// file names; Spain for any other language.
|
||||
fn variety(lang: ProofLang) -> (usize, &'static str, &'static [&'static str]) {
|
||||
/// `lang`'s variety's dictionary; Spain's for any other language.
|
||||
fn slot(lang: ProofLang) -> &'static DictSlot<AffixDict> {
|
||||
match lang {
|
||||
ProofLang::EsMx => (1, "Spanish (Mexico)", MEXICO),
|
||||
ProofLang::Es419 => (2, "Spanish (Latin America)", LATIN_AMERICA),
|
||||
ProofLang::EsUy => (3, "Spanish (Uruguay)", URUGUAY),
|
||||
_ => (0, "Spanish (Spain)", SPAIN),
|
||||
ProofLang::EsMx => &MEXICO_DICT,
|
||||
ProofLang::Es419 => &LATIN_AMERICA_DICT,
|
||||
ProofLang::EsUy => &URUGUAY_DICT,
|
||||
_ => &SPAIN_DICT,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`,
|
||||
/// `es_UY`): the first of its file names, used for the status-bar hint and load errors.
|
||||
pub fn dictionary_name(lang: ProofLang) -> &'static str {
|
||||
variety(lang).2.first().copied().unwrap_or("es_ES")
|
||||
slot(lang).name()
|
||||
}
|
||||
|
||||
/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given
|
||||
/// with [`install_dictionary`], or an empty one when none is available (spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
pub fn dictionary(lang: ProofLang) -> Arc<AffixDict> {
|
||||
let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) };
|
||||
if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed(lang));
|
||||
let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed(lang: ProofLang) -> AffixDict {
|
||||
let (_, name, files) = variety(lang);
|
||||
let key = dictionary_name(lang);
|
||||
let Some((aff, dic)) = crate::hunspell::find(files) else {
|
||||
crate::hunspell::clear_load_error(key);
|
||||
log::info!("{name} spelling: no Hunspell {key} dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("{name} spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
slot(lang).get()
|
||||
}
|
||||
|
||||
/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text,
|
||||
/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
if let Some(slot) = DICTS.get(variety(lang).0) {
|
||||
*slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget every variety's loaded dictionary, so the next check looks for an installed one again
|
||||
/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
for slot in &DICTS {
|
||||
*slot.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder
|
||||
/// or the system locations now (File › Options › Proofing shows it); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path(lang: ProofLang) -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(variety(lang).2).map(|(_, dic)| dic)
|
||||
slot(lang).install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a dictionary available for `lang`'s variety (spelling is checked)?
|
||||
pub fn has_dictionary(lang: ProofLang) -> bool {
|
||||
!dictionary(lang).is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
slot(lang).has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in
|
||||
@@ -143,21 +100,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, lang: ProofLang, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Spanish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang));
|
||||
}
|
||||
@@ -200,7 +145,7 @@ mod tests {
|
||||
std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
|
||||
for lang in VARIETIES {
|
||||
let found = crate::hunspell::find_in(&dir, variety(lang).2);
|
||||
let found = crate::hunspell::find_in(&dir, slot(lang).names());
|
||||
assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}");
|
||||
if let Some((aff, dic)) = found {
|
||||
assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang));
|
||||
@@ -215,7 +160,7 @@ mod tests {
|
||||
std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap();
|
||||
}
|
||||
for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] {
|
||||
assert_eq!(crate::hunspell::find_in(&dir, variety(lang).2).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
|
||||
assert_eq!(crate::hunspell::find_in(&dir, slot(lang).names()).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic"))));
|
||||
assert_eq!(dictionary_name(lang), name);
|
||||
}
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses
|
||||
//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to
|
||||
//! Spanish suggestions. The ranking is the shared dictionary-driven one
|
||||
//! ([`crate::affix_suggest`]: case, MAP — the Spanish dictionaries map each vowel to
|
||||
//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces,
|
||||
//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary.
|
||||
//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use crate::ProofLang;
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::cs::suggest::MAX_LETTERS;
|
||||
use crate::affix_suggest::{MAX_LETTERS, Tuning};
|
||||
|
||||
/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's
|
||||
/// case.
|
||||
@@ -30,7 +30,7 @@ pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec
|
||||
_ => v,
|
||||
});
|
||||
}
|
||||
for s in crate::cs::suggest::suggest_in(d, word, max) {
|
||||
for s in crate::affix_suggest::suggest_in(d, word, max, &Tuning::DEFAULT) {
|
||||
if !out.contains(&s) {
|
||||
out.push(s);
|
||||
}
|
||||
|
||||
+10
-60
@@ -9,73 +9,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Croatian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Croatian Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["hr_HR", "hr-HR", "hr"];
|
||||
|
||||
/// The Croatian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Croatian");
|
||||
|
||||
/// The Croatian dictionary: the installed Hunspell `hr_HR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Croatian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Croatian spelling: no Hunspell hr_HR dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Croatian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Croatian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `hr_HR` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Croatian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Croatian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Croatian? Numbers, single letters, short words in capitals
|
||||
@@ -99,21 +61,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Croatian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
+25
-327
@@ -1,12 +1,11 @@
|
||||
//! Croatian spelling suggestions using dictionary MAP/REP rules, edit distance,
|
||||
//! missing spaces and a bounded search. Proper Croatian corrections depend on
|
||||
//! the actual installed dictionary entries.
|
||||
//! Croatian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! the words Croatian runs into their neighbours (`neznam` → `ne znam`), the letter groups of
|
||||
//! Croatian diacritics and letters to try, for a dictionary without MAP or TRY lines.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
|
||||
/// Short words often run into the next one (`ne znam`, `na primjer`, `da li`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
"a", "ali", "i", "ili", "da", "do", "od", "na", "ne", "ni", "o", "po", "pod", "pri", "sa", "s", "se", "su", "u", "uz", "za", "iz", "je", "će",
|
||||
"bi", "bih", "što", "koji", "koja", "koje", "kao", "kad", "dok", "bez", "pred", "nad",
|
||||
@@ -14,159 +13,13 @@ const LEADING_WORDS: &[&str] = &[
|
||||
const TRAILING_WORDS: &[&str] =
|
||||
&["je", "su", "se", "ga", "mi", "ti", "mu", "joj", "ih", "im", "li", "će", "ću", "bi", "bih", "i", "a", "u", "na", "za", "od", "do"];
|
||||
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Croatian typed without accents: `coracao`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table
|
||||
/// (`seje` → `seja` before `sejé`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
/// What Croatian adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aeiourntsjkvlmdpbgczčćšžđfh",
|
||||
map: &["cčć", "dđ", "sš", "zž"],
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
/// Spelling suggestions for a Croatian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -175,179 +28,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let fallback_map = vec![vec!['c', 'č', 'ć'], vec!['d', 'đ'], vec!['s', 'š'], vec!['z', 'ž']];
|
||||
let map = if d.map().is_empty() { &fallback_map } else { d.map() };
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiourntsjkvlmdpbgczčćšžđfh".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (`saúde` before `saudê`: both add an accent, but at the first difference).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn nosuggest_stems_and_forms_are_never_offered() {
|
||||
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
|
||||
}
|
||||
/// A dictionary without MAP lines still gets the Croatian diacritic groups: `kuca` → `kuća`
|
||||
/// first, before a typo that costs more (`kuha`).
|
||||
#[test]
|
||||
fn diacritics_without_map_lines() {
|
||||
let d = AffixDict::from_hunspell("", "2\nkuća\nkuha\n", &[]).unwrap();
|
||||
assert_eq!(suggest_in(&d, "kuca", 1), ["kuća"]);
|
||||
}
|
||||
#[test]
|
||||
fn croatian_suggestions() {
|
||||
crate::hr::test_dict::install();
|
||||
for (wrong, right) in [("kuca", "kuća"), ("racunalo", "računalo"), ("covjek", "čovjek"), ("rijec", "riječ"), ("skolla", "škola")] {
|
||||
|
||||
+12
-61
@@ -9,78 +9,41 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Indonesian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Indonesian Hunspell dictionary (`in` is the old ISO 639 code, still used by
|
||||
/// Java locales).
|
||||
const NAMES: &[&str] = &["id_ID", "id-ID", "id", "in_ID"];
|
||||
|
||||
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
|
||||
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
|
||||
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
|
||||
/// The Indonesian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Indonesian");
|
||||
|
||||
/// The Indonesian dictionary: the installed Hunspell `id_ID` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Indonesian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Indonesian spelling: no Hunspell id_ID dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Indonesian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Indonesian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `id_ID` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it); `None` when there is none. A
|
||||
/// dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is an Indonesian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Indonesian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
/// Clitics PUEBI joins with a hyphen to a capitalized word, an abbreviation or a number
|
||||
/// (`se-Indonesia`, `KTP-nya`, `Jakarta-lah`); they aren't dictionary words on their own.
|
||||
const CLITICS: &[&str] = &["se", "nya", "ku", "mu", "lah", "kah", "pun"];
|
||||
|
||||
/// Is `word` spelled correctly in Indonesian? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `DPR`, `KTP`), words with digits and words in other scripts are accepted, and so
|
||||
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Indonesian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return known(d, w)
|
||||
|| w.split('-').all(|p| {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! Indonesian suggestions. The ranking is the dictionary-driven one Czech uses
|
||||
//! ([`crate::cs::suggest::suggest_in`]: case, MAP, REP, missing spaces, KEY, TRY edits and a stem
|
||||
//! scan); every table comes from the installed `id_ID` dictionary, so nothing Indonesian is
|
||||
//! hard-coded here.
|
||||
//! Indonesian suggestions: the shared ranking ([`crate::affix_suggest`]: case, MAP, REP, missing
|
||||
//! spaces, KEY, TRY edits and a stem scan); every table comes from the installed `id_ID`
|
||||
//! dictionary, so nothing Indonesian is hard-coded here.
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
@@ -12,7 +11,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
crate::cs::suggest::suggest_in(d, word, max)
|
||||
crate::affix_suggest::suggest_in(d, word, max, &crate::affix_suggest::Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+10
-59
@@ -18,72 +18,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Italian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
|
||||
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
|
||||
|
||||
/// The Italian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Italian");
|
||||
|
||||
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Italian spelling: no Hunspell it_IT dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Italian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `it_IT` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is an Italian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
|
||||
@@ -125,21 +88,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Italian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
let lower = w.to_lowercase();
|
||||
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
|
||||
return true;
|
||||
|
||||
+16
-336
@@ -1,25 +1,9 @@
|
||||
//! Italian spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`roma` → `Roma`);
|
||||
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`,
|
||||
//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the
|
||||
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`);
|
||||
//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which an accent slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Italian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
|
||||
//! words Italian runs into their neighbours (`perme` → `per me`, `amilano` → `a milano`) and
|
||||
//! letters to try for a dictionary without a TRY line.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
@@ -35,158 +19,12 @@ const TRAILING_WORDS: &[&str] = &[
|
||||
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
|
||||
"è",
|
||||
];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance.
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
/// What Italian adds to its dictionary's tables.
|
||||
const TUNING: Tuning =
|
||||
Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, alphabet: "aeiortnslcdupmghfbqvàèéìòóùz", ..Tuning::DEFAULT
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `à` → `a`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -195,182 +33,24 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent.
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nosuggest_stems_and_forms_are_never_offered() {
|
||||
crate::affix::tests::assert_nosuggest_is_honoured(suggest_in);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_misspellings_get_the_right_suggestion() {
|
||||
crate::it::test_dict::install();
|
||||
|
||||
+197
-135
@@ -50,6 +50,7 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod affix;
|
||||
mod affix_suggest;
|
||||
pub mod bg;
|
||||
pub mod cs;
|
||||
pub mod dict;
|
||||
@@ -65,6 +66,8 @@ pub mod patterns;
|
||||
pub mod pl;
|
||||
pub mod pt;
|
||||
pub mod ru;
|
||||
mod slot;
|
||||
mod spelling;
|
||||
pub mod tr;
|
||||
pub mod vi;
|
||||
|
||||
@@ -157,57 +160,150 @@ pub fn ignore_word(w: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// A language proofed against an installed Hunspell dictionary: all the functions below need to
|
||||
/// know about it, so a language joins with one row in [`LANGUAGES`].
|
||||
struct Language {
|
||||
lang: ProofLang,
|
||||
slot: &'static dyn slot::Slot,
|
||||
/// The spelling mark's message, in the language.
|
||||
message: &'static str,
|
||||
is_correct: fn(&str, ProofLang, &Options) -> bool,
|
||||
suggest: fn(&str, usize, ProofLang) -> Vec<String>,
|
||||
/// Grammar issues; the flag says whether the first word must have a capital (not in a list
|
||||
/// item).
|
||||
grammar: fn(&str, ProofLang, bool) -> Vec<Issue>,
|
||||
}
|
||||
|
||||
/// Every language proofed against a Hunspell dictionary, in the order File › Options › Proofing
|
||||
/// lists them. English has a built-in word list and is handled apart.
|
||||
static LANGUAGES: [Language; 12] = [
|
||||
Language {
|
||||
lang: ProofLang::Pl,
|
||||
slot: &pl::DICT,
|
||||
message: "Możliwy błąd pisowni",
|
||||
is_correct: |w, _, o| pl::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| pl::suggest(w, n),
|
||||
grammar: |t, _, c| pl::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Pt,
|
||||
slot: &pt::DICT,
|
||||
message: "Possível erro de ortografia",
|
||||
is_correct: |w, _, o| pt::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| pt::suggest(w, n),
|
||||
grammar: |t, _, c| pt::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::It,
|
||||
slot: &it::DICT,
|
||||
message: "Possibile errore di ortografia",
|
||||
is_correct: |w, _, o| it::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| it::suggest(w, n),
|
||||
grammar: |t, _, c| it::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Hr,
|
||||
slot: &hr::DICT,
|
||||
message: "Moguća pravopisna pogreška",
|
||||
is_correct: |w, _, o| hr::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| hr::suggest(w, n),
|
||||
grammar: |t, _, c| hr::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Tr,
|
||||
slot: &tr::DICT,
|
||||
message: "Olası yazım hatası",
|
||||
is_correct: |w, _, o| tr::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| tr::suggest(w, n),
|
||||
grammar: |t, _, c| tr::grammar::check_with(t, c),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Ru,
|
||||
slot: &ru::DICT,
|
||||
message: "Возможна орфографическая ошибка",
|
||||
is_correct: |w, _, o| ru::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| ru::suggest(w, n),
|
||||
grammar: |t, _, c| ru::grammar::check_with(t, c),
|
||||
},
|
||||
// Czech, Indonesian and Spanish have no sentence-capital rule, so a list item needs no
|
||||
// exception.
|
||||
Language {
|
||||
lang: ProofLang::Cs,
|
||||
slot: &cs::DICT,
|
||||
message: "Možná pravopisná chyba",
|
||||
is_correct: |w, _, o| cs::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| cs::suggest(w, n),
|
||||
grammar: |t, _, _| cs::grammar::check(t),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Id,
|
||||
slot: &id::DICT,
|
||||
message: "Kemungkinan salah eja",
|
||||
is_correct: |w, _, o| id::is_correct_with_options(w, o),
|
||||
suggest: |w, n, _| id::suggest(w, n),
|
||||
grammar: |t, _, _| id::grammar::check(t),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Es,
|
||||
slot: &es::SPAIN_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::EsMx,
|
||||
slot: &es::MEXICO_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::Es419,
|
||||
slot: &es::LATIN_AMERICA_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
Language {
|
||||
lang: ProofLang::EsUy,
|
||||
slot: &es::URUGUAY_DICT,
|
||||
message: "Posible error ortográfico",
|
||||
is_correct: es::is_correct_with_options,
|
||||
suggest: es::suggest,
|
||||
grammar: |t, l, _| es::grammar::check(t, l),
|
||||
},
|
||||
];
|
||||
|
||||
/// `lang`'s row in [`LANGUAGES`]; `None` for English and the languages WordCraft doesn't proof.
|
||||
fn language(lang: ProofLang) -> Option<&'static Language> {
|
||||
LANGUAGES.iter().find(|l| l.lang == lang)
|
||||
}
|
||||
|
||||
/// Search `dir` for Hunspell dictionaries before the system locations (File › Options ›
|
||||
/// Proofing › Dictionaries folder, #1063); `None` clears it. The dictionaries load again from
|
||||
/// the new search path, and text is checked again.
|
||||
pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
|
||||
hunspell::set_dictionary_dir(dir);
|
||||
pl::reload_dictionary();
|
||||
pt::reload_dictionary();
|
||||
it::reload_dictionary();
|
||||
hr::reload_dictionary();
|
||||
tr::reload_dictionary();
|
||||
ru::reload_dictionary();
|
||||
cs::reload_dictionary();
|
||||
id::reload_dictionary();
|
||||
es::reload_dictionary();
|
||||
for l in &LANGUAGES {
|
||||
l.slot.reload();
|
||||
}
|
||||
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it
|
||||
/// now (`None`: not installed, so the language has no spelling marks).
|
||||
pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)> {
|
||||
vec![
|
||||
(ProofLang::Pl, pl::installed_path()),
|
||||
(ProofLang::Pt, pt::installed_path()),
|
||||
(ProofLang::It, it::installed_path()),
|
||||
(ProofLang::Hr, hr::installed_path()),
|
||||
(ProofLang::Tr, tr::installed_path()),
|
||||
(ProofLang::Ru, ru::installed_path()),
|
||||
(ProofLang::Cs, cs::installed_path()),
|
||||
(ProofLang::Id, id::installed_path()),
|
||||
(ProofLang::Es, es::installed_path(ProofLang::Es)),
|
||||
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
|
||||
(ProofLang::Es419, es::installed_path(ProofLang::Es419)),
|
||||
(ProofLang::EsUy, es::installed_path(ProofLang::EsUy)),
|
||||
]
|
||||
LANGUAGES.iter().map(|l| (l.lang, l.slot.installed_path())).collect()
|
||||
}
|
||||
|
||||
/// Why the dictionary found for `lang` failed to load (unsupported encoding, too large, a parse
|
||||
/// error), if it was read and failed. Doesn't read anything itself: a dictionary nobody has
|
||||
/// checked yet reports `None` (#2830).
|
||||
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
|
||||
let key = match lang {
|
||||
ProofLang::Pl => "pl_PL",
|
||||
ProofLang::Pt => "pt_BR",
|
||||
ProofLang::It => "it_IT",
|
||||
ProofLang::Hr => "hr_HR",
|
||||
ProofLang::Tr => "tr_TR",
|
||||
ProofLang::Id => "id_ID",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang),
|
||||
_ => return None,
|
||||
};
|
||||
hunspell::load_error(key).map(|(_, reason)| reason)
|
||||
hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason)
|
||||
}
|
||||
|
||||
/// A number that changes whenever a word is added to or removed from the user dictionary, or the
|
||||
@@ -438,36 +534,13 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p
|
||||
|
||||
/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English).
|
||||
fn dictionary_loaded(lang: ProofLang) -> Option<bool> {
|
||||
Some(match lang {
|
||||
ProofLang::Pl => pl::has_dictionary(),
|
||||
ProofLang::Pt => pt::has_dictionary(),
|
||||
ProofLang::It => it::has_dictionary(),
|
||||
ProofLang::Hr => hr::has_dictionary(),
|
||||
ProofLang::Tr => tr::has_dictionary(),
|
||||
ProofLang::Ru => ru::has_dictionary(),
|
||||
ProofLang::Cs => cs::has_dictionary(),
|
||||
ProofLang::Id => id::has_dictionary(),
|
||||
l if l.is_spanish() => es::has_dictionary(l),
|
||||
_ => return None,
|
||||
})
|
||||
language(lang).map(|l| l.slot.has_dictionary())
|
||||
}
|
||||
|
||||
/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for;
|
||||
/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`).
|
||||
fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> {
|
||||
match lang {
|
||||
_ if installed => None,
|
||||
ProofLang::Pl => Some("pl_PL"),
|
||||
ProofLang::Pt => Some("pt_BR"),
|
||||
ProofLang::It => Some("it_IT"),
|
||||
ProofLang::Hr => Some("hr_HR"),
|
||||
ProofLang::Tr => Some("tr_TR"),
|
||||
ProofLang::Ru => Some("ru_RU"),
|
||||
ProofLang::Cs => Some("cs_CZ"),
|
||||
ProofLang::Id => Some("id_ID"),
|
||||
l if l.is_spanish() => Some(es::dictionary_name(l)),
|
||||
_ => None,
|
||||
}
|
||||
language(lang).filter(|_| !installed).map(|l| l.slot.name())
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary
|
||||
@@ -739,18 +812,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
|
||||
if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) {
|
||||
return None;
|
||||
}
|
||||
let message = match lang {
|
||||
ProofLang::Pl => "Możliwy błąd pisowni",
|
||||
ProofLang::Hr => "Moguća pravopisna pogreška",
|
||||
ProofLang::Pt => "Possível erro de ortografia",
|
||||
ProofLang::It => "Possibile errore di ortografia",
|
||||
ProofLang::Tr => "Olası yazım hatası",
|
||||
ProofLang::Ru => "Возможна орфографическая ошибка",
|
||||
ProofLang::Cs => "Možná pravopisná chyba",
|
||||
ProofLang::Id => "Kemungkinan salah eja",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico",
|
||||
_ => "Possible spelling mistake",
|
||||
};
|
||||
let message = language(lang).map_or("Possible spelling mistake", |l| l.message);
|
||||
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
|
||||
}
|
||||
|
||||
@@ -774,40 +836,21 @@ fn composed(word: &str) -> std::borrow::Cow<'_, str> {
|
||||
/// checked as its composed form ([`composed`]).
|
||||
pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
let word = &*composed(word);
|
||||
match lang {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts),
|
||||
ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct_with_options(word, opts),
|
||||
ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct_with_options(word, opts),
|
||||
ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct_with_options(word, opts),
|
||||
ProofLang::It => accepted_by_options(word, opts) || it::is_correct_with_options(word, opts),
|
||||
ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word),
|
||||
ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => {
|
||||
accepted_by_options(word, opts) || es::is_correct_with_options(word, lang, opts)
|
||||
}
|
||||
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct_with_options(word, opts),
|
||||
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct_with_options(word, opts),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
|
||||
if lang.is_english() {
|
||||
return is_correct_english(word, lang, opts);
|
||||
}
|
||||
// A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks.
|
||||
language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts))
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`), composed
|
||||
/// ([`composed`]) like the dictionaries' words.
|
||||
pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
let word = &*composed(word);
|
||||
match lang {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang),
|
||||
ProofLang::Pl => pl::suggest(word, max),
|
||||
ProofLang::Hr => hr::suggest(word, max),
|
||||
ProofLang::Pt => pt::suggest(word, max),
|
||||
ProofLang::It => it::suggest(word, max),
|
||||
ProofLang::Tr => tr::suggest(word, max),
|
||||
ProofLang::Ru => ru::suggest(word, max),
|
||||
ProofLang::Cs => cs::suggest(word, max),
|
||||
ProofLang::Id => id::suggest(word, max),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
if lang.is_english() {
|
||||
return suggest_english(word, max, lang);
|
||||
}
|
||||
language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang))
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
|
||||
@@ -824,18 +867,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
|
||||
match lang {
|
||||
ProofLang::En => grammar_en(text, capital_first),
|
||||
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
|
||||
ProofLang::Pl => pl::grammar::check_with(text, capital_first),
|
||||
ProofLang::Hr => hr::grammar::check_with(text, capital_first),
|
||||
ProofLang::Pt => pt::grammar::check_with(text, capital_first),
|
||||
ProofLang::It => it::grammar::check_with(text, capital_first),
|
||||
ProofLang::Tr => tr::grammar::check_with(text, capital_first),
|
||||
ProofLang::Ru => ru::grammar::check_with(text, capital_first),
|
||||
// Czech has no sentence-capital rule, so a list item needs no exception.
|
||||
ProofLang::Cs => cs::grammar::check(text),
|
||||
ProofLang::Id => id::grammar::check(text),
|
||||
// Spanish has no sentence-capital rule either.
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
_ => language(lang).map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1060,7 +1092,9 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land mid-test.
|
||||
/// Held by tests that swap or rely on the shared Spanish dictionary, so a folder reset can't land
|
||||
/// mid-test, and by those that save the whole user dictionary and load it back, which would
|
||||
/// bring back a word another test just removed.
|
||||
static DICTIONARY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// #1063: the dictionaries folder is searched before the system locations; the dictionaries
|
||||
@@ -1077,6 +1111,7 @@ mod tests {
|
||||
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
|
||||
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
|
||||
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
|
||||
("tr_TR", tr::test_dict::AFF, tr::test_dict::DIC),
|
||||
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
|
||||
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
|
||||
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
|
||||
@@ -1096,41 +1131,67 @@ mod tests {
|
||||
let before = dictionary_generation();
|
||||
set_dictionaries_folder(Some(dir.clone()));
|
||||
assert!(dictionary_generation() > before, "cached checks are dropped");
|
||||
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
|
||||
// Every language is listed, Russian and Czech too (#3226: they were once missing, so
|
||||
// neither showed up in File › Options › Proofing). A language reports the folder's file,
|
||||
// or none while a dictionary given with install_dictionary() is in use (#3252 review);
|
||||
// the languages' own tests install theirs concurrently, so an exact path would race them.
|
||||
// The folder is searched first whatever is in use.
|
||||
let found = installed_dictionaries();
|
||||
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
|
||||
let names = ["pl_PL", "pt_BR", "it_IT", "hr_HR", "tr_TR", "ru_RU", "cs_CZ", "id_ID", "es_ES", "es_MX", "es_419", "es_UY"];
|
||||
assert_eq!(
|
||||
found[7..],
|
||||
found.iter().map(|(lang, _)| *lang).collect::<Vec<_>>(),
|
||||
[
|
||||
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
|
||||
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
|
||||
(ProofLang::EsMx, Some(dir.join("es_MX.dic"))),
|
||||
(ProofLang::Es419, Some(dir.join("es_419.dic"))),
|
||||
(ProofLang::EsUy, Some(dir.join("es_UY.dic"))),
|
||||
ProofLang::Pl,
|
||||
ProofLang::Pt,
|
||||
ProofLang::It,
|
||||
ProofLang::Hr,
|
||||
ProofLang::Tr,
|
||||
ProofLang::Ru,
|
||||
ProofLang::Cs,
|
||||
ProofLang::Id,
|
||||
ProofLang::Es,
|
||||
ProofLang::EsMx,
|
||||
ProofLang::Es419,
|
||||
ProofLang::EsUy
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
found[..4],
|
||||
[
|
||||
(ProofLang::Pl, Some(dir.join("pl_PL.dic"))),
|
||||
(ProofLang::Pt, Some(dir.join("pt_BR.dic"))),
|
||||
(ProofLang::It, Some(dir.join("it_IT.dic"))),
|
||||
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
|
||||
]
|
||||
);
|
||||
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
|
||||
// installed_path() in either module), so neither ever showed up in File › Options ›
|
||||
// Proofing, installed or not. Their languages, not their exact path, is what belongs
|
||||
// here: unlike the four above, installed_path() for Russian and Czech also reflects
|
||||
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
|
||||
// test modules do, concurrently, in tests of their own — asserting an exact path here
|
||||
// would race them for no reason this test needs to care about.
|
||||
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
|
||||
for ((lang, path), name) in found.iter().zip(names) {
|
||||
let file = dir.join(format!("{name}.dic"));
|
||||
assert!(path.is_none() || path.as_ref() == Some(&file), "{lang:?}: {path:?}");
|
||||
assert_eq!(hunspell::find(&[name]).map(|(_, dic)| dic), Some(file), "{lang:?}");
|
||||
}
|
||||
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
|
||||
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
|
||||
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
|
||||
}
|
||||
|
||||
/// A dictionary that fails to load says why in File › Options › Proofing in every language
|
||||
/// that reads one; Russian and Czech were missing from the list that named their dictionaries
|
||||
/// there.
|
||||
#[test]
|
||||
fn every_dictionary_language_reports_its_load_error() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for (lang, _) in installed_dictionaries() {
|
||||
let name = dictionary_needed(lang, false).unwrap();
|
||||
let reason = format!("zq unreadable {name}");
|
||||
let _ = hunspell::note_load::<()>(name, std::path::Path::new("zq.dic"), Err(reason.clone()));
|
||||
assert_eq!(dictionary_load_error(lang), Some(reason), "{lang:?}");
|
||||
hunspell::clear_load_error(name);
|
||||
}
|
||||
}
|
||||
|
||||
/// Every proofed language other than English has a row in the language table, and only those.
|
||||
#[test]
|
||||
fn the_language_table_covers_the_proofed_languages() {
|
||||
for l in ProofLang::PROOFED {
|
||||
assert_eq!(language(l).is_some(), !l.is_english(), "{l:?}");
|
||||
}
|
||||
for (i, row) in LANGUAGES.iter().enumerate() {
|
||||
assert!(row.lang.is_proofed(), "{:?}", row.lang);
|
||||
assert!(LANGUAGES.iter().skip(i + 1).all(|other| other.lang != row.lang), "{:?} twice", row.lang);
|
||||
}
|
||||
}
|
||||
|
||||
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
|
||||
/// with the rule; a plain paragraph's does, and later sentences in the item are still checked.
|
||||
#[test]
|
||||
@@ -1592,6 +1653,7 @@ mod tests {
|
||||
/// #40: the user dictionary survives a save and a fresh load.
|
||||
#[test]
|
||||
fn user_dictionary_persists_through_a_file() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "persist"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
// The folder is created on the first save.
|
||||
|
||||
+14
-63
@@ -17,78 +17,41 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Polish dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Polish Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"];
|
||||
|
||||
/// The Polish dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Polish");
|
||||
|
||||
/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Polish spelling: no Hunspell pl_PL dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Polish spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
|
||||
/// for example one the user picked. On error the current dictionary stays.
|
||||
/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Polish dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Polish dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced
|
||||
/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`).
|
||||
const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"];
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so
|
||||
/// is every word when no Polish dictionary is installed.
|
||||
@@ -110,21 +73,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Polish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
let w = w.replace('’', "'");
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
|
||||
+19
-342
@@ -1,26 +1,9 @@
|
||||
//! Polish spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`warszawa` → `Warszawa`);
|
||||
//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy`
|
||||
//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most
|
||||
//! common slip, so adding them ranks first, while taking away a diacritic the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` →
|
||||
//! `huśtawka`, `pszyjaciel` → `przyjaciel`);
|
||||
//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham
|
||||
//! cię`), cheaper when one half is a short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which a diacritic slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Polish spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with the
|
||||
//! words Polish runs into their neighbours (`napewno` → `na pewno`, `kochamcię` → `kocham cię`),
|
||||
//! letters to try for a dictionary without a TRY line, and final devoicing (`chlep` → `chleb`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{COST_ADD_DIACRITIC, COST_MISTYPE, Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede
|
||||
/// wszystkim`).
|
||||
@@ -32,162 +15,20 @@ const LEADING_WORDS: &[&str] = &[
|
||||
/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`).
|
||||
const TRAILING_WORDS: &[&str] =
|
||||
&["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
/// What Polish adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx",
|
||||
mistype_cost,
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moze` → `może`, then
|
||||
// `Mozę`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the
|
||||
/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the
|
||||
/// writer did type is likelier right, so taking one away costs more than a known confusion from
|
||||
/// the REP table (`żeka` → `rzeka` before `zeka`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
/// Final devoicing makes a voiced consonant at the end of a word sound voiceless, so writing it
|
||||
/// voiceless (`chlep` for `chleb`) is a likelier slip than any other typo.
|
||||
fn mistype_cost(chars: &[char], i: usize, c: char) -> u32 {
|
||||
let last = i + 1 == chars.len();
|
||||
if last && matches!((chars.get(i), c), (Some('p'), 'b') | (Some('t'), 'd') | (Some('k'), 'g')) { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE }
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case.
|
||||
@@ -197,177 +38,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Diacritics and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half (`że ka`).
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
// Final devoicing can make a voiced consonant sound voiceless.
|
||||
let final_devoicing = i == n - 1 && matches!((chars[i], c), ('p', 'b') | ('t', 'd') | ('k', 'g'));
|
||||
let cost = if final_devoicing { COST_ADD_DIACRITIC - 1 } else { COST_MISTYPE };
|
||||
col.offer(&s(&v), cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
found.sort_by_key(|(_, k)| *k);
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
|
||||
+10
-59
@@ -16,72 +16,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Portuguese dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Portuguese Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["pt_BR", "pt-BR", "pt"];
|
||||
|
||||
/// The Portuguese dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Portuguese");
|
||||
|
||||
/// The Portuguese dictionary: the installed Hunspell `pt_BR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Portuguese spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Portuguese spelling: no Hunspell pt_BR dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Portuguese spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Use this Portuguese dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Portuguese dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #1063); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Portuguese dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Portuguese dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Portuguese? Numbers, single letters, short words in capitals
|
||||
@@ -105,21 +68,9 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Portuguese dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
|
||||
+13
-343
@@ -1,26 +1,9 @@
|
||||
//! Portuguese spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`brasília` → `Brasília`);
|
||||
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`coraçao` → `coração`,
|
||||
//! `saude` → `saúde`), up to four letters changed — Portuguese typed without accents is the
|
||||
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`seje` → `seja`, `excessão` →
|
||||
//! `exceção`, `previlégio` → `privilégio`);
|
||||
//! 4. a missing space (`nãoé` → `não é`, `paraela` → `para ela`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which an accent slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
||||
//! dictionary lookups at most.
|
||||
//! Portuguese spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! the words Portuguese runs into their neighbours (`nãoé` → `não é`, `paraela` → `para ela`) and
|
||||
//! letters to try for a dictionary without a TRY line.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{Split, Tuning};
|
||||
|
||||
/// Short words often run into the next one (`não é`, `da água`, `para isso`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
@@ -35,162 +18,13 @@ const TRAILING_WORDS: &[&str] = &[
|
||||
"me", "te", "se", "lhe", "nos", "vos", "lhes", "o", "a", "os", "as", "ele", "ela", "eles", "elas", "isso", "isto", "aquilo", "mim", "ti", "si",
|
||||
"eu", "tu", "que", "de", "em", "é",
|
||||
];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moça`, then `Moça`).
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
if !text.split(' ').all(|part| self.d.can_suggest(part)) {
|
||||
return;
|
||||
}
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Portuguese typed without accents: `coracao`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table
|
||||
/// (`seje` → `seja` before `sejé`).
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `ã` → `a`, `ç` → `c`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`á`, `c`/`ç`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
/// What Portuguese adds to its dictionary's tables.
|
||||
const TUNING: Tuning = Tuning {
|
||||
split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS },
|
||||
alphabet: "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw",
|
||||
..Tuning::DEFAULT
|
||||
};
|
||||
|
||||
/// Spelling suggestions for a Portuguese word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
@@ -199,177 +33,13 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiousrntmdlcpvgqbzfhjxçãõáéíóúâêôkw".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
// (`saúde` before `saudê`: both add an accent, but at the first difference).
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
|
||||
+51
-66
@@ -28,83 +28,35 @@ mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Russian Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
|
||||
|
||||
/// The Russian dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<AffixDict> = DictSlot::new(NAMES, "Russian");
|
||||
|
||||
/// The Russian dictionary: the installed Hunspell `ru_RU` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Russian spelling: no Hunspell ru_RU dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Russian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8),
|
||||
/// for example one the user picked. On error the current dictionary stays.
|
||||
/// Use this Russian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Russian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Cyrillic script (the only one the Russian dictionary covers)?
|
||||
@@ -130,18 +82,30 @@ pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words in UPPERCASE off,
|
||||
/// short words in capitals are checked against the dictionary too, and with Ignore words that
|
||||
/// contain numbers off, so are words with digits or superscripts.
|
||||
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
|
||||
check(&dictionary(), word, opts)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
|
||||
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
check(d, word, &crate::Options::default())
|
||||
}
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Russian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '-');
|
||||
let w = w.trim_matches('-');
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(char::is_numeric) || !w.chars().any(char::is_alphabetic) {
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
if w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
|
||||
return true;
|
||||
}
|
||||
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
|
||||
@@ -161,7 +125,7 @@ pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
return true;
|
||||
}
|
||||
if w.contains('-') {
|
||||
return hyphenated(d, w);
|
||||
return hyphenated(d, w, opts);
|
||||
}
|
||||
solid_compound(d, w)
|
||||
}
|
||||
@@ -340,7 +304,7 @@ fn known(d: &AffixDict, w: &str) -> bool {
|
||||
|
||||
/// A hyphenated word that isn't in the dictionary whole: its parts, with the particles,
|
||||
/// prefixes and linking vowels Russian joins with a hyphen.
|
||||
fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
|
||||
let parts: Vec<&str> = w.split('-').collect();
|
||||
if parts.iter().any(|p| p.is_empty()) || parts.len() > 4 {
|
||||
return false;
|
||||
@@ -353,7 +317,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
&& lp.iter().skip(k).all(|p| PARTICLES.contains(p))
|
||||
{
|
||||
let base = parts.get(..k).map(|b| b.join("-")).unwrap_or_default();
|
||||
return known(d, &base) || (base.contains('-') && hyphenated(d, &base));
|
||||
return known(d, &base) || (base.contains('-') && hyphenated(d, &base, opts));
|
||||
}
|
||||
match lp.as_slice() {
|
||||
// `кое-кто`, `кой-какой`.
|
||||
@@ -369,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str) -> bool {
|
||||
let last = parts.len() - 1;
|
||||
parts.iter().enumerate().all(|(i, p)| {
|
||||
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
|
||||
if known(d, p) || (p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
|
||||
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
|
||||
return true;
|
||||
}
|
||||
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
|
||||
@@ -497,4 +461,25 @@ mod tests {
|
||||
crate::add_word("Зкщфрусскоеслово");
|
||||
assert!(is_correct("зкщфрусскоеслово"));
|
||||
}
|
||||
|
||||
/// With Ignore words in UPPERCASE or Ignore words that contain numbers off, Russian checks
|
||||
/// those words against the dictionary instead of accepting them.
|
||||
#[test]
|
||||
fn proofing_options_reach_the_russian_check() {
|
||||
test_dict::install();
|
||||
let lang = crate::ProofLang::Ru;
|
||||
let defaults = crate::Options::default();
|
||||
let check_caps = crate::Options { ignore_uppercase: false, ..defaults };
|
||||
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
|
||||
assert!(crate::is_correct_in_with("ПРЕВЕТ", lang, &defaults), "short capitals are abbreviations by default");
|
||||
assert!(!crate::is_correct_in_with("ПРЕВЕТ", lang, &check_caps));
|
||||
assert!(crate::is_correct_in_with("ПРИВЕТ", lang, &check_caps));
|
||||
assert!(crate::is_correct_in_with("ИИ-консультант", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("ИИ-консультант", lang, &check_caps), "a part in capitals is checked too");
|
||||
assert!(crate::is_correct_in_with("превет2", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("превет2", lang, &check_digits));
|
||||
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
|
||||
}
|
||||
}
|
||||
|
||||
+70
-334
@@ -1,30 +1,14 @@
|
||||
//! Russian spelling suggestions, best first.
|
||||
//!
|
||||
//! Russian Hunspell dictionaries carry no `MAP` or `REP` tables, so the letter groups and the
|
||||
//! common confusions are defined here. Candidates come in rounds of rising cost, each checked
|
||||
//! against the dictionary:
|
||||
//! 1. letter case (`москва` → `Москва`);
|
||||
//! 2. letter slips within a group ([`MAP`]): `е`/`ё` first, then unstressed vowels (`о`/`а`,
|
||||
//! `е`/`и`/`я`), voiced and voiceless pairs (`з`/`с`, `д`/`т`, `б`/`п`…) and `ь`/`ъ`, up to
|
||||
//! three letters changed (`превет` → `привет`, `сдесь` → `здесь`);
|
||||
//! 3. the common confusions of [`REP`] (`жы` → `жи`, `чя` → `ча`, `тся`/`ться`, a dropped or
|
||||
//! doubled consonant, `здраствуй` → `здравствуй`);
|
||||
//! 4. a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped;
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word and share the
|
||||
//! most letter pairs with it, whose forms are ranked by an edit distance in which a slip within
|
||||
//! a letter group costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round.
|
||||
//! Russian spelling suggestions, best first: the shared ranking ([`crate::affix_suggest`]) with
|
||||
//! Russian's own letter groups and confusions, since Russian Hunspell dictionaries carry no `MAP`
|
||||
//! or `REP` tables (`е`/`ё` first, then unstressed vowels, voiced and voiceless pairs, `жы` →
|
||||
//! `жи`, `тся`/`ться`, dropped or doubled consonants; up to three letters changed), and its rule
|
||||
//! for a missing space (`незнаю` → `не знаю`, `вобщем` → `в общем`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::{COST_OTHER_MAP, COST_SPLIT, COST_SPLIT_LONG, Split, Tuning};
|
||||
|
||||
/// Letters confused with each other: a slip inside a group is cheaper than another typo.
|
||||
const MAP: &[&str] = &["её", "оа", "еия", "зс", "дт", "бп", "вф", "гк", "жш", "ьъ", "еэ", "ыи"];
|
||||
|
||||
/// Common confusions: a written sequence and what was meant.
|
||||
const REP: &[(&str, &str)] = &[
|
||||
("жы", "жи"),
|
||||
@@ -67,7 +51,6 @@ const REP: &[(&str, &str)] = &[
|
||||
("ц", "тс"),
|
||||
("тс", "ц"),
|
||||
];
|
||||
|
||||
/// Short words often run into the next one (`не знаю`, `в общем`, `на верху`, `потому что`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
"не",
|
||||
@@ -123,158 +106,42 @@ const PREPOSITIONS: &[&str] = &[
|
||||
const TRAILING_WORDS: &[&str] = &["ли", "же", "бы", "ж", "б", "что"];
|
||||
/// The Russian alphabet, when the dictionary has no `TRY` line.
|
||||
const ALPHABET: &str = "оеаинтсрвлкмдпуяыьгзбчйхжшюцщэфъё";
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 3;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
/// `е` written for `ё`, or the other way round, is barely a slip.
|
||||
const COST_YO: u32 = 3;
|
||||
const COST_MAP: u32 = 7;
|
||||
const COST_REP: u32 = 9;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
/// A one- or two-letter preposition run into the next word (`вдоме` → `в доме`).
|
||||
const COST_SPLIT_PREPOSITION: u32 = 17;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
/// What Russian adds to its dictionary's tables.
|
||||
const TUNING: Tuning =
|
||||
Tuning { split: Split::Custom(split), alphabet: ALPHABET, map: MAP, rep: REP, rep_cost: 9, max_map_changes: 3, map_cost, ..Tuning::DEFAULT };
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance.
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The other members of each [`MAP`] group holding `c`.
|
||||
fn alternatives(c: char) -> Vec<char> {
|
||||
let mut out: Vec<char> = MAP.iter().filter(|g| g.contains(c)).flat_map(|g| g.chars()).filter(|&x| x != c).collect();
|
||||
out.sort_unstable();
|
||||
out.dedup();
|
||||
out
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its group relative `to`: `е`/`ё` is the cheapest slip
|
||||
/// (`ё` is optional in ordinary writing).
|
||||
/// The cost of changing `from` to `to` within a letter group: `е`/`ё` cheapest.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_MAP }
|
||||
if matches!((from, to), ('е', 'ё') | ('ё', 'е')) { COST_YO } else { COST_OTHER_MAP }
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a group relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
/// A missing space: cheap after a short leading word or before a short trailing one, dearer after
|
||||
/// a short preposition or between two longer words, never leaving a stray short half.
|
||||
fn split(a: &str, b: &str) -> Option<u32> {
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
|
||||
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
|
||||
let preposition = PREPOSITIONS.contains(&a);
|
||||
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
|
||||
return None;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
|
||||
Some(COST_SPLIT_PREPOSITION)
|
||||
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
Some(COST_SPLIT)
|
||||
} else if long(a) && long(b) {
|
||||
Some(COST_SPLIT_LONG)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's group representative (the first of its first group), lower case: `ё` → `е`,
|
||||
/// `з` → `з`, `с` → `з`.
|
||||
fn fold(c: char) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
MAP.iter().find(|g| g.contains(l)).and_then(|g| g.chars().next()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a slip within a letter group costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if fold(ai) == fold(bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// Spelling suggestions for a Russian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
suggest_in(&super::dictionary(), word, max)
|
||||
@@ -282,171 +149,7 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Slips within a letter group.
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in REP {
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
// A preposition takes a noun, not a verb (`с генерировать` isn't a fix for
|
||||
// `сгенерировать`); after one or two letters a split is likelier wrong than a typo.
|
||||
let preposition = PREPOSITIONS.contains(&a);
|
||||
if preposition && ["ть", "ться", "ти", "чь"].iter().any(|e| b.ends_with(e)) {
|
||||
continue;
|
||||
}
|
||||
let cost = if preposition && a.chars().count() <= 2 && LEADING_WORDS.contains(&a) && long(b) {
|
||||
COST_SPLIT_PREPOSITION
|
||||
} else if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = ALPHABET.chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
found.sort_by_key(|(_, k)| *k);
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a group relative of it,
|
||||
/// either case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(fold).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
crate::affix_suggest::suggest_in(d, word, max, &TUNING)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -458,6 +161,36 @@ mod tests {
|
||||
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
||||
}
|
||||
|
||||
/// Russian's own costs: `е`/`ё` is the cheapest slip, so a `ё` and one more group slip
|
||||
/// (made-up `нёснаю`) beat a missing space; a REP confusion beats a `ё` and a group slip;
|
||||
/// a preposition is never split off a verb, and after a one-letter one a typo is likelier.
|
||||
#[test]
|
||||
fn russian_costs_rank_its_slips() {
|
||||
let dict = |words: &[&str]| AffixDict::from_hunspell("", &format!("{}\n{}\n", words.len(), words.join("\n")), &[]).unwrap();
|
||||
assert_eq!(suggest_in(&dict(&["не", "знаю", "нёснаю"]), "незнаю", 1), ["нёснаю"]);
|
||||
assert_eq!(suggest_in(&dict(&["еча", "ёчи"]), "ечя", 1), ["еча"]);
|
||||
assert!(!suggest_in(&dict(&["с", "генерировать"]), "сгенерировать", 8).contains(&"с генерировать".into()));
|
||||
let s = suggest_in(&dict(&["в", "доме"]), "вдоме", 4);
|
||||
assert_eq!(s.first().map(String::as_str), Some("доме"), "{s:?}");
|
||||
assert!(s.contains(&"в доме".into()), "{s:?}");
|
||||
}
|
||||
|
||||
/// A word the dictionary marks NOSUGGEST (a vulgarism, a rare form) is correct but never
|
||||
/// offered, in any round or letter case. Russian's MAP and REP tables are its own, so the
|
||||
/// shared Latin-script check doesn't reach it.
|
||||
#[test]
|
||||
fn nosuggest_words_are_never_offered() {
|
||||
let offered = |aff: &str, input: &str| {
|
||||
let d = AffixDict::from_hunspell(aff, "2\nпривет/N\nпредмет/N\n", &[]).unwrap();
|
||||
suggest_in(&d, input, 16)
|
||||
};
|
||||
assert!(offered("SET UTF-8\n", "превет").contains(&"привет".into()), "offered without the flag declared");
|
||||
for input in ["превет", "ПРЕВЕТ", "предмот", "приветт", "привте", "привет"] {
|
||||
let s = offered("SET UTF-8\nNOSUGGEST N\n", input);
|
||||
assert!(!s.iter().any(|w| w.to_lowercase().contains("привет") || w.to_lowercase().contains("предмет")), "{input}: {s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn russian_misspellings_get_the_right_suggestion() {
|
||||
crate::ru::test_dict::install();
|
||||
@@ -499,12 +232,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn distance_weighs_letter_groups_half() {
|
||||
use crate::affix_suggest::{fold, weighted_distance};
|
||||
let map: Vec<Vec<char>> = MAP.iter().map(|g| g.chars().collect()).collect();
|
||||
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
||||
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
||||
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), 9), 1);
|
||||
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), 9), 1);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), 9), 2);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("домами"), 9), 6);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("книга"), 2), 3);
|
||||
assert_eq!(weighted_distance(&c("сдесь"), &c("здесь"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("ёлка"), &c("елка"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("дмо"), &related, 9), 2);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("домами"), &related, 9), 6);
|
||||
assert_eq!(weighted_distance(&c("дом"), &c("книга"), &related, 2), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
//! The installed Hunspell dictionary of a proofing language, in one shape for every language:
|
||||
//! found on first use, replaced with [`DictSlot::install`] and forgotten with [`DictSlot::reload`].
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// A dictionary a language reads: the affix engine's ([`AffixDict`]) or Turkish's own.
|
||||
pub(crate) trait Dictionary: Default + Send + Sync {
|
||||
/// The installed `.aff` and `.dic` files, read.
|
||||
fn load(aff: &Path, dic: &Path) -> Result<Self, String>;
|
||||
/// Hunspell `.aff` and `.dic` text (decoded to UTF-8), parsed.
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String>;
|
||||
/// No words: the language has no spelling marks.
|
||||
fn is_empty(&self) -> bool;
|
||||
}
|
||||
|
||||
impl Dictionary for AffixDict {
|
||||
fn load(aff: &Path, dic: &Path) -> Result<Self, String> {
|
||||
crate::hunspell::load(aff, dic)
|
||||
}
|
||||
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
|
||||
AffixDict::from_hunspell(aff, dic, &[])
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
AffixDict::is_empty(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// What every language's slot answers, whatever its dictionary type (the language table in
|
||||
/// lib.rs holds them side by side).
|
||||
pub(crate) trait Slot: Sync {
|
||||
/// See [`DictSlot::name`].
|
||||
fn name(&self) -> &'static str;
|
||||
/// See [`DictSlot::installed_path`].
|
||||
fn installed_path(&self) -> Option<PathBuf>;
|
||||
/// See [`DictSlot::has_dictionary`].
|
||||
fn has_dictionary(&self) -> bool;
|
||||
/// See [`DictSlot::reload`].
|
||||
fn reload(&self);
|
||||
}
|
||||
|
||||
impl<D: Dictionary> Slot for DictSlot<D> {
|
||||
fn name(&self) -> &'static str {
|
||||
DictSlot::name(self)
|
||||
}
|
||||
|
||||
fn installed_path(&self) -> Option<PathBuf> {
|
||||
DictSlot::installed_path(self)
|
||||
}
|
||||
|
||||
fn has_dictionary(&self) -> bool {
|
||||
DictSlot::has_dictionary(self)
|
||||
}
|
||||
|
||||
fn reload(&self) {
|
||||
DictSlot::reload(self);
|
||||
}
|
||||
}
|
||||
|
||||
/// The dictionary in use and whether it came from [`DictSlot::install`] rather than a file on
|
||||
/// disk (it then has no file to report, even if a matching file also happens to exist): one lock,
|
||||
/// so a reload and an install can't interleave their two fields.
|
||||
struct State<D> {
|
||||
dict: Option<Arc<D>>,
|
||||
in_memory: bool,
|
||||
/// Counts reloads, so a load that started before one is not kept.
|
||||
reloads: u64,
|
||||
}
|
||||
|
||||
/// One language's dictionary.
|
||||
pub(crate) struct DictSlot<D> {
|
||||
/// File names the dictionary may have, the usual one first (also the key of its load error).
|
||||
names: &'static [&'static str],
|
||||
/// The language's English name, for the log.
|
||||
english: &'static str,
|
||||
state: RwLock<State<D>>,
|
||||
}
|
||||
|
||||
impl<D: Dictionary> DictSlot<D> {
|
||||
pub(crate) const fn new(names: &'static [&'static str], english: &'static str) -> Self {
|
||||
DictSlot { names, english, state: RwLock::new(State { dict: None, in_memory: false, reloads: 0 }) }
|
||||
}
|
||||
|
||||
fn read(&self) -> RwLockReadGuard<'_, State<D>> {
|
||||
self.state.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn write(&self) -> RwLockWriteGuard<'_, State<D>> {
|
||||
self.state.write().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// The Hunspell name the dictionary is known by (`pl_PL`): the first of its file names.
|
||||
pub(crate) fn name(&self) -> &'static str {
|
||||
self.names.first().copied().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The file names the dictionary may have, the usual one first.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn names(&self) -> &'static [&'static str] {
|
||||
self.names
|
||||
}
|
||||
|
||||
/// The dictionary: the installed one (read on first use), one given with [`Self::install`], or
|
||||
/// an empty one when none is available (the language's spelling is off then).
|
||||
pub(crate) fn get(&self) -> Arc<D> {
|
||||
let reloads = {
|
||||
let s = self.read();
|
||||
if let Some(d) = s.dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
s.reloads
|
||||
};
|
||||
self.keep(reloads, Arc::new(self.load_installed()))
|
||||
}
|
||||
|
||||
/// Keep a dictionary loaded after `reloads` reloads. If the folder was reloaded since, the
|
||||
/// load may have read the old folder: it serves this one check and the next one loads again.
|
||||
fn keep(&self, reloads: u64, loaded: Arc<D>) -> Arc<D> {
|
||||
let mut s = self.write();
|
||||
if s.reloads != reloads {
|
||||
return s.dict.clone().unwrap_or(loaded);
|
||||
}
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
s.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed(&self) -> D {
|
||||
let name = self.name();
|
||||
let Some((aff, dic)) = crate::hunspell::find(self.names) else {
|
||||
crate::hunspell::clear_load_error(name);
|
||||
log::info!("{} spelling: no Hunspell {name} dictionary installed", self.english);
|
||||
return D::default();
|
||||
};
|
||||
crate::hunspell::note_load(name, &dic, D::load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("{} spelling: {e}", self.english);
|
||||
D::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), for
|
||||
/// example one the user picked. On error the current dictionary stays.
|
||||
pub(crate) fn install(&self, aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = D::parse(aff, dic)?;
|
||||
let mut slot = self.write();
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub(crate) fn reload(&self) {
|
||||
let mut slot = self.write();
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
slot.reloads = slot.reloads.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed dictionary, searching the dictionaries folder and the
|
||||
/// system locations now (File › Options › Proofing shows it); `None` when there is none, or
|
||||
/// when the dictionary in use was given with [`Self::install`] and so has no file.
|
||||
pub(crate) fn installed_path(&self) -> Option<PathBuf> {
|
||||
if self.read().in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(self.names).map(|(_, dic)| dic)
|
||||
}
|
||||
|
||||
/// Is a dictionary available (spelling is checked)?
|
||||
pub(crate) fn has_dictionary(&self) -> bool {
|
||||
!self.get().is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
static SLOT: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_TEST"], "Test");
|
||||
|
||||
#[test]
|
||||
fn an_installed_dictionary_has_no_file_until_reloaded() {
|
||||
assert!(!SLOT.has_dictionary(), "nothing installed under this name");
|
||||
assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
assert!(!SLOT.has_dictionary(), "a refused install keeps the current one");
|
||||
SLOT.install("", "1\nkot\n").unwrap();
|
||||
assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));
|
||||
assert_eq!(SLOT.installed_path(), None);
|
||||
SLOT.reload();
|
||||
assert!(!SLOT.has_dictionary(), "reloaded from disk, where there is none");
|
||||
assert_eq!(SLOT.name(), "zz_SLOT_TEST");
|
||||
}
|
||||
|
||||
/// A load that started before a dictionaries-folder change is not kept after its reload.
|
||||
#[test]
|
||||
fn a_load_from_before_a_reload_is_not_kept() {
|
||||
let slot: DictSlot<AffixDict> = DictSlot::new(&["zz_SLOT_RELOAD_TEST"], "Test");
|
||||
let before = slot.read().reloads;
|
||||
slot.reload();
|
||||
let _ = slot.keep(before, Arc::new(AffixDict::default()));
|
||||
assert!(slot.read().dict.is_none(), "the next check loads from the new folder");
|
||||
let now = slot.read().reloads;
|
||||
let _ = slot.keep(now, Arc::new(AffixDict::default()));
|
||||
assert!(slot.read().dict.is_some(), "a load since the reload is kept");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! The start of every Latin-script dictionary language's spelling check (Czech, Croatian,
|
||||
//! Indonesian, Italian, Polish, Portuguese, Spanish): the words that need no lookup, under the
|
||||
//! File › Options › Proofing choices. Each language goes on from there with its own rules.
|
||||
|
||||
use crate::Options;
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one these dictionaries cover)?
|
||||
fn latin(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// `word` trimmed of quotes and apostrophes, when `d` must say whether it is right; `None` when it
|
||||
/// is right without a lookup: there is no dictionary (no spelling marks), the word is a single
|
||||
/// letter or has no letters, it has digits or is a short word in capitals (an acronym) and the
|
||||
/// options skip those, or it is in another script.
|
||||
pub(crate) fn word_to_check<'a>(d: &AffixDict, word: &'a str, opts: &Options) -> Option<&'a str> {
|
||||
if d.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return None;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return None;
|
||||
}
|
||||
w.chars().any(latin).then_some(w)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn words_needing_no_lookup() {
|
||||
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
|
||||
let defaults = Options::default();
|
||||
assert_eq!(word_to_check(&AffixDict::default(), "kot", &defaults), None, "no dictionary");
|
||||
for w in ["", "a", "2026", "PKO", "B2B", "日本語", "--"] {
|
||||
assert_eq!(word_to_check(&d, w, &defaults), None, "{w}");
|
||||
}
|
||||
assert_eq!(word_to_check(&d, "„kot”", &defaults), Some("kot"));
|
||||
assert_eq!(word_to_check(&d, "'kot'", &defaults), Some("kot"));
|
||||
let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults };
|
||||
assert_eq!(word_to_check(&d, "PKO", &strict), Some("PKO"));
|
||||
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
|
||||
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,20 @@ pub struct TrDict {
|
||||
max_suffix_len: usize,
|
||||
}
|
||||
|
||||
impl crate::slot::Dictionary for TrDict {
|
||||
fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
|
||||
super::load(aff, dic)
|
||||
}
|
||||
|
||||
fn parse(aff: &str, dic: &str) -> Result<Self, String> {
|
||||
TrDict::from_hunspell(aff, dic)
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
TrDict::is_empty(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl TrDict {
|
||||
/// Returns true if empty (no dictionary loaded).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
|
||||
+43
-46
@@ -15,39 +15,26 @@ pub mod dict;
|
||||
pub mod grammar;
|
||||
pub mod hyphen;
|
||||
pub mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::slot::DictSlot;
|
||||
|
||||
pub use dict::TrDict;
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<TrDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of a Turkish Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"];
|
||||
|
||||
/// The Turkish dictionary, found on first use (see [`DictSlot`]).
|
||||
pub(crate) static DICT: DictSlot<TrDict> = DictSlot::new(NAMES, "Turkish");
|
||||
|
||||
/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available.
|
||||
pub fn dictionary() -> Arc<TrDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> TrDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
crate::hunspell::clear_load_error(NAMES[0]);
|
||||
log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed");
|
||||
return TrDict::default();
|
||||
};
|
||||
crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| {
|
||||
log::warn!("Turkish spelling: {e}");
|
||||
TrDict::default()
|
||||
})
|
||||
DICT.get()
|
||||
}
|
||||
|
||||
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
|
||||
@@ -57,28 +44,15 @@ pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDic
|
||||
TrDict::from_hunspell(&aff_text, &dic_text)
|
||||
}
|
||||
|
||||
/// Use this Turkish dictionary from now on.
|
||||
/// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
|
||||
/// current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = TrDict::from_hunspell(aff, dic)?;
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reload the dictionary on next check.
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(std::sync::PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the Hunspell `tr_TR` dictionary found in the dictionaries folder or the
|
||||
/// system locations now (File › Options › Proofing shows it, #1063); `None` when there is none.
|
||||
/// A dictionary given with [`install_dictionary`] has no file and isn't reported.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
DICT.install(aff, dic)
|
||||
}
|
||||
|
||||
/// Is a Turkish dictionary available?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
DICT.has_dictionary()
|
||||
}
|
||||
|
||||
/// Is `word` spelled correctly in Turkish?
|
||||
@@ -86,15 +60,26 @@ pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] with the File › Options › Proofing choices: with Ignore words that contain
|
||||
/// numbers off, words with digits are checked against the dictionary too.
|
||||
pub fn is_correct_with_options(word: &str, opts: &crate::Options) -> bool {
|
||||
check(&dictionary(), word, opts)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary.
|
||||
pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
|
||||
check(d, word, &crate::Options::default())
|
||||
}
|
||||
|
||||
/// [`is_correct_with_options`] against a given dictionary.
|
||||
fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // No Turkish dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
// Check user dictionary first
|
||||
@@ -119,7 +104,7 @@ pub fn is_correct_with(d: &TrDict, word: &str) -> bool {
|
||||
}
|
||||
|
||||
// Compound words separated by hyphens (e.g., `afyon-karahisar`)
|
||||
if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) {
|
||||
if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -145,11 +130,8 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn installed_dictionary_checks_words_if_present() {
|
||||
// Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check.
|
||||
if !has_dictionary() {
|
||||
return;
|
||||
}
|
||||
fn installed_dictionary_checks_words() {
|
||||
test_dict::install();
|
||||
assert!(is_correct("kitap"));
|
||||
assert!(is_correct("kitaplar"));
|
||||
assert!(is_correct("evde"));
|
||||
@@ -164,4 +146,19 @@ mod tests {
|
||||
crate::add_word("benimbilmedigimkelime");
|
||||
assert!(is_correct("benimbilmedigimkelime"));
|
||||
}
|
||||
|
||||
/// With Ignore words that contain numbers off, Turkish checks words with digits against the
|
||||
/// dictionary instead of accepting them, hyphen parts included.
|
||||
#[test]
|
||||
fn proofing_options_reach_the_turkish_check() {
|
||||
test_dict::install();
|
||||
let lang = crate::ProofLang::Tr;
|
||||
let defaults = crate::Options::default();
|
||||
let check_digits = crate::Options { ignore_numbers: false, ..defaults };
|
||||
assert!(crate::is_correct_in_with("kitapp2", lang, &defaults));
|
||||
assert!(!crate::is_correct_in_with("kitapp2", lang, &check_digits));
|
||||
assert!(!crate::is_correct_in_with("ev-kitapp2", lang, &check_digits));
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,10 +318,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn typo_transposition_is_suggested() {
|
||||
// Suggestions come from the installed Turkish dictionary; without one there are none.
|
||||
if !super::super::has_dictionary() {
|
||||
return;
|
||||
}
|
||||
crate::tr::test_dict::install();
|
||||
let sugs = suggest("sorna", 3);
|
||||
assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}");
|
||||
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
//! A tiny synthetic Turkish Hunspell dictionary for tests (written for these tests; not taken from
|
||||
//! any real dictionary). Real dictionaries are installed on the system, never bundled.
|
||||
|
||||
/// No affix rules: the forms the tests look up are listed whole.
|
||||
pub const AFF: &str = "SET UTF-8\n";
|
||||
|
||||
/// The words the Turkish tests expect to be known.
|
||||
pub const DIC: &str = "7
|
||||
kitap
|
||||
kitaplar
|
||||
ev
|
||||
evde
|
||||
olacaklardan
|
||||
sonra
|
||||
geldin
|
||||
";
|
||||
|
||||
pub fn install() {
|
||||
if let Err(e) = super::install_dictionary(AFF, DIC) {
|
||||
panic!("test dictionary: {e}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user