mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 11:52:26 +00:00
Merge origin/main
main split Czech's suggest_in into one helper per round for the cognitive_complexity limit (storytold/wordcraft#4501). On this branch that code is the shared affix_suggest::suggest_in (36 of 30), so cs/suggest.rs stays the thin wrapper and the shared function gets the same split. The candidates are offered in the same order; a dump of 13,081 calls over every language's test dictionary matches the old engine except one: the typo round counts the lower-cased letters, so "İ" (two letters lower case) now gets "I" in Polish, where the old count stopped one letter short.
This commit is contained in:
VoiceLessQ
@@ -279,104 +279,13 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
add_case(&mut col, word, &input);
|
||||
let map = groups(d, t);
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, &map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, t, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
let rep = d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
|
||||
for (from, to) in rep {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space; both halves must be dictionary words.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
if let Some(cost) = split_cost(&t.split, a, b) {
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
}
|
||||
// 5. Keyboard neighbours from the dictionary's KEY rows.
|
||||
for row in d.key().split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [a, b] = pair else { continue };
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let replacement = if c == *a {
|
||||
Some(*b)
|
||||
} else if c == *b {
|
||||
Some(*a)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = replacement;
|
||||
}
|
||||
col.offer(&v.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// One typo, with the TRY letters (the language's when the dictionary has none), bounded and
|
||||
// each once, in their order: a TRY line lists the commonest letters first, so equal-cost
|
||||
// candidates come out in that order.
|
||||
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
|
||||
let mut letters: Vec<char> = Vec::new();
|
||||
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
|
||||
if !letters.contains(&c) {
|
||||
letters.push(c);
|
||||
}
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), (t.mistype_cost)(&chars, i, c));
|
||||
}
|
||||
}
|
||||
}
|
||||
add_map_slips(&mut col, &chars, &map, t);
|
||||
add_confusions(&mut col, &input, t);
|
||||
add_splits(&mut col, &input, &t.split);
|
||||
add_key_neighbours(&mut col, &chars);
|
||||
add_typos(&mut col, &chars, t);
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars, &map);
|
||||
@@ -403,6 +312,123 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 1: letter case.
|
||||
fn add_case(col: &mut Collector, word: &str, input: &str) {
|
||||
if col.d.contains(input) && word != input {
|
||||
col.keep(input.to_string(), COST_CASE);
|
||||
} else if col.d.contains(&capitalize(input)) && word != capitalize(input) {
|
||||
col.keep(capitalize(input), COST_CASE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 2: accents and other MAP slips.
|
||||
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 3: common confusions.
|
||||
fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) {
|
||||
let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied());
|
||||
for (from, to) in rep {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), t.rep_cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 4: a missing space; both halves must be dictionary words.
|
||||
fn add_splits(col: &mut Collector, input: &str, split: &Split) {
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
if let Some(cost) = split_cost(split, a, b) {
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5: keyboard neighbours from the dictionary's KEY rows.
|
||||
fn add_key_neighbours(col: &mut Collector, chars: &[char]) {
|
||||
for row in col.d.key().split('|').take(32) {
|
||||
let keys: Vec<char> = row.chars().take(128).collect();
|
||||
for pair in keys.windows(2) {
|
||||
let [a, b] = pair else { continue };
|
||||
for (i, &c) in chars.iter().enumerate() {
|
||||
let replacement = if c == *a {
|
||||
Some(*b)
|
||||
} else if c == *b {
|
||||
Some(*a)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = replacement;
|
||||
}
|
||||
col.offer(&v.iter().collect::<String>(), COST_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has
|
||||
/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so
|
||||
/// equal-cost candidates come out in that order.
|
||||
fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) {
|
||||
let d = col.d;
|
||||
let n = chars.len();
|
||||
let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() };
|
||||
let mut letters: Vec<char> = Vec::new();
|
||||
for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) {
|
||||
if !letters.contains(&c) {
|
||||
letters.push(c);
|
||||
}
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.to_vec();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.to_vec();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.to_vec();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.to_vec();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), (t.mistype_cost)(chars, i, c));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
|
||||
+57
-63
@@ -276,84 +276,69 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_tags(tags: &[&str], expected: ProofLang) {
|
||||
for tag in tags {
|
||||
assert_eq!(ProofLang::from_tag(Some(tag)), expected, "{tag}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_resolve() {
|
||||
for t in ["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pl, "{t}");
|
||||
}
|
||||
for t in ["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pt, "{t}");
|
||||
}
|
||||
for t in ["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::It, "{t}");
|
||||
}
|
||||
for t in ["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Cs, "{t}");
|
||||
}
|
||||
fn tags_resolve_primary_languages() {
|
||||
assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl);
|
||||
assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"], ProofLang::Pt);
|
||||
assert_tags(&["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"], ProofLang::It);
|
||||
assert_tags(&["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"], ProofLang::Cs);
|
||||
assert_tags(&["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"], ProofLang::Id);
|
||||
assert_eq!(ProofLang::Cs.code(), "cs");
|
||||
assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs));
|
||||
for t in ["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Id, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::Id.code(), "id");
|
||||
assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id));
|
||||
for t in ["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}");
|
||||
}
|
||||
for t in [
|
||||
"en-GB",
|
||||
"EN_gb",
|
||||
"en_GB.UTF-8",
|
||||
"en-AU",
|
||||
"en_AU",
|
||||
"en-NZ",
|
||||
"en-IE",
|
||||
"en-ZA",
|
||||
"en-IN",
|
||||
"en-Latn-GB",
|
||||
"en-UK",
|
||||
"en-150",
|
||||
"eng-GB",
|
||||
"en_GB@euro",
|
||||
] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnGb, "{t}");
|
||||
}
|
||||
for t in ["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnCa, "{t}");
|
||||
}
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
assert!(l.is_english() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}");
|
||||
assert_eq!(l.code(), "en");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_resolve_english_regions() {
|
||||
assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En);
|
||||
assert_tags(
|
||||
&[
|
||||
"en-GB",
|
||||
"EN_gb",
|
||||
"en_GB.UTF-8",
|
||||
"en-AU",
|
||||
"en_AU",
|
||||
"en-NZ",
|
||||
"en-IE",
|
||||
"en-ZA",
|
||||
"en-IN",
|
||||
"en-Latn-GB",
|
||||
"en-UK",
|
||||
"en-150",
|
||||
"eng-GB",
|
||||
"en_GB@euro",
|
||||
],
|
||||
ProofLang::EnGb,
|
||||
);
|
||||
assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa);
|
||||
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}");
|
||||
assert_eq!(language.code(), "en");
|
||||
}
|
||||
assert!(!ProofLang::Pl.is_english() && !ProofLang::Other.is_english());
|
||||
// The region is the first two-letter or three-digit subtag: a script or variant isn't one.
|
||||
assert_eq!(ProofLang::from_tag(Some("en-Brai")), ProofLang::En);
|
||||
assert_eq!(ProofLang::from_tag(Some("en-GB-oxendict")), ProofLang::EnGb);
|
||||
for t in ["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Bg, "{t}");
|
||||
}
|
||||
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
|
||||
}
|
||||
for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_resolve_additional_locales_and_defaults() {
|
||||
assert_tags(&["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"], ProofLang::Bg);
|
||||
assert_tags(&["tr", "tr-TR", "tr-CY", "TR_tr", "tur"], ProofLang::Tr);
|
||||
assert_tags(&["de", "de-DE", "de-CH", "DE_at", "deu", "ger"], ProofLang::De);
|
||||
assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru);
|
||||
assert!(!ProofLang::De.is_proofed());
|
||||
for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
|
||||
assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En);
|
||||
assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En);
|
||||
assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed());
|
||||
// Bulgarian has hyphenation only: no spelling or grammar marks.
|
||||
assert!(!ProofLang::Bg.is_proofed());
|
||||
assert_eq!(ProofLang::Bg.code(), "bg");
|
||||
assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed());
|
||||
assert!(ProofLang::Tr.is_proofed());
|
||||
for t in ["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Ru, "{t}");
|
||||
}
|
||||
assert!(ProofLang::Ru.is_proofed());
|
||||
assert_eq!(ProofLang::Ru.code(), "ru");
|
||||
assert!(ProofLang::PROOFED.contains(&ProofLang::Ru));
|
||||
@@ -365,6 +350,15 @@ mod tests {
|
||||
assert_eq!(ProofLang::Hr.code(), "hr");
|
||||
assert_eq!(ProofLang::Tr.code(), "tr");
|
||||
assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed");
|
||||
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
|
||||
assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En);
|
||||
assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En);
|
||||
assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_reject_unknown_or_partial_language_codes() {
|
||||
assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other);
|
||||
}
|
||||
|
||||
/// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and
|
||||
|
||||
Reference in New Issue
Block a user