From a4edc0dce7f86742ea90dd4b2fa71f667413e87b Mon Sep 17 00:00:00 2001 From: janis luks <173142783+JanisLuks@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:34:42 +0000 Subject: [PATCH] Hunspell: support FULLSTRIP, NEEDAFFIX and CIRCUMFIX The affix engine refused any dictionary that uses these three directives, so a dictionary such as Latvian lv_LV (hunspell-lv, LGPL-2.1+) never loaded and its language showed "(dictionary unreadable)". - FULLSTRIP: an affix may strip, and so replace, a whole stem. - NEEDAFFIX: a stem carrying the flag is not a word on its own, and a rule with it among its continuation classes gives forms that need one more affix (a prefix satisfies it for a suffix and the other way). - CIRCUMFIX: a prefix or suffix carrying the flag is valid only together with one that carries it too; plain affixes never pair with circumfix ones. forms() follows the same rules, so every form it lists is still accepted by the checker, and the compact format keeps the three directives. Tests cover each directive, the forms invariant and the compact round trip. Ported from storytold/wordcraft#5012 (989e942bc) onto craft-proof. Adapted to craft-proof's flag ids (this branch sits on the FLAG num/long port, storytold/wordcraft#5142): NEEDAFFIX and CIRCUMFIX are flag ids read in the FLAG mode, continuation-class tests are set lookups, and NEEDAFFIX joins the lookup exclusions. The ROADMAP.md part and the Latvian language module (c99142ae9) are WordCraft's, not ported. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 206 ++++++++++++++++++++++++++++++-------- 1 file changed, 163 insertions(+), 43 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index b742f995..a3b9b511 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -13,18 +13,19 @@ //! //! Supported: one-character flags (any Unicode character — `FLAG UTF-8`), two-character flags //! (`FLAG long`) and numeric flags (`FLAG num`: comma-separated numbers from 0 to 65,535), up to -//! 65,536 distinct flags in all ([`FlagMode`]), flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and -//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes -//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the -//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` -//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` -//! and `REP`. Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English +//! 65,536 distinct flags in all ([`FlagMode`]), flag aliases (`AF`: a number standing for a set of +//! flags), `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross +//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), +//! `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the stem is a word only in its own letter case), +//! `NEEDAFFIX`, `CIRCUMFIX`, `FULLSTRIP`, input conversion (`ICONV`: ligatures such as `fi` read as +//! `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. +//! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English //! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written //! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a //! compound are stems, without affixes, as in Hunspell. //! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: flag-based -//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…), `NEEDAFFIX` and the other directives that -//! change which words are correct. +//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…) and the other directives that change which +//! words are correct. //! //! Each distinct flag gets a dense `u16` id; a stem's flags and a rule's continuation classes are //! a sorted set of ids, interned once in a flat table, so membership is a binary search in a few @@ -68,7 +69,6 @@ const MAX_DISTINCT_FLAGS: usize = 1 << 16; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ "CHECKSHARPS", - "CIRCUMFIX", "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND", @@ -77,8 +77,6 @@ const UNSUPPORTED: &[&str] = &[ "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG", - "FULLSTRIP", - "NEEDAFFIX", "OCONV", "PSEUDOROOT", ]; @@ -291,6 +289,14 @@ pub struct AffixDict { rep: Vec<(String, String)>, /// The `KEEPCASE` flag: such stems are words only in their own letter case. keepcase: Option, + /// `NEEDAFFIX`: a stem carrying it is not a word on its own, and a rule carrying it in its + /// continuation classes gives forms that need one more affix. + need_affix: Option, + /// `CIRCUMFIX`: a prefix or suffix carrying it in its continuation classes is only valid + /// together with a suffix or prefix that carries it too. + circumfix: Option, + /// `FULLSTRIP`: an affix may strip (and so replace) a whole stem. + full_strip: bool, /// `ICONV`: input conversions applied before a lookup, `(from, to)`. iconv: Vec<(String, String)>, /// `IGNORE`: characters removed from input words (and from the stems and appended texts). @@ -379,6 +385,9 @@ struct Rules { substandard: Option, /// The `KEEPCASE` flag. keepcase: Option, + need_affix: Option, + circumfix: Option, + full_strip: bool, iconv: Vec<(String, String)>, ignore: Vec, /// `COMPOUNDRULE` patterns, as written (`n*1t`); read in the `FLAG` mode by @@ -621,9 +630,10 @@ impl FlagMode { } /// Stem flags that rule a lookup out: `FORBIDDENWORD` always, and per lookup `NOSUGGEST` and -/// `SUBSTANDARD` (suggestions) or `KEEPCASE` (case variants of a word). +/// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on +/// its own). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 4]); +struct Excluded([Option; 5]); impl Excluded { /// Does the flag set `set` carry an excluded flag? @@ -756,6 +766,9 @@ fn parse_aff(aff: &str, english: bool) -> Result { let min = at(1).parse::().map_err(|_| format!("line {}: COMPOUNDMIN needs a number", n + 1))?; r.compound_min = Some(min.clamp(1, MAX_COMPOUND_BYTES)); } + "NEEDAFFIX" => r.need_affix = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), + "CIRCUMFIX" => r.circumfix = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), + "FULLSTRIP" => r.full_strip = true, "FLAG" => r.set_mode(at(1)).map_err(|e| format!("line {}: {e}", n + 1))?, "FORBIDDENWORD" => r.forbidden = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), "NOSUGGEST" => r.no_suggest = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), @@ -939,7 +952,10 @@ impl AffixDict { let mut distinct: FxMap = FxMap::default(); let all = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0); let all = all.chain(pfx_cont.iter().chain(&sfx_cont).flatten().copied()).chain(stem_keys.iter().copied()); - for k in all.chain(rules.forbidden).chain(rules.no_suggest).chain(rules.substandard).chain(rules.keepcase).chain(rules.only_in_compound) { + // NEEDAFFIX and CIRCUMFIX flags may only ever appear in rules' continuation classes. + let directives = + [rules.forbidden, rules.no_suggest, rules.substandard, rules.keepcase, rules.only_in_compound, rules.need_affix, rules.circumfix]; + for k in all.chain(directives.into_iter().flatten()) { distinct.entry(k).or_insert(0); if distinct.len() > MAX_DISTINCT_FLAGS { return Err(format!("more than {MAX_DISTINCT_FLAGS} distinct flags")); @@ -964,6 +980,9 @@ impl AffixDict { no_suggest: rules.no_suggest.and_then(id_of), substandard: rules.substandard.and_then(id_of), keepcase: rules.keepcase.and_then(id_of), + need_affix: rules.need_affix.and_then(id_of), + circumfix: rules.circumfix.and_then(id_of), + full_strip: rules.full_strip, flag_mode: mode, iconv: rules.iconv, ignore: rules.ignore.into_boxed_slice(), @@ -1254,6 +1273,11 @@ impl AffixDict { None } + /// Do the continuation classes of `r` hold `flag`? + fn cont_holds(&self, r: &Rule, flag: Option) -> bool { + flag.is_some_and(|f| has(self.set(r.cont), f)) + } + /// Can a derived form carry `flag` (does some rule's continuation classes hold it)? fn is_cont_flag(&self, flag: u16) -> bool { self.cont_flag.get(usize::from(flag)).copied().unwrap_or(false) @@ -1280,7 +1304,7 @@ impl AffixDict { // total size of the distinct sets it meets, not rules × set size. let probe = FlagProbe::new(flags); let mut seen: FxMap = FxMap::default(); - let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; let Some(groups) = self.sfx_index.get(add) else { continue }; @@ -1291,7 +1315,7 @@ impl AffixDict { inner.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); - if meets && r.cond_matches(&root, true) { + if meets && !self.cont_holds(r, self.circumfix) && r.cond_matches(&root, true) { inner.push(r.flag); } } @@ -1353,13 +1377,14 @@ impl AffixDict { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags_except(w, excluded).is_some() || self.suffixed(w, None, excluded) { + // A stem that needs an affix is no word on its own. + if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, excluded) { return true; } let mut root = String::with_capacity(w.len() + 8); for r in &self.prefixes { let Some(rest) = w.strip_prefix(&*r.add) else { continue }; - if rest.is_empty() { + if rest.is_empty() && !self.full_strip { continue; } root.clear(); @@ -1368,13 +1393,18 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { - return true; + // A circumfix prefix works only with its suffix, and one that needs another affix + // only with a suffix too. + let circ = self.cont_holds(r, self.circumfix); + if !circ && !self.cont_holds(r, self.need_affix) { + if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { + return true; + } + if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { + return true; + } } - if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { - return true; - } - if r.cross && self.suffixed(&root, Some(r.flag), excluded) { + if r.cross && self.suffixed(&root, Some(r.flag), circ, excluded) { return true; } } @@ -1417,13 +1447,16 @@ impl AffixDict { } /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must - /// carry that flag too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: Option, excluded: Excluded) -> bool { + /// carry that flag too and the suffix must allow cross products. `circ` says whether that + /// prefix is a circumfix one: the suffix must be one too, and otherwise must not be. + fn suffixed(&self, w: &str, need: Option, circ: bool, excluded: Excluded) -> bool { + // Fits the suffix rule `r`: circumfix alike, and not in need of another affix unless a prefix is there. + let fits = |r: &Rule| self.cont_holds(r, self.circumfix) == circ && (need.is_some() || !self.cont_holds(r, self.need_affix)); let mut root = String::with_capacity(w.len() + 8); let mut flags: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. - let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; let Some(groups) = self.sfx_index.get(add) else { continue }; @@ -1436,7 +1469,7 @@ impl AffixDict { // when a cross product is being checked). if let Some(f) = self.stem_flags_except(&root, excluded) && need.is_none_or(|n| has(f, n)) - && rules().any(|r| has(f, r.flag) && (need.is_none() || r.cross) && r.cond_matches(&root, true)) + && rules().any(|r| has(f, r.flag) && (need.is_none() || r.cross) && fits(r) && r.cond_matches(&root, true)) { return true; } @@ -1447,7 +1480,7 @@ impl AffixDict { if need.is_none() && self.has_cont { flags.clear(); // Only a flag some continuation class holds can come from another rule. - flags.extend(rules().filter(|r| self.is_cont_flag(r.flag) && r.cond_matches(&root, true)).map(|r| r.flag)); + flags.extend(rules().filter(|r| self.is_cont_flag(r.flag) && fits(r) && r.cond_matches(&root, true)).map(|r| r.flag)); flags.sort_unstable(); flags.dedup(); if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { @@ -1500,8 +1533,12 @@ impl AffixDict { /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; - let mut out = vec![stem.to_string()]; - let mut cross_sfx = Vec::new(); + let mut out = Vec::new(); + if !self.need_affix.is_some_and(|n| has(f, n)) { + out.push(stem.to_string()); + } + // Suffix forms a prefix may be added to, with whether they are circumfix ones. + let mut cross_sfx: Vec<(String, bool)> = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once. let mut chain: Vec<(String, u32)> = Vec::new(); @@ -1512,14 +1549,21 @@ impl AffixDict { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; + if base.is_empty() && !self.full_strip { + continue; + } let form = format!("{base}{}", r.add); + let circ = self.cont_holds(r, self.circumfix); if r.cross { - cross_sfx.push(form.clone()); + cross_sfx.push((form.clone(), circ)); } if r.cont != 0 && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } - out.push(form); + // A circumfix suffix or one that needs another affix is no word on its own. + if !circ && !self.cont_holds(r, self.need_affix) { + out.push(form); + } if out.len() >= MAX_FORMS { return out; } @@ -1557,9 +1601,15 @@ impl AffixDict { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; - out.push(format!("{}{rest}", r.add)); + if rest.is_empty() && !self.full_strip { + continue; + } + let circ = self.cont_holds(r, self.circumfix); + if !circ && !self.cont_holds(r, self.need_affix) { + out.push(format!("{}{rest}", r.add)); + } if r.cross { - for s in &cross_sfx { + for (s, _) in cross_sfx.iter().filter(|(_, c)| *c == circ) { work += 1; if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { return out; @@ -1654,9 +1704,17 @@ impl AffixDict { self.flags.iter().any(|&id| self.set(id).contains(&flag)) || self.prefixes.iter().chain(&self.suffixes).any(|r| r.flag == flag || self.set(r.cont).contains(&flag)) }; - for (directive, flag) in - [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] - { + if self.full_strip { + out.push_str("FULLSTRIP\n"); + } + for (directive, flag) in [ + ("FORBIDDENWORD", self.forbidden), + ("NOSUGGEST", self.no_suggest), + ("SUBSTANDARD", self.substandard), + ("KEEPCASE", self.keepcase), + ("NEEDAFFIX", self.need_affix), + ("CIRCUMFIX", self.circumfix), + ] { if let Some(flag) = flag.filter(|&f| self.flag_mode != FlagMode::Char || used(f)) { out.push_str(directive); out.push(' '); @@ -1754,14 +1812,17 @@ impl AffixDict { "FLAG" => rules.set_mode(at(1)).map_err(|e| format!("affix dictionary: {e}"))?, "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE" | "ONLYINCOMPOUND") => { + "FULLSTRIP" => rules.full_strip = true, + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE" | "ONLYINCOMPOUND" | "NEEDAFFIX" | "CIRCUMFIX") => { let c = rules.one_flag(at(1)).map_err(|e| format!("affix dictionary: bad {directive} in `{line}`: {e}"))?; match directive { "FORBIDDENWORD" => rules.forbidden = Some(c), "NOSUGGEST" => rules.no_suggest = Some(c), "SUBSTANDARD" => rules.substandard = Some(c), + "KEEPCASE" => rules.keepcase = Some(c), "ONLYINCOMPOUND" => rules.only_in_compound = Some(c), - _ => rules.keepcase = Some(c), + "NEEDAFFIX" => rules.need_affix = Some(c), + _ => rules.circumfix = Some(c), } } "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), @@ -2480,9 +2541,7 @@ pub(crate) mod tests { assert!(d.check("1st") && d.check("11th") && d.check("111th") && !d.check("11st"), "ordinals by COMPOUNDRULE"); assert!(!d.check("housed") && !d.check("hous")); // The directives that change which words are right are still refused. - for dir in ["NEEDAFFIX", "CIRCUMFIX", "PSEUDOROOT", "FULLSTRIP"] { - assert!(AffixDict::from_hunspell_english(&format!("{dir} X\n"), dic, &[]).is_err(), "{dir}"); - } + assert!(AffixDict::from_hunspell_english("PSEUDOROOT X\n", dic, &[]).is_err(), "PSEUDOROOT"); } /// The English path reads `ONLYINCOMPOUND` in the dictionary's FLAG mode too. @@ -2952,4 +3011,65 @@ x/12 } } } + /// `FULLSTRIP` lets an affix replace a whole stem; without it a stem must outlive its strip. + #[test] + fn fullstrip_allows_an_affix_to_replace_the_whole_stem() { + let aff = "SET UTF-8\nSFX A Y 1\nSFX A ab x .\nPFX B Y 1\nPFX B ab y .\n"; + let dic = "1\nab/AB\n"; + let without = AffixDict::from_hunspell(aff, dic, &[]).unwrap(); + for w in ["ab", "x", "y"] { + assert_eq!(without.contains(w), w == "ab", "{w} without FULLSTRIP"); + } + let with = AffixDict::from_hunspell(&format!("FULLSTRIP\n{aff}"), dic, &[]).unwrap(); + for w in ["ab", "x", "y"] { + assert!(with.contains(w), "{w} with FULLSTRIP"); + } + assert!(with.forms("ab").iter().all(|f| with.contains(f))); + assert!(!without.forms("ab").iter().any(|f| f == "x" || f == "y")); + let back = AffixDict::from_bytes(&with.to_bytes().unwrap()).unwrap(); + assert!(back.contains("x") && back.contains("y")); + } + + /// `NEEDAFFIX`: a stem or a suffix form that needs another affix is not a word without one. + #[test] + fn needaffix_stems_and_forms_need_one_more_affix() { + let aff = "SET UTF-8\nNEEDAFFIX N\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x/AN .\nPFX P Y 1\nPFX P 0 re .\n"; + let d = AffixDict::from_hunspell(aff, "3\nfoo/NA\nbar/AB\nbaz/NP\n", &[]).unwrap(); + for w in ["foos", "bar", "bars", "barxs", "rebaz"] { + assert!(d.contains(w), "{w}"); + } + assert!(!d.contains("foo"), "a stem that needs an affix"); + assert!(!d.contains("baz"), "a stem that needs an affix"); + // `barx` carries NEEDAFFIX through its continuation class: it needs `s` after it. + assert!(!d.contains("barx"), "a form that needs another affix"); + for stem in d.stems().map(str::to_string).collect::>() { + for f in d.forms(&stem) { + assert!(d.contains(&f), "{stem} → {f}"); + } + } + assert!(!d.forms("foo").iter().any(|f| f == "foo") && d.forms("foo").iter().any(|f| f == "foos")); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + assert!(!back.contains("foo") && back.contains("foos")); + } + + /// `CIRCUMFIX`: a circumfix prefix and suffix only work as a pair. + #[test] + fn circumfix_prefix_and_suffix_only_work_together() { + let aff = "SET UTF-8\nCIRCUMFIX c\nPFX P Y 1\nPFX P 0 ne/c .\nSFX S Y 1\nSFX S 0 y/c .\nSFX T Y 1\nSFX T 0 z .\n"; + let d = AffixDict::from_hunspell(aff, "1\nstrom/PST\n", &[]).unwrap(); + for w in ["strom", "stromz", "nestromy"] { + assert!(d.contains(w), "{w}"); + } + for w in ["stromy", "nestrom", "nestromz"] { + assert!(!d.contains(w), "{w}"); + } + for f in d.forms("strom") { + assert!(d.contains(&f), "strom → {f}"); + } + let mut forms = d.forms("strom"); + forms.sort(); + assert_eq!(forms, ["nestromy", "strom", "stromz"]); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + assert!(back.contains("nestromy") && !back.contains("stromy") && !back.contains("nestrom")); + } }