From 244907cbf46bb971c323f22e55afe01f3e8a7abf Mon Sep 17 00:00:00 2001 From: Skaywalker <115165402+Skaywalker@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:02:51 +0200 Subject: [PATCH] proof: read count comments, phrase entries and undefined aliases as Hunspell does (storytold/wordcraft#735) Checked against the Hunspell 1.7 command line: - The first AF and ICONV line is the count even with a trailing comment (AF 2 # aliases), so alias numbers no longer shift and ICONV 1 # no longer turns 1 into #. - A .dic entry ends where Hunspell ends it (dic_entry_end), so ad hoc/B is one entry and ad alone is no word. - An undefined alias in a .dic entry reads as no flags: the stem loads without affixes instead of failing the whole dictionary. - IGNORE characters are removed from the appended text only; a stripped text that holds one never matches a stem, as in Hunspell. - FLAG UTF-8 and SET are found anywhere in the .aff, not only in the first 8 KB. Also: normalize borrows the input when no conversion or ignored character applies, the UTF-8 .aff reader counts fields as it goes and builds its error context only on error, and the single-byte flag reader is decode_latin1. --- crates/proof/src/affix.rs | 81 ++++++++++++++++++++++++++++++++---- crates/proof/src/hunspell.rs | 67 +++++++++++++++++------------ 2 files changed, 114 insertions(+), 34 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 4f15bd56..2d3a7717 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -266,7 +266,7 @@ pub struct AffixDict { keepcase: Mask, /// `ICONV`: input conversions applied before a lookup, `(from, to)`. iconv: Vec<(String, String)>, - /// `IGNORE`: characters removed from input words (and from the stems and affixes). + /// `IGNORE`: characters removed from input words (and from the stems and appended texts). ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, @@ -361,6 +361,12 @@ impl Rules { .ok_or_else(|| format!("flag alias {written} is not defined")) } + /// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the + /// stem is still a word, without affixes. + fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str { + self.flags(written).unwrap_or("") + } + /// `w` without the `IGNORE` characters. fn strip_ignored(&self, w: &str) -> String { if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } @@ -437,15 +443,17 @@ fn parse_aff(aff: &str) -> Result { // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // trailing comment (`AF 1559 # aliases`). "AF" => { - if !af_header && f.len() == 2 && at(1).parse::().is_ok() { + if !af_header && at(1).parse::().is_ok() { af_header = true; } else { r.aliases.push(at(1).to_string()); } } "ICONV" => { - if !iconv_header && f.len() == 2 && at(1).parse::().is_ok() { + if !iconv_header && at(1).parse::().is_ok() { iconv_header = true; } else if f.len() >= 3 { r.iconv.push((at(1).to_string(), at(2).to_string())); @@ -512,10 +520,11 @@ fn parse_aff(aff: &str) -> Result { } } // Aliases may be declared after the rules that use them; resolve once all are read. + // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that + // holds one never matches a stem (stems lose them too), so the rule never applies. let resolve = |rules: &mut Vec, r: &Rules| -> Result<(), String> { for rule in rules.iter_mut() { rule.4 = r.flags(&rule.4)?.to_string(); - rule.2 = r.strip_ignored(&rule.2); rule.3 = r.strip_ignored(&rule.3); } Ok(()) @@ -538,7 +547,12 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { - let Some(tok) = split_fields(line).next() else { continue }; + // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. + let line = line.trim_start_matches([' ', '\t']); + let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); + if tok.is_empty() { + continue; + } if n == 0 && tok.parse::().is_ok() { continue; } @@ -565,7 +579,7 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } - let flags = rules.flags(&flags).map_err(|e| format!(".dic line {}: {e}", n + 1))?.to_string(); + let flags = rules.dic_flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -814,7 +828,8 @@ impl AffixDict { /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { - if self.iconv.is_empty() && self.ignore.is_empty() { + let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str())); + if !w.contains(&*self.ignore) && !converts(w) { return std::borrow::Cow::Borrowed(w); } let mut out = String::with_capacity(w.len()); @@ -1577,7 +1592,10 @@ pub(crate) mod tests { let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); assert!(!d.contains("házban"), "the stem has a and b, not c"); - assert!(AffixDict::from_hunspell(aff, "1\nház/3\n", &[]).is_err(), "alias 3 is not defined"); + // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes. + let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap(); + assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined"); + assert!(undefined.check("kertok"), "the other entries keep their aliases"); let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); } @@ -1594,6 +1612,53 @@ pub(crate) mod tests { assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); } + #[test] + fn count_lines_may_carry_a_comment() { + // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` + // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`. + let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n"; + let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap(); + assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`"); + assert!(d.check("a1") && !d.check("a#")); + // The UTF-8 reader keeps such a line as it is, so both readers agree on it. + let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap(); + assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}"); + assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok")); + } + + #[test] + fn dic_entries_may_hold_spaces() { + // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. + let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n kot/B 12\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads")); + assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + } + + #[test] + fn ignored_characters_leave_the_stripped_text_alone() { + // Hunspell removes `IGNORE` characters from the appended text, not from the stripped + // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong, + // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line). + let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap(); + assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac")); + let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap(); + assert!(add.check("ac") && add.check("acx")); + for d in [&strip, &add] { + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.check("ac"), d.check("ac")); + } + } + + #[test] + fn normalizing_borrows_when_nothing_changes() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); + assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine"))); + assert_eq!(d.normalize("(fi)ne"), "fine"); + assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot"))); + } + #[test] fn keepcase_stems_match_only_as_written() { let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 934520be..7152f7a4 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -224,15 +224,16 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per /// byte); the words, affixes and conditions around them are UTF-8. pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); + // The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic` + // flags are read, as in Hunspell. let directive = |name: &str| { - head.lines().find_map(|l| { - let l = l.trim_start_matches('\u{feff}'); + aff.split(|&b| b == b'\n').find_map(|l| { + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8. - l.strip_prefix(name) - .filter(|rest| rest.starts_with([' ', '\t'])) - .and_then(|rest| rest.split([' ', '\t']).find(|f| !f.is_empty())) - .map(str::to_ascii_uppercase) + l.strip_prefix(name.as_bytes()) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())) + .map(|f| String::from_utf8_lossy(f).to_ascii_uppercase()) }) }; let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); @@ -284,13 +285,9 @@ const FLAG_DIRECTIVES: &[&str] = &[ /// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; -/// One byte per char: a flag field read as single-byte flags. -fn bytes_as_flags(b: &[u8]) -> String { - b.iter().map(|&x| char::from(x)).collect() -} - -/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). -fn text_and_flags(field: &[u8], what: &str) -> Result { +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what` +/// names the line in an error. +fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result { let mut slash = None; let mut i = 0; while let Some(&b) = field.get(i) { @@ -308,8 +305,8 @@ fn text_and_flags(field: &[u8], what: &str) -> Result { Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), None => (field, &[][..]), }; - let mut out = std::str::from_utf8(text).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(); - out.push_str(&bytes_as_flags(flags)); + let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(); + out.push_str(&decode_latin1(flags)); Ok(out) } @@ -353,7 +350,7 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { .and_then(|&i| parts.get(i)) .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) .unwrap_or_default(); - let what = format!(".aff line {}", n + 1); + let what = || format!(".aff line {}", n + 1); if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { out.push_str(&String::from_utf8_lossy(line)); continue; @@ -364,18 +361,23 @@ fn decode_utf8_aff(aff: &[u8]) -> Result { let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); let first_free = comment_field(&dir, field_text); let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); + // The field index of the next non-separator part. + let mut next_field = 0; for (i, p) in parts.iter().enumerate() { - let field = words.iter().position(|&w| w == i); + let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| { + next_field += 1; + next_field - 1 + }); let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); let decoded = match (dir.as_str(), field) { _ if in_comment => String::from_utf8_lossy(p).into_owned(), - ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => bytes_as_flags(p), - ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?, + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?, // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. - ("PFX" | "SFX", Some(1)) => bytes_as_flags(p.get(..1).unwrap_or_default()), - (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p.get(..1).unwrap_or_default()), - _ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(), + ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()), + _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(), }; out.push_str(&decoded); } @@ -433,13 +435,13 @@ fn decode_utf8_dic(dic: &[u8]) -> Result { out.push_str(s); continue; } - let what = format!(".dic line {}", n + 1); + let what = || format!(".dic line {}", n + 1); // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); // morphological fields after it are UTF-8 text. let end = dic_entry_end(line); - out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?); let rest = line.get(end..).unwrap_or_default(); - out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?); } Ok(out) } @@ -522,6 +524,19 @@ mod tests { assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); } + #[test] + fn flag_utf8_counts_anywhere_in_the_aff() { + // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. + let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec(); + for _ in 0..400 { + aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n"); + } + aff.extend_from_slice(b"FLAG UTF-8\r\n"); + assert!(aff.len() > 8192); + let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + } + #[test] fn a_comment_after_flag_utf8_keeps_unicode_flags() { let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();