pauljd/craft-libs
1
0
Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-08 21:10:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

proof: read count comments, phrase entries and undefined aliases as Hunspell does (storytold/wordcraft#735)

Browse Source 
Checked against the Hunspell 1.7 command line:
- The first AF and ICONV line is the count even with a trailing comment
  (AF 2 # aliases), so alias numbers no longer shift and ICONV 1 # no
  longer turns 1 into #.
- A .dic entry ends where Hunspell ends it (dic_entry_end), so ad hoc/B
  is one entry and ad alone is no word.
- An undefined alias in a .dic entry reads as no flags: the stem loads
  without affixes instead of failing the whole dictionary.
- IGNORE characters are removed from the appended text only; a stripped
  text that holds one never matches a stem, as in Hunspell.
- FLAG UTF-8 and SET are found anywhere in the .aff, not only in the
  first 8 KB.

Also: normalize borrows the input when no conversion or ignored character
applies, the UTF-8 .aff reader counts fields as it goes and builds its
error context only on error, and the single-byte flag reader is
decode_latin1.
This commit is contained in:
Skaywalker
2026-10-27 13:09:33 +01:00
parent 1e2a81bb93
commit 244907cbf4
2 changed files with 114 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/proof/src/affix.rs
+73 -8
View File
@@ -266,7 +266,7 @@ pub struct AffixDict {
keepcase: Mask,
/// `ICONV`: input conversions applied before a lookup, `(from, to)`.
iconv: Vec<(String, String)>,
/// `IGNORE`: characters removed from input words (and from the stems and affixes).
/// `IGNORE`: characters removed from input words (and from the stems and appended texts).
ignore: Box<[char]>,
/// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
comments: Vec<String>,
@@ -361,6 +361,12 @@ impl Rules {
.ok_or_else(|| format!("flag alias {written} is not defined"))
}
/// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the
/// stem is still a word, without affixes.
fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str {
self.flags(written).unwrap_or("")
}
/// `w` without the `IGNORE` characters.
fn strip_ignored(&self, w: &str) -> String {
if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }
@@ -437,15 +443,17 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
// the words it marks are correct.
"AM" | "LEMMA_PRESENT" => {}
"SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?),
// The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
// trailing comment (`AF 1559 # aliases`).
"AF" => {
if !af_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
if !af_header && at(1).parse::<usize>().is_ok() {
af_header = true;
} else {
r.aliases.push(at(1).to_string());
}
}
"ICONV" => {
if !iconv_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
if !iconv_header && at(1).parse::<usize>().is_ok() {
iconv_header = true;
} else if f.len() >= 3 {
r.iconv.push((at(1).to_string(), at(2).to_string()));
@@ -512,10 +520,11 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
}
}
// Aliases may be declared after the rules that use them; resolve once all are read.
// Hunspell removes `IGNORE` characters from the appended text only: a stripped text that
// holds one never matches a stem (stems lose them too), so the rule never applies.
let resolve = |rules: &mut Vec<RawRule>, r: &Rules| -> Result<(), String> {
for rule in rules.iter_mut() {
rule.4 = r.flags(&rule.4)?.to_string();
rule.2 = r.strip_ignored(&rule.2);
rule.3 = r.strip_ignored(&rule.3);
}
Ok(())
@@ -538,7 +547,12 @@ impl AffixDict {
let rules = parse_aff(aff)?;
let mut stems: Vec<(String, String)> = Vec::new();
for (n, line) in dic.lines().enumerate() {
let Some(tok) = split_fields(line).next() else { continue };
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
let line = line.trim_start_matches([' ', '\t']);
let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();
if tok.is_empty() {
continue;
}
if n == 0 && tok.parse::<usize>().is_ok() {
continue;
}
@@ -565,7 +579,7 @@ impl AffixDict {
if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
continue;
}
let flags = rules.flags(&flags).map_err(|e| format!(".dic line {}: {e}", n + 1))?.to_string();
let flags = rules.dic_flags(&flags).to_string();
stems.push((word, flags));
}
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
@@ -814,7 +828,8 @@ impl AffixDict {
/// `w` after the `ICONV` conversions (at each position the longest match wins) and without
/// the `IGNORE` characters.
fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
if self.iconv.is_empty() && self.ignore.is_empty() {
let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str()));
if !w.contains(&*self.ignore) && !converts(w) {
return std::borrow::Cow::Borrowed(w);
}
let mut out = String::with_capacity(w.len());
@@ -1577,7 +1592,10 @@ pub(crate) mod tests {
let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();
assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));
assert!(!d.contains("házban"), "the stem has a and b, not c");
assert!(AffixDict::from_hunspell(aff, "1\nház/3\n", &[]).is_err(), "alias 3 is not defined");
// An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes.
let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap();
assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined");
assert!(undefined.check("kertok"), "the other entries keep their aliases");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");
}
@@ -1594,6 +1612,53 @@ pub(crate) mod tests {
assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
}
#[test]
fn count_lines_may_carry_a_comment() {
// Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`
// is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`.
let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n";
let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap();
assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`");
assert!(d.check("a1") && !d.check("a#"));
// The UTF-8 reader keeps such a line as it is, so both readers agree on it.
let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap();
assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}");
assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok"));
}
#[test]
fn dic_entries_may_hold_spaces() {
// `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.
let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n kot/B 12\n", &[]).unwrap();
assert!(d.contains("ad hoc") && d.contains("ad hocs"));
assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads"));
assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));
}
#[test]
fn ignored_characters_leave_the_stripped_text_alone() {
// Hunspell removes `IGNORE` characters from the appended text, not from the stripped
// one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong,
// while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line).
let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap();
assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac"));
let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap();
assert!(add.check("ac") && add.check("acx"));
for d in [&strip, &add] {
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert_eq!(back.check("ac"), d.check("ac"));
}
}
#[test]
fn normalizing_borrows_when_nothing_changes() {
let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();
assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine")));
assert_eq!(d.normalize("(fi)ne"), "fine");
assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot")));
}
#[test]
fn keepcase_stems_match_only_as_written() {
let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";
crates/proof/src/hunspell.rs
+41 -26
View File
@@ -224,15 +224,16 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
/// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per
/// byte); the words, affixes and conditions around them are UTF-8.
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
// The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic`
// flags are read, as in Hunspell.
let directive = |name: &str| {
head.lines().find_map(|l| {
let l = l.trim_start_matches('\u{feff}');
aff.split(|&b| b == b'\n').find_map(|l| {
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
// Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8.
l.strip_prefix(name)
.filter(|rest| rest.starts_with([' ', '\t']))
.and_then(|rest| rest.split([' ', '\t']).find(|f| !f.is_empty()))
.map(str::to_ascii_uppercase)
l.strip_prefix(name.as_bytes())
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()))
.map(|f| String::from_utf8_lossy(f).to_ascii_uppercase())
})
};
let set = directive("SET").unwrap_or_else(|| "UTF-8".into());
@@ -284,13 +285,9 @@ const FLAG_DIRECTIVES: &[&str] = &[
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// One byte per char: a flag field read as single-byte flags.
fn bytes_as_flags(b: &[u8]) -> String {
b.iter().map(|&x| char::from(x)).collect()
}
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash).
fn text_and_flags(field: &[u8], what: &str) -> Result<String, String> {
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
/// names the line in an error.
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
@@ -308,8 +305,8 @@ fn text_and_flags(field: &[u8], what: &str) -> Result<String, String> {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string();
out.push_str(&bytes_as_flags(flags));
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
out.push_str(&decode_latin1(flags));
Ok(out)
}
@@ -353,7 +350,7 @@ fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = format!(".aff line {}", n + 1);
let what = || format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
@@ -364,18 +361,23 @@ fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
let first_free = comment_field(&dir, field_text);
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
// The field index of the next non-separator part.
let mut next_field = 0;
for (i, p) in parts.iter().enumerate() {
let field = words.iter().position(|&w| w == i);
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
next_field += 1;
next_field - 1
});
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => bytes_as_flags(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?,
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
("PFX" | "SFX", Some(1)) => bytes_as_flags(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(),
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
};
out.push_str(&decoded);
}
@@ -433,13 +435,13 @@ fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
out.push_str(s);
continue;
}
let what = format!(".dic line {}", n + 1);
let what = || format!(".dic line {}", n + 1);
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
// morphological fields after it are UTF-8 text.
let end = dic_entry_end(line);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?);
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
}
Ok(out)
}
@@ -522,6 +524,19 @@ mod tests {
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
}
#[test]
fn flag_utf8_counts_anywhere_in_the_aff() {
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
for _ in 0..400 {
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
}
aff.extend_from_slice(b"FLAG UTF-8\r\n");
assert!(aff.len() > 8192);
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
}
#[test]
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();