Files
craft-libs/crates/proof/src/hunspell.rs
T
Martin BallandClaude Opus 5.5 86345ebcf5 FU.4: craft-proof packs — remembered hyphenation patterns keep to a 256 MB budget of their own; a pack's spelling files are never candidates through a symbolic link
Follow-ups from craft-libs#485: up to 256 languages' pack patterns (64 MB files each) were kept
by count only; `hunspell::ordered` checked pack files with `is_file`, which follows links,
while the pack scan refuses them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
2026-11-02 01:18:42 +00:00

1049 lines
54 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Hunspell dictionaries installed on the system, read at runtime.
//!
//! WordCraft doesn't bundle large spelling dictionaries. For languages with an affix dictionary
//! (Polish, Russian), it reads the standard Hunspell `.aff`/`.dic` pair installed by the operating system,
//! a package manager or LibreOffice, with [`crate::affix::AffixDict::from_hunspell`]. When none is
//! installed the language has no spelling check (no marks, no errors), like Word without the
//! language's proofing tools.
//!
//! Search order: the folder set with [`set_dictionary_dir`], the directory in
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder under the user's application
//! data (on every desktop system), then an installed Language pack ([`crate::packs`]), then the
//! usual system locations. On the web there are only the first and the packs.
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use crate::affix::AffixDict;
/// Largest `.aff` or `.dic` file read (real ones are a few MB).
pub(crate) const MAX_FILE: u64 = 64 << 20;
/// An extra directory searched first (a setting), set by the front end.
static EXTRA_DIR: RwLock<Option<PathBuf>> = RwLock::new(None);
/// Why an installed dictionary failed to load, by `.dic` name (`pt_BR`): the file and the reason.
/// Without it a file that is there but unreadable looks the same as no file (#2830).
static LOAD_ERRORS: RwLock<Vec<(String, PathBuf, String)>> = RwLock::new(Vec::new());
/// Remember (or, for `Ok`, forget) why the dictionary `key` found at `dic` failed to load, and
/// hand the result back unchanged.
pub fn note_load<T>(key: &str, dic: &Path, result: Result<T, String>) -> Result<T, String> {
let mut errors = LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner);
errors.retain(|(k, _, _)| k != key);
if let Err(e) = &result {
errors.push((key.to_string(), dic.to_path_buf(), e.clone()));
}
result
}
/// Forget the load error of the dictionary `key` (none found, or it is about to be searched for again).
pub fn clear_load_error(key: &str) {
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).retain(|(k, _, _)| k != key);
}
/// The `.dic` file and the reason of the last failed load of the dictionary `key`, if any.
pub fn load_error(key: &str) -> Option<(PathBuf, String)> {
LOAD_ERRORS.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().find(|(k, _, _)| k == key).map(|(_, p, e)| (p.clone(), e.clone()))
}
/// Search `dir` before the system locations (a "dictionaries folder" setting); `None` clears it.
/// Takes effect for dictionaries not loaded yet.
pub fn set_dictionary_dir(dir: Option<PathBuf>) {
*EXTRA_DIR.write().unwrap_or_else(std::sync::PoisonError::into_inner) = dir;
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).clear(); // they belong to the old search path
}
/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's
/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both
/// the primary-language folder and the dashed full name are tried.
fn lo_ext_dirs(name: &str) -> Vec<String> {
let primary = name.split(['-', '_']).next().unwrap_or(name);
let dashed = name.replace('_', "-");
let first = format!("dict-{primary}");
let full = format!("dict-{dashed}");
if full == first { vec![first] } else { vec![first, full] }
}
/// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a
/// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`).
pub fn search_dirs(lo_exts: Option<&[String]>) -> Vec<PathBuf> {
let (mut dirs, system) = search_tiers(lo_exts);
dirs.extend(system);
dirs
}
/// [`search_dirs`] in two tiers: the dictionaries folders (the one set with
/// [`set_dictionary_dir`], `WORDCRAFT_DICTIONARIES` and WordCraft's own folder under the user's
/// application data), then the system's Hunspell and LibreOffice folders. Installed Language
/// packs ([`crate::packs`]) are searched between the two.
fn search_tiers(lo_exts: Option<&[String]>) -> (Vec<PathBuf>, Vec<PathBuf>) {
let mut dirs: Vec<PathBuf> = Vec::new();
let mut system: Vec<PathBuf> = Vec::new();
if let Some(d) = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() {
dirs.push(d);
}
if cfg!(target_arch = "wasm32") {
return (dirs, system);
}
if let Some(d) = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()) {
dirs.push(PathBuf::from(d));
}
// LibreOffice keeps each language under `…/share/extensions/<dict-xx>`.
let lo_dirs =
|prefix: &str| -> Vec<PathBuf> { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| PathBuf::from(prefix).join(e)).collect()) };
let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from);
if cfg!(windows) {
for var in ["LOCALAPPDATA", "APPDATA"] {
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
dirs.push(base.join("WordCraft").join("dictionaries"));
system.push(base.join("hunspell"));
}
}
for var in ["ProgramFiles", "ProgramFiles(x86)"] {
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
system.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy()));
}
}
system.extend(lo_drive_dirs(lo_exts, || LO_DRIVE_BASES.get_or_init(|| lo_drive_bases(Path::is_dir))));
} else if cfg!(target_os = "macos") {
if let Some(h) = &home {
// An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this
// folder needs no setting (#1063).
dirs.push(h.join("Library/Application Support/WordCraft/dictionaries"));
system.push(h.join("Library/Spelling"));
}
system.push(PathBuf::from("/Library/Spelling"));
system.push(PathBuf::from("/opt/homebrew/share/hunspell"));
system.push(PathBuf::from("/usr/local/share/hunspell"));
system.extend(lo_dirs("/Applications/LibreOffice.app/Contents/Resources/extensions"));
} else {
// Nor does one started from a desktop launcher (#1063).
if let Some(data) = xdg_data_home(std::env::var_os("XDG_DATA_HOME"), home.as_deref()) {
dirs.push(data.join("wordcraft").join("dictionaries"));
}
if let Some(h) = &home {
system.push(h.join(".local/share/hunspell"));
}
for d in ["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"] {
system.push(PathBuf::from(d));
}
for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] {
system.extend(lo_dirs(d));
}
}
(dirs, system)
}
/// LibreOffice's language folders (`<drive>:\Program Files\LibreOffice\share\extensions\<ext>`)
/// on the drives C to Z where LibreOffice is installed (`bases`). No drive is probed without
/// `lo_exts` (no LibreOffice folders are wanted then).
fn lo_drive_dirs<'a>(lo_exts: Option<&[String]>, bases: impl FnOnce() -> &'a [PathBuf]) -> Vec<PathBuf> {
let Some(exts) = lo_exts else { return Vec::new() };
bases().iter().flat_map(|base| exts.iter().map(|e| base.join(e))).collect()
}
/// The `LibreOffice\share\extensions` folders on drives C to Z (48 probes).
fn lo_drive_bases(is_dir: impl Fn(&Path) -> bool) -> Vec<PathBuf> {
('C'..='Z')
.flat_map(|drive| ["Program Files", "Program Files (x86)"].map(|pf| PathBuf::from(format!(r"{drive}:\{pf}\LibreOffice\share\extensions"))))
.filter(|base| is_dir(base))
.collect()
}
/// [`lo_drive_bases`], probed once per process.
// ponytail: a LibreOffice installed while the app runs is seen after a restart.
static LO_DRIVE_BASES: std::sync::OnceLock<Vec<PathBuf>> = std::sync::OnceLock::new();
/// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the
/// XDG Base Directory spec ignores a relative one), else `~/.local/share`.
fn xdg_data_home(xdg: Option<std::ffi::OsString>, home: Option<&Path>) -> Option<PathBuf> {
xdg.map(PathBuf::from).filter(|p| p.is_absolute()).or_else(|| home.map(|h| h.join(".local/share")))
}
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`), in the
/// order of [`candidates`].
pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> {
candidates(names).into_iter().next()
}
/// Every `<name>.aff`/`<name>.dic` pair for `names`, best first. A caller that can't read one
/// tries the next. The order:
///
/// 1. the dictionaries folders ([`set_dictionary_dir`] and the others of [`search_dirs`]), any
/// of `names`;
/// 2. an installed Language pack of the first name's region ([`crate::packs`],
/// `spelling.aff`/`spelling.dic`; the bare `es` and `de` packs are Spain's and Germany's);
/// 3. the system's folders, the first name's region (`pt_BR`, `pt-BR`);
/// 4. any other installed pack of the language (another region: `pt-pt`, the bare `pt`);
/// 5. the system's folders, the other names (`pt`).
pub fn candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
let lo = names.first().copied().map(lo_ext_dirs);
let (dirs, system) = search_tiers(lo.as_deref());
let (region_packs, other_packs) = crate::packs::spelling_dirs(names);
ordered(names, &dirs, &region_packs, &system, &other_packs)
}
/// [`candidates`] over the given folders.
pub(crate) fn ordered(
names: &[&str],
dirs: &[PathBuf],
region_packs: &[PathBuf],
system: &[PathBuf],
other_packs: &[PathBuf],
) -> Vec<(PathBuf, PathBuf)> {
let norm = |n: &str| n.to_ascii_lowercase().replace('_', "-");
let first = names.first().map(|n| norm(n));
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
let pack: &[&str] = &[crate::packs::SPELLING];
// (folder, names, a Language pack's: never read through a symbolic link, as the pack scan
// checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`).
let tiers = dirs
.iter()
.map(|d| (d, names, false))
.chain(region_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &region[..], false)))
.chain(other_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &rest[..], false)));
let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() };
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
for (dir, names, in_pack) in tiers {
for n in names {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) {
out.push((aff, dic));
}
}
}
out
}
/// [`candidates`] without another region's or dialect's dictionary: the dictionaries folders,
/// then an installed pack named exactly by one of `names` (`en_GB` → `en-gb`), then the system's
/// folders, `names` only. English uses it: en-GB never falls back to an `en_US` file or pack.
pub fn candidates_exact(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
let lo = names.first().copied().map(lo_ext_dirs);
let (dirs, system) = search_tiers(lo.as_deref());
ordered(names, &dirs, &crate::packs::exact_spelling_dirs(names), &system, &[])
}
/// [`candidates`] in the folders the user keeps dictionaries in for WordCraft ([`user_dirs`])
/// only, not the system locations.
pub fn user_candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
candidates_in(&user_dirs(), names)
}
/// The folders of [`search_dirs`] the user keeps dictionaries in for WordCraft: the one set with
/// [`set_dictionary_dir`], `WORDCRAFT_DICTIONARIES`, and WordCraft's own `dictionaries` folder
/// under the user's application data. The rest are the system's (and LibreOffice's).
pub fn user_dirs() -> Vec<PathBuf> {
let extra = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone();
let env = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()).map(PathBuf::from);
let wordcraft_own = |d: &Path| {
d.file_name().is_some_and(|n| n == "dictionaries")
&& d.parent().and_then(Path::file_name).is_some_and(|n| n.eq_ignore_ascii_case("wordcraft"))
};
search_dirs(None).into_iter().filter(|d| Some(d) == extra.as_ref() || Some(d) == env.as_ref() || wordcraft_own(d)).collect()
}
/// [`candidates`] in `dirs`.
pub fn candidates_in(dirs: &[PathBuf], names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
for n in names {
for dir in dirs {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
out.push((aff, dic));
}
}
}
out
}
/// The first `<name>.aff`/`<name>.dic` pair for any of `names` in `dir`.
pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
names.iter().find_map(|n| {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
(aff.is_file() && dic.is_file()).then_some((aff, dic))
})
}
/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is
/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong
/// size) can't make it read more (#3661).
pub(crate) fn read_capped(p: &Path) -> Result<Vec<u8>, String> {
use std::io::Read;
let err = |e: std::io::Error| format!("{}: {e}", p.display());
let f = std::fs::File::open(p).map_err(err)?;
let meta = f.metadata().map_err(err)?;
if !meta.is_file() {
return Err(format!("{}: not a regular file", p.display()));
}
if meta.len() > MAX_FILE {
return Err(format!("{}: too large ({} bytes)", p.display(), meta.len()));
}
let mut buf = Vec::new();
f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?;
if buf.len() as u64 > MAX_FILE {
return Err(format!("{}: too large", p.display()));
}
Ok(buf)
}
/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
let source = format!("Hunspell dictionary {}", dic.display());
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
}
/// [`load`] for an English dictionary: flag-based compounding (`COMPOUNDFLAG`…), which this engine
/// does not implement, is ignored rather than refused ([`AffixDict::from_hunspell_english`]); `OCONV`
/// is applied to suggestions, as for every language.
pub fn load_english(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
let source = format!("Hunspell dictionary {}", dic.display());
AffixDict::from_hunspell_english(&aff_text, &dic_text, &[source.as_str()])
}
/// ISO 8859-1: every byte is the code point of the same value (the original VERO pt_BR uses it).
fn decode_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&b| char::from(b)).collect()
}
/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The
/// encoding of older Turkish dictionaries (`SET ISO8859-9`).
fn decode_latin5(bytes: &[u8]) -> String {
bytes
.iter()
.map(|&b| match b {
0xd0 => 'Ğ',
0xdd => 'İ',
0xde => 'Ş',
0xf0 => 'ğ',
0xfd => 'ı',
0xfe => 'ş',
_ => char::from(b),
})
.collect()
}
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
/// ISO 8859-7 (Greek) code points 0xA0..=0xFF; 0xAE, 0xD2 and 0xFF are unassigned and all read
/// as U+FFFD. From the
/// Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-7.TXT` (2003 edition, with € ₯ ͺ).
/// The encoding of the Greek dictionary (`SET ISO8859-7`).
const ISO_8859_7_HIGH: &str =
"\u{a0}‘’£€₯¦§¨©ͺ«¬\u{ad}\u{fffd}―°±²³΄΅Ά·ΈΉΊ»Ό½ΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ\u{fffd}ΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\u{fffd}";
/// ISO 8859-15 (Latin-9) code points 0xA0..=0xFF: ISO 8859-1 with € Š š Ž ž Œ œ Ÿ in eight
/// places. From the Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-15.TXT`. The
/// encoding of the Estonian dictionary (`SET ISO8859-15`).
const ISO_8859_15_HIGH: &str = "\u{a0}¡¢£€¥Š§š©ª«¬\u{ad}®¯°±²³Žµ¶·ž¹º»ŒœŸ¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ";
/// Decode an ISO 8859 part whose bytes 0xA0..=0xFF are `high` (0x00..=0x9F map to themselves).
fn decode_iso(bytes: &[u8], high: &str) -> String {
let high: Vec<char> = high.chars().collect();
bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
}
/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian
/// dictionaries.
const KOI8_R_HIGH: &str =
"─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ";
/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries
/// use.
const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя";
/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below).
fn decode_high(bytes: &[u8], high: &str) -> String {
let high: Vec<char> = high.chars().collect();
bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect()
}
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, ISO 8859-7 the Greek one, ISO 8859-15 the Estonian one,
/// ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
///
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
/// its number (the Turkish dictionary's `FLAG num`).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let (set, flag) = directives(aff);
let set = set.unwrap_or_else(|| "UTF-8".into());
match set.as_str() {
"UTF-8" | "UTF8" => {
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
}
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_iso(aff, ISO_8859_2_HIGH), decode_iso(dic, ISO_8859_2_HIGH))),
"ISO8859-7" | "ISO-8859-7" => Ok((decode_iso(aff, ISO_8859_7_HIGH), decode_iso(dic, ISO_8859_7_HIGH))),
"ISO8859-15" | "ISO-8859-15" => Ok((decode_iso(aff, ISO_8859_15_HIGH), decode_iso(dic, ISO_8859_15_HIGH))),
"ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
other => Err(format!("unsupported encoding {other}")),
}
}
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
let (mut set, mut flag) = (None, None);
for l in aff.split(|&b| b == b'\n') {
if set.is_some() && flag.is_some() {
break;
}
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
let slot = match l.first() {
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
_ => continue,
};
let arg = l
.strip_prefix(slot.1)
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
if let Some(a) = arg {
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
}
}
(set, flag)
}
/// Directives whose one argument is a flag.
const FLAG_DIRECTIVES: &[&str] = &[
"CIRCUMFIX",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFLAG",
"COMPOUNDFORBIDFLAG",
"COMPOUNDLAST",
"COMPOUNDFIRST",
"COMPOUNDMIDDLE",
"COMPOUNDPERMITFLAG",
"COMPOUNDROOT",
"FORBIDDENWORD",
"FORCEUCASE",
"GENERATE",
"HU_KOTOHANGZO",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"NOSUGGEST",
"ONLYINCOMPOUND",
"ONLYROOT",
"PSEUDOROOT",
"SUBSTANDARD",
"WARN",
];
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
/// names the line in an error.
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
i += 2;
continue;
}
if b == b'/' {
slash = Some(i);
break;
}
i += 1;
}
let (text, flags) = match slash {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
out.push_str(&decode_latin1(flags));
Ok(out)
}
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
/// its parts is the original).
fn fields(line: &[u8]) -> Vec<&[u8]> {
let mut out = Vec::new();
let mut start = 0;
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
for (i, b) in line.iter().enumerate() {
let sep = matches!(b, b' ' | b'\t');
if sep != in_sep {
out.push(line.get(start..i).unwrap_or_default());
start = i;
in_sep = sep;
}
}
out.push(line.get(start..).unwrap_or_default());
out
}
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(aff.len());
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.is_ascii()
{
out.push_str(s);
continue;
}
let parts = fields(line);
// Field index among the non-separator parts.
let words: Vec<usize> =
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
let dir = words
.first()
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = || format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
}
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
// a flag; from there on the line is kept as text. It can only start after the directive's
// own fields, since a field such as an affix condition may itself begin with `#`.
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
let first_free = comment_field(&dir, field_text);
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
// The field index of the next non-separator part.
let mut next_field = 0;
for (i, p) in parts.iter().enumerate() {
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
next_field += 1;
next_field - 1
});
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
};
out.push_str(&decoded);
}
}
Ok(out)
}
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
match dir {
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
"PFX" | "SFX" => 5,
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
{
2
}
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
"CHECKCOMPOUNDPATTERN" => 4,
_ => 2,
}
}
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
let blank = |b: &u8| matches!(b, b' ' | b'\t');
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
let tab = line.iter().position(|&b| b == b'\t');
let end = match (morph, tab) {
(Some(m), Some(t)) => m.min(t),
(m, t) => match m.or(t) {
Some(e) => e,
None => line.len(),
},
};
// Blanks before the morphological field or the line end aren't part of the entry.
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
}
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(dic.len());
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
{
out.push_str(s);
continue;
}
let what = || format!(".dic line {}", n + 1);
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
// morphological fields after it are UTF-8 text.
let end = dic_entry_end(line);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// CodeRabbit on craft-libs#489: without LibreOffice extension names no drive is probed; with
/// them, only drives where LibreOffice is found give folders.
#[test]
fn libreoffice_drives_are_probed_only_when_wanted() {
let probes = std::cell::Cell::new(0);
let count = |_: &Path| {
probes.set(probes.get() + 1);
false
};
// craft-libs#495 review: the drives are probed once, not on every lookup.
let cache = std::sync::OnceLock::new();
let cached = || cache.get_or_init(|| lo_drive_bases(count)).as_slice();
assert!(lo_drive_dirs(None, cached).is_empty());
assert_eq!(probes.get(), 0);
let exts = ["dict-tr".to_string()];
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
assert_eq!(probes.get(), 48);
let found = lo_drive_bases(|p: &Path| p.to_string_lossy().starts_with(r"D:\Program Files\"));
assert_eq!(
lo_drive_dirs(Some(&exts), || found.as_slice()),
[PathBuf::from(r"D:\Program Files\LibreOffice\share\extensions").join("dict-tr")]
);
}
/// Another region's Language pack never beats the system's dictionary of the right region
/// (`pt-pt` pack vs system `pt_BR`; `de-ch` vs `de_DE`; `en-gb` vs `en_US`), but beats the
/// system's other names; the right region's pack beats the system; the dictionaries folder
/// beats everything.
#[test]
fn another_regions_pack_comes_after_the_systems_exact_region() {
let root = std::env::temp_dir().join(format!("wc-proof-order-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let pair = |dir: &Path, name: &str| {
std::fs::create_dir_all(dir).unwrap();
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
dir.join(format!("{name}.dic"))
};
let (folder, system) = (root.join("folder"), root.join("system"));
let en: &[&str] = &["en_US", "en-US", "en", "en_CA", "en_GB", "en-GB", "en_AU"];
for (names, exact, other, cross) in [
(&["pt_BR", "pt-BR", "pt"][..], "pt_BR", "pt", "pt-pt"),
(&["de_DE", "de-DE", "de"][..], "de_DE", "de", "de-ch"),
(&["nl_NL", "nl-NL", "nl"][..], "nl_NL", "nl", "nl-be"),
(&["es_ES", "es-ES", "es_ANY", "es"][..], "es_ES", "es", "es-mx"),
(en, "en_US", "en_GB", "en-gb"),
] {
let sys_exact = pair(&system, exact);
let sys_other = pair(&system, other);
let cross_pack = pair(&root.join("packs").join(cross), crate::packs::SPELLING);
let dics = |v: Vec<(PathBuf, PathBuf)>| v.into_iter().map(|(_, d)| d).collect::<Vec<_>>();
let got =
dics(ordered(names, &[], &[], std::slice::from_ref(&system), std::slice::from_ref(&cross_pack.parent().unwrap().to_path_buf())));
assert_eq!(got, [sys_exact.clone(), cross_pack.clone(), sys_other.clone()], "{names:?}");
// The region's own pack first, then the system's exact region.
let region_pack = pair(&root.join("packs").join(format!("{}-region", names[0])), crate::packs::SPELLING);
let got = dics(ordered(names, &[], &[region_pack.parent().unwrap().to_path_buf()], std::slice::from_ref(&system), &[]));
assert_eq!(got.first(), Some(&region_pack), "{names:?}");
assert_eq!(got.get(1), Some(&sys_exact), "{names:?}");
// The dictionaries folder before all of them, whatever name.
let mine = pair(&folder, other);
let got = dics(ordered(
names,
std::slice::from_ref(&folder),
&[region_pack.parent().unwrap().to_path_buf()],
std::slice::from_ref(&system),
&[],
));
assert_eq!(got.first(), Some(&mine), "{names:?}");
assert_eq!(find_in(&folder, &[other]).map(|(_, d)| d), Some(mine));
let _ = std::fs::remove_dir_all(&root);
}
}
/// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link
/// (swapped in after the pack scan) is never a candidate; one in a system folder still is.
#[cfg(unix)]
#[test]
fn pack_files_are_never_read_through_a_link() {
let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere"));
for d in [&pack, &system, &elsewhere] {
std::fs::create_dir_all(d).unwrap();
}
std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap();
std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap();
std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never");
std::fs::remove_file(pack.join("spelling.dic")).unwrap();
std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]);
let _ = std::fs::remove_dir_all(&root);
}
/// WordCraft's own dictionaries folder under the user's application data is searched on every
/// desktop system, after the setting and before the shared system locations (#1063).
#[test]
fn the_app_folder_comes_before_the_system_locations() {
if cfg!(target_arch = "wasm32") {
return;
}
let user = if cfg!(windows) { "LOCALAPPDATA" } else { "HOME" };
if std::env::var_os(user).filter(|v| !v.is_empty()).is_none() {
return; // no user folder to derive it from
}
let dirs = search_dirs(None);
let at = |suffix: &str| dirs.iter().position(|d| d.ends_with(suffix));
let (app, shared) = if cfg!(windows) {
("WordCraft/dictionaries", "hunspell")
} else if cfg!(target_os = "macos") {
("Application Support/WordCraft/dictionaries", "/Library/Spelling")
} else {
("wordcraft/dictionaries", "/usr/share/hunspell")
};
assert!(matches!((at(app), at(shared)), (Some(app), Some(shared)) if app < shared), "{dirs:?}");
}
/// `$XDG_DATA_HOME` counts only as an absolute path; otherwise (unset, empty or relative) the
/// spec's default `~/.local/share` applies, and with no home there is no folder.
#[test]
fn xdg_data_home_follows_the_spec() {
let home = Path::new("/home/a");
assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data")));
assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data")));
for xdg in [None, Some(""), Some("relative/data")] {
assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}");
assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}");
}
}
#[test]
fn decodes_latin2_and_utf8() {
let aff = b"SET ISO8859-2\nTRY a\n";
let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap();
assert!(a.contains("TRY a"));
assert_eq!(d, "2\nżółw\n");
let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
assert_eq!(d, "żółw");
let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap();
assert_eq!(d, "ĞİŞğışÖç");
assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
}
/// #2830: a dictionary that is there but can't be read leaves its reason behind; a good
/// load or no file clears it.
#[test]
fn load_error_is_remembered_until_the_next_good_load() {
let key = "test_unreadable_2830";
assert_eq!(load_error(key), None);
let dic = Path::new("/dicts/test.dic");
assert!(note_load(key, dic, decode(b"SET SHIFT_JIS\n", b"")).is_err());
let (file, reason) = load_error(key).expect("error kept");
assert_eq!(file, dic);
assert_eq!(reason, "unsupported encoding SHIFT_JIS");
assert!(note_load(key, dic, decode(b"SET UTF-8\n", b"")).is_ok());
assert_eq!(load_error(key), None);
let _ = note_load::<()>(key, dic, Err("too large".into()));
clear_load_error(key);
assert_eq!(load_error(key), None);
}
/// Every table covers its 96 (ISO 8859) or 128 (KOI8-R, Windows-1251) high bytes.
#[test]
fn tables_are_complete() {
for t in [ISO_8859_2_HIGH, ISO_8859_7_HIGH, ISO_8859_15_HIGH] {
assert_eq!(t.chars().count(), 96, "{t}");
}
for t in [KOI8_R_HIGH, CP1251_HIGH] {
assert_eq!(t.chars().count(), 128, "{t}");
}
}
#[test]
fn decodes_greek_and_latin9() {
// `σπίτι νερό` in ISO 8859-7, and a flag after the slash stays ASCII.
let (a, d) = decode(b"SET ISO8859-7\nTRY \xe1\n", b"2\n\xf3\xf0\xdf\xf4\xe9/A\n\xed\xe5\xf1\xfc\n").unwrap();
assert!(a.contains("TRY α"), "{a}");
assert_eq!(d, "2\nσπίτι/A\nνερό\n");
// Unassigned bytes and the 2003 additions.
let (_, d) = decode(b"SET ISO-8859-7\n", b"\xae\xd2\xff\xa4\xa5\xaa\xa2").unwrap();
assert_eq!(d, "\u{fffd}\u{fffd}\u{fffd}€₯ͺ’");
// `šokolaad õun` in ISO 8859-15, and its eight differences from ISO 8859-1.
let (_, d) = decode(b"SET ISO8859-15\n", b"\xa8okolaad \xf5un").unwrap();
assert_eq!(d, "šokolaad õun");
let (_, d) = decode(b"SET ISO-8859-15\n", b"\xa4\xa6\xa8\xb4\xb8\xbc\xbd\xbe\xe4").unwrap();
assert_eq!(d, "€ŠšŽžŒœŸä");
}
#[test]
fn decodes_latin1() {
// `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1.
let (a, d) = decode(b"SET ISO8859-1\nTRY \xe7\n", b"1\nor\xe7amento/A\nesper\xe1vamos\n").unwrap();
assert!(a.contains("TRY ç"), "{a}");
assert_eq!(d, "1\norçamento/A\nesperávamos\n");
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn utf8_dictionaries_keep_single_byte_flags() {
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
let (a, d) = decode(aff, dic).unwrap();
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
assert!(!dict.check("házban"));
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
}
#[test]
fn single_flag_fields_keep_their_first_byte() {
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
assert!(d.contains("kert/\u{c3}\u{a9}"));
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("kert") && dict.check("kertok"));
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
}
#[test]
fn numeric_and_long_flags_stay_text() {
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
// number with a non-ASCII rule or comment keeps every digit.
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
assert_eq!(a, aff);
assert_eq!(d, "1\nkitap/1234\n");
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
assert!(tr.contains("kitapları") && tr.contains("kitap"));
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
// `FLAG DEFAULT` is the single-byte default.
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
assert_eq!(d, "1\nkert/\u{e9}\n");
}
#[test]
fn flag_utf8_counts_anywhere_in_the_aff() {
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
for _ in 0..400 {
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
}
aff.extend_from_slice(b"FLAG UTF-8\r\n");
assert!(aff.len() > 8192);
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
}
#[test]
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap();
assert!(a.contains("TRY ое"), "{a}");
assert_eq!(d, "1\nёлка/A\n");
let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap();
assert_eq!(d, "1\nёлка Дом\n");
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
}
#[test]
fn comments_start_after_the_directive_fields() {
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
assert_eq!(comment_field("REP", f("REP 12")), 2);
assert_eq!(comment_field("REP", f("REP # x")), 3);
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
}
#[test]
fn dic_entries_end_where_hunspell_ends_them() {
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
assert_eq!(end("ad hoc/B "), "ad hoc/B");
assert_eq!(end("word"), "word");
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
// Without a tab or a morphological field the whole line is the entry: its byte flags
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\n");
}
#[test]
fn libreoffice_extension_dirs_follow_the_language() {
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
// `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried.
assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]);
assert_eq!(super::lo_ext_dirs("id_ID"), ["dict-id", "dict-id-ID"]);
assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);
assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]);
assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]);
let dirs = search_dirs(Some(&super::lo_ext_dirs("pt_BR")));
assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}");
assert!(search_dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess");
}
/// craft-libs#482 re-review: a name is looked for in every folder before the next name, so
/// an `en_US` in the dictionaries folder doesn't beat a system `en_GB` for British English.
#[test]
fn candidates_go_name_by_name_across_the_folders() {
let base = std::env::temp_dir().join(format!("wordcraft-candidates-{}", std::process::id()));
let (folder, system) = (base.join("folder"), base.join("system"));
for (dir, name) in [(&folder, "en_US"), (&system, "en_GB"), (&folder, "en-GB")] {
std::fs::create_dir_all(dir).unwrap();
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
}
let dirs = [folder.clone(), system.clone()];
let dics: Vec<PathBuf> = candidates_in(&dirs, &["en_GB", "en-GB"]).into_iter().map(|(_, d)| d).collect();
assert_eq!(dics, vec![system.join("en_GB.dic"), folder.join("en-GB.dic")]);
let us: Vec<PathBuf> = candidates_in(&dirs, &["en_US", "en_GB"]).into_iter().map(|(_, d)| d).collect();
assert_eq!(us, vec![folder.join("en_US.dic"), system.join("en_GB.dic")]);
assert!(candidates_in(&dirs, &["en_CA"]).is_empty());
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn finds_and_loads_a_pair_in_a_directory() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found");
std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap();
assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary");
std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap();
let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap();
let d = load(&aff, &dic).unwrap();
assert!(d.check("kot") && d.check("koty") && !d.check("kotx"));
assert!(load(&dir.join("missing.aff"), &dic).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn reads_are_bounded_and_regular_files_only() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let aff = dir.join("xx.aff");
std::fs::write(&aff, "SET UTF-8\n").unwrap();
// One byte over the cap (sparse: nothing is written) is refused without reading it.
let big = dir.join("big.dic");
std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap();
assert!(load(&aff, &big).unwrap_err().contains("too large"));
// A directory, and on Unix an endless device, are not dictionary files. (No clock: a
// reader that read `/dev/zero` would never return, so the error itself is the proof.)
assert!(load(&aff, &dir).unwrap_err().contains("not a regular file"));
#[cfg(unix)]
assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file"));
// Turkish reads through the same reader.
let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap();
std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap();
assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar"));
assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large"));
let _ = std::fs::remove_dir_all(&dir);
}
/// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from
/// disk through the capped reader and decoded per its `SET` line: its words with Turkish
/// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the
/// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not.
#[test]
fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap();
// ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş.
let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n";
std::fs::write(&dic, words).unwrap();
assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8");
let d = crate::tr::load(&aff, &dic).unwrap();
for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] {
assert!(d.contains(w), "{w}");
}
// The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those.
assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir"));
assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them");
let _ = std::fs::remove_dir_all(&dir);
}
}