mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 15:44:26 +00:00
Follow-ups from craft-libs#485: up to 256 languages' pack patterns (64 MB files each) were kept by count only; `hunspell::ordered` checked pack files with `is_file`, which follows links, while the pack scan refuses them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
1049 lines
54 KiB
Rust
1049 lines
54 KiB
Rust
//! Hunspell dictionaries installed on the system, read at runtime.
|
||
//!
|
||
//! WordCraft doesn't bundle large spelling dictionaries. For languages with an affix dictionary
|
||
//! (Polish, Russian), it reads the standard Hunspell `.aff`/`.dic` pair installed by the operating system,
|
||
//! a package manager or LibreOffice, with [`crate::affix::AffixDict::from_hunspell`]. When none is
|
||
//! installed the language has no spelling check (no marks, no errors), like Word without the
|
||
//! language's proofing tools.
|
||
//!
|
||
//! Search order: the folder set with [`set_dictionary_dir`], the directory in
|
||
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder under the user's application
|
||
//! data (on every desktop system), then an installed Language pack ([`crate::packs`]), then the
|
||
//! usual system locations. On the web there are only the first and the packs.
|
||
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::RwLock;
|
||
|
||
use crate::affix::AffixDict;
|
||
|
||
/// Largest `.aff` or `.dic` file read (real ones are a few MB).
|
||
pub(crate) const MAX_FILE: u64 = 64 << 20;
|
||
|
||
/// An extra directory searched first (a setting), set by the front end.
|
||
static EXTRA_DIR: RwLock<Option<PathBuf>> = RwLock::new(None);
|
||
|
||
/// Why an installed dictionary failed to load, by `.dic` name (`pt_BR`): the file and the reason.
|
||
/// Without it a file that is there but unreadable looks the same as no file (#2830).
|
||
static LOAD_ERRORS: RwLock<Vec<(String, PathBuf, String)>> = RwLock::new(Vec::new());
|
||
|
||
/// Remember (or, for `Ok`, forget) why the dictionary `key` found at `dic` failed to load, and
|
||
/// hand the result back unchanged.
|
||
pub fn note_load<T>(key: &str, dic: &Path, result: Result<T, String>) -> Result<T, String> {
|
||
let mut errors = LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||
errors.retain(|(k, _, _)| k != key);
|
||
if let Err(e) = &result {
|
||
errors.push((key.to_string(), dic.to_path_buf(), e.clone()));
|
||
}
|
||
result
|
||
}
|
||
|
||
/// Forget the load error of the dictionary `key` (none found, or it is about to be searched for again).
|
||
pub fn clear_load_error(key: &str) {
|
||
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).retain(|(k, _, _)| k != key);
|
||
}
|
||
|
||
/// The `.dic` file and the reason of the last failed load of the dictionary `key`, if any.
|
||
pub fn load_error(key: &str) -> Option<(PathBuf, String)> {
|
||
LOAD_ERRORS.read().unwrap_or_else(std::sync::PoisonError::into_inner).iter().find(|(k, _, _)| k == key).map(|(_, p, e)| (p.clone(), e.clone()))
|
||
}
|
||
|
||
/// Search `dir` before the system locations (a "dictionaries folder" setting); `None` clears it.
|
||
/// Takes effect for dictionaries not loaded yet.
|
||
pub fn set_dictionary_dir(dir: Option<PathBuf>) {
|
||
*EXTRA_DIR.write().unwrap_or_else(std::sync::PoisonError::into_inner) = dir;
|
||
LOAD_ERRORS.write().unwrap_or_else(std::sync::PoisonError::into_inner).clear(); // they belong to the old search path
|
||
}
|
||
|
||
/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's
|
||
/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both
|
||
/// the primary-language folder and the dashed full name are tried.
|
||
fn lo_ext_dirs(name: &str) -> Vec<String> {
|
||
let primary = name.split(['-', '_']).next().unwrap_or(name);
|
||
let dashed = name.replace('_', "-");
|
||
let first = format!("dict-{primary}");
|
||
let full = format!("dict-{dashed}");
|
||
if full == first { vec![first] } else { vec![first, full] }
|
||
}
|
||
|
||
/// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a
|
||
/// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`).
|
||
pub fn search_dirs(lo_exts: Option<&[String]>) -> Vec<PathBuf> {
|
||
let (mut dirs, system) = search_tiers(lo_exts);
|
||
dirs.extend(system);
|
||
dirs
|
||
}
|
||
|
||
/// [`search_dirs`] in two tiers: the dictionaries folders (the one set with
|
||
/// [`set_dictionary_dir`], `WORDCRAFT_DICTIONARIES` and WordCraft's own folder under the user's
|
||
/// application data), then the system's Hunspell and LibreOffice folders. Installed Language
|
||
/// packs ([`crate::packs`]) are searched between the two.
|
||
fn search_tiers(lo_exts: Option<&[String]>) -> (Vec<PathBuf>, Vec<PathBuf>) {
|
||
let mut dirs: Vec<PathBuf> = Vec::new();
|
||
let mut system: Vec<PathBuf> = Vec::new();
|
||
if let Some(d) = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone() {
|
||
dirs.push(d);
|
||
}
|
||
if cfg!(target_arch = "wasm32") {
|
||
return (dirs, system);
|
||
}
|
||
if let Some(d) = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()) {
|
||
dirs.push(PathBuf::from(d));
|
||
}
|
||
// LibreOffice keeps each language under `…/share/extensions/<dict-xx>`.
|
||
let lo_dirs =
|
||
|prefix: &str| -> Vec<PathBuf> { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| PathBuf::from(prefix).join(e)).collect()) };
|
||
let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from);
|
||
if cfg!(windows) {
|
||
for var in ["LOCALAPPDATA", "APPDATA"] {
|
||
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
|
||
dirs.push(base.join("WordCraft").join("dictionaries"));
|
||
system.push(base.join("hunspell"));
|
||
}
|
||
}
|
||
for var in ["ProgramFiles", "ProgramFiles(x86)"] {
|
||
if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) {
|
||
system.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy()));
|
||
}
|
||
}
|
||
system.extend(lo_drive_dirs(lo_exts, || LO_DRIVE_BASES.get_or_init(|| lo_drive_bases(Path::is_dir))));
|
||
} else if cfg!(target_os = "macos") {
|
||
if let Some(h) = &home {
|
||
// An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this
|
||
// folder needs no setting (#1063).
|
||
dirs.push(h.join("Library/Application Support/WordCraft/dictionaries"));
|
||
system.push(h.join("Library/Spelling"));
|
||
}
|
||
system.push(PathBuf::from("/Library/Spelling"));
|
||
system.push(PathBuf::from("/opt/homebrew/share/hunspell"));
|
||
system.push(PathBuf::from("/usr/local/share/hunspell"));
|
||
system.extend(lo_dirs("/Applications/LibreOffice.app/Contents/Resources/extensions"));
|
||
} else {
|
||
// Nor does one started from a desktop launcher (#1063).
|
||
if let Some(data) = xdg_data_home(std::env::var_os("XDG_DATA_HOME"), home.as_deref()) {
|
||
dirs.push(data.join("wordcraft").join("dictionaries"));
|
||
}
|
||
if let Some(h) = &home {
|
||
system.push(h.join(".local/share/hunspell"));
|
||
}
|
||
for d in ["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"] {
|
||
system.push(PathBuf::from(d));
|
||
}
|
||
for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] {
|
||
system.extend(lo_dirs(d));
|
||
}
|
||
}
|
||
(dirs, system)
|
||
}
|
||
|
||
/// LibreOffice's language folders (`<drive>:\Program Files\LibreOffice\share\extensions\<ext>`)
|
||
/// on the drives C to Z where LibreOffice is installed (`bases`). No drive is probed without
|
||
/// `lo_exts` (no LibreOffice folders are wanted then).
|
||
fn lo_drive_dirs<'a>(lo_exts: Option<&[String]>, bases: impl FnOnce() -> &'a [PathBuf]) -> Vec<PathBuf> {
|
||
let Some(exts) = lo_exts else { return Vec::new() };
|
||
bases().iter().flat_map(|base| exts.iter().map(|e| base.join(e))).collect()
|
||
}
|
||
|
||
/// The `LibreOffice\share\extensions` folders on drives C to Z (48 probes).
|
||
fn lo_drive_bases(is_dir: impl Fn(&Path) -> bool) -> Vec<PathBuf> {
|
||
('C'..='Z')
|
||
.flat_map(|drive| ["Program Files", "Program Files (x86)"].map(|pf| PathBuf::from(format!(r"{drive}:\{pf}\LibreOffice\share\extensions"))))
|
||
.filter(|base| is_dir(base))
|
||
.collect()
|
||
}
|
||
|
||
/// [`lo_drive_bases`], probed once per process.
|
||
// ponytail: a LibreOffice installed while the app runs is seen after a restart.
|
||
static LO_DRIVE_BASES: std::sync::OnceLock<Vec<PathBuf>> = std::sync::OnceLock::new();
|
||
|
||
/// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the
|
||
/// XDG Base Directory spec ignores a relative one), else `~/.local/share`.
|
||
fn xdg_data_home(xdg: Option<std::ffi::OsString>, home: Option<&Path>) -> Option<PathBuf> {
|
||
xdg.map(PathBuf::from).filter(|p| p.is_absolute()).or_else(|| home.map(|h| h.join(".local/share")))
|
||
}
|
||
|
||
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`), in the
|
||
/// order of [`candidates`].
|
||
pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> {
|
||
candidates(names).into_iter().next()
|
||
}
|
||
|
||
/// Every `<name>.aff`/`<name>.dic` pair for `names`, best first. A caller that can't read one
|
||
/// tries the next. The order:
|
||
///
|
||
/// 1. the dictionaries folders ([`set_dictionary_dir`] and the others of [`search_dirs`]), any
|
||
/// of `names`;
|
||
/// 2. an installed Language pack of the first name's region ([`crate::packs`],
|
||
/// `spelling.aff`/`spelling.dic`; the bare `es` and `de` packs are Spain's and Germany's);
|
||
/// 3. the system's folders, the first name's region (`pt_BR`, `pt-BR`);
|
||
/// 4. any other installed pack of the language (another region: `pt-pt`, the bare `pt`);
|
||
/// 5. the system's folders, the other names (`pt`).
|
||
pub fn candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
|
||
let lo = names.first().copied().map(lo_ext_dirs);
|
||
let (dirs, system) = search_tiers(lo.as_deref());
|
||
let (region_packs, other_packs) = crate::packs::spelling_dirs(names);
|
||
ordered(names, &dirs, ®ion_packs, &system, &other_packs)
|
||
}
|
||
|
||
/// [`candidates`] over the given folders.
|
||
pub(crate) fn ordered(
|
||
names: &[&str],
|
||
dirs: &[PathBuf],
|
||
region_packs: &[PathBuf],
|
||
system: &[PathBuf],
|
||
other_packs: &[PathBuf],
|
||
) -> Vec<(PathBuf, PathBuf)> {
|
||
let norm = |n: &str| n.to_ascii_lowercase().replace('_', "-");
|
||
let first = names.first().map(|n| norm(n));
|
||
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
|
||
let pack: &[&str] = &[crate::packs::SPELLING];
|
||
// (folder, names, a Language pack's: never read through a symbolic link, as the pack scan
|
||
// checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`).
|
||
let tiers = dirs
|
||
.iter()
|
||
.map(|d| (d, names, false))
|
||
.chain(region_packs.iter().map(|d| (d, pack, true)))
|
||
.chain(system.iter().map(|d| (d, ®ion[..], false)))
|
||
.chain(other_packs.iter().map(|d| (d, pack, true)))
|
||
.chain(system.iter().map(|d| (d, &rest[..], false)));
|
||
let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() };
|
||
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
|
||
for (dir, names, in_pack) in tiers {
|
||
for n in names {
|
||
let aff = dir.join(format!("{n}.aff"));
|
||
let dic = dir.join(format!("{n}.dic"));
|
||
if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) {
|
||
out.push((aff, dic));
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// [`candidates`] without another region's or dialect's dictionary: the dictionaries folders,
|
||
/// then an installed pack named exactly by one of `names` (`en_GB` → `en-gb`), then the system's
|
||
/// folders, `names` only. English uses it: en-GB never falls back to an `en_US` file or pack.
|
||
pub fn candidates_exact(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
|
||
let lo = names.first().copied().map(lo_ext_dirs);
|
||
let (dirs, system) = search_tiers(lo.as_deref());
|
||
ordered(names, &dirs, &crate::packs::exact_spelling_dirs(names), &system, &[])
|
||
}
|
||
|
||
/// [`candidates`] in the folders the user keeps dictionaries in for WordCraft ([`user_dirs`])
|
||
/// only, not the system locations.
|
||
pub fn user_candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
|
||
candidates_in(&user_dirs(), names)
|
||
}
|
||
|
||
/// The folders of [`search_dirs`] the user keeps dictionaries in for WordCraft: the one set with
|
||
/// [`set_dictionary_dir`], `WORDCRAFT_DICTIONARIES`, and WordCraft's own `dictionaries` folder
|
||
/// under the user's application data. The rest are the system's (and LibreOffice's).
|
||
pub fn user_dirs() -> Vec<PathBuf> {
|
||
let extra = EXTRA_DIR.read().unwrap_or_else(std::sync::PoisonError::into_inner).clone();
|
||
let env = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()).map(PathBuf::from);
|
||
let wordcraft_own = |d: &Path| {
|
||
d.file_name().is_some_and(|n| n == "dictionaries")
|
||
&& d.parent().and_then(Path::file_name).is_some_and(|n| n.eq_ignore_ascii_case("wordcraft"))
|
||
};
|
||
search_dirs(None).into_iter().filter(|d| Some(d) == extra.as_ref() || Some(d) == env.as_ref() || wordcraft_own(d)).collect()
|
||
}
|
||
|
||
/// [`candidates`] in `dirs`.
|
||
pub fn candidates_in(dirs: &[PathBuf], names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
|
||
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
|
||
for n in names {
|
||
for dir in dirs {
|
||
let aff = dir.join(format!("{n}.aff"));
|
||
let dic = dir.join(format!("{n}.dic"));
|
||
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
|
||
out.push((aff, dic));
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// The first `<name>.aff`/`<name>.dic` pair for any of `names` in `dir`.
|
||
pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
|
||
names.iter().find_map(|n| {
|
||
let aff = dir.join(format!("{n}.aff"));
|
||
let dic = dir.join(format!("{n}.dic"));
|
||
(aff.is_file() && dic.is_file()).then_some((aff, dic))
|
||
})
|
||
}
|
||
|
||
/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is
|
||
/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong
|
||
/// size) can't make it read more (#3661).
|
||
pub(crate) fn read_capped(p: &Path) -> Result<Vec<u8>, String> {
|
||
use std::io::Read;
|
||
let err = |e: std::io::Error| format!("{}: {e}", p.display());
|
||
let f = std::fs::File::open(p).map_err(err)?;
|
||
let meta = f.metadata().map_err(err)?;
|
||
if !meta.is_file() {
|
||
return Err(format!("{}: not a regular file", p.display()));
|
||
}
|
||
if meta.len() > MAX_FILE {
|
||
return Err(format!("{}: too large ({} bytes)", p.display(), meta.len()));
|
||
}
|
||
let mut buf = Vec::new();
|
||
f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?;
|
||
if buf.len() as u64 > MAX_FILE {
|
||
return Err(format!("{}: too large", p.display()));
|
||
}
|
||
Ok(buf)
|
||
}
|
||
|
||
/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
|
||
pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
|
||
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
|
||
let source = format!("Hunspell dictionary {}", dic.display());
|
||
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
|
||
}
|
||
|
||
/// [`load`] for an English dictionary: flag-based compounding (`COMPOUNDFLAG`…), which this engine
|
||
/// does not implement, is ignored rather than refused ([`AffixDict::from_hunspell_english`]); `OCONV`
|
||
/// is applied to suggestions, as for every language.
|
||
pub fn load_english(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
|
||
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
|
||
let source = format!("Hunspell dictionary {}", dic.display());
|
||
AffixDict::from_hunspell_english(&aff_text, &dic_text, &[source.as_str()])
|
||
}
|
||
|
||
/// ISO 8859-1: every byte is the code point of the same value (the original VERO pt_BR uses it).
|
||
fn decode_latin1(bytes: &[u8]) -> String {
|
||
bytes.iter().map(|&b| char::from(b)).collect()
|
||
}
|
||
|
||
/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The
|
||
/// encoding of older Turkish dictionaries (`SET ISO8859-9`).
|
||
fn decode_latin5(bytes: &[u8]) -> String {
|
||
bytes
|
||
.iter()
|
||
.map(|&b| match b {
|
||
0xd0 => 'Ğ',
|
||
0xdd => 'İ',
|
||
0xde => 'Ş',
|
||
0xf0 => 'ğ',
|
||
0xfd => 'ı',
|
||
0xfe => 'ş',
|
||
_ => char::from(b),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
|
||
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||
|
||
/// ISO 8859-7 (Greek) code points 0xA0..=0xFF; 0xAE, 0xD2 and 0xFF are unassigned and all read
|
||
/// as U+FFFD. From the
|
||
/// Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-7.TXT` (2003 edition, with € ₯ ͺ).
|
||
/// The encoding of the Greek dictionary (`SET ISO8859-7`).
|
||
const ISO_8859_7_HIGH: &str =
|
||
"\u{a0}‘’£€₯¦§¨©ͺ«¬\u{ad}\u{fffd}―°±²³΄΅Ά·ΈΉΊ»Ό½ΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ\u{fffd}ΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\u{fffd}";
|
||
|
||
/// ISO 8859-15 (Latin-9) code points 0xA0..=0xFF: ISO 8859-1 with € Š š Ž ž Œ œ Ÿ in eight
|
||
/// places. From the Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-15.TXT`. The
|
||
/// encoding of the Estonian dictionary (`SET ISO8859-15`).
|
||
const ISO_8859_15_HIGH: &str = "\u{a0}¡¢£€¥Š§š©ª«¬\u{ad}®¯°±²³Žµ¶·ž¹º»ŒœŸ¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ";
|
||
|
||
/// Decode an ISO 8859 part whose bytes 0xA0..=0xFF are `high` (0x00..=0x9F map to themselves).
|
||
fn decode_iso(bytes: &[u8], high: &str) -> String {
|
||
let high: Vec<char> = high.chars().collect();
|
||
bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
|
||
}
|
||
|
||
/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian
|
||
/// dictionaries.
|
||
const KOI8_R_HIGH: &str =
|
||
"─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ";
|
||
|
||
/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries
|
||
/// use.
|
||
const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя";
|
||
|
||
/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below).
|
||
fn decode_high(bytes: &[u8], high: &str) -> String {
|
||
let high: Vec<char> = high.chars().collect();
|
||
bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect()
|
||
}
|
||
|
||
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
|
||
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
|
||
/// one the original Portuguese VERO uses, ISO 8859-7 the Greek one, ISO 8859-15 the Estonian one,
|
||
/// ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
|
||
/// and Windows-1251 the ones older Russian dictionaries use).
|
||
///
|
||
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
|
||
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
|
||
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
|
||
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
|
||
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
|
||
/// its number (the Turkish dictionary's `FLAG num`).
|
||
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
|
||
let (set, flag) = directives(aff);
|
||
let set = set.unwrap_or_else(|| "UTF-8".into());
|
||
match set.as_str() {
|
||
"UTF-8" | "UTF8" => {
|
||
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
|
||
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
|
||
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
|
||
}
|
||
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
|
||
}
|
||
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
|
||
"ISO8859-2" | "ISO-8859-2" => Ok((decode_iso(aff, ISO_8859_2_HIGH), decode_iso(dic, ISO_8859_2_HIGH))),
|
||
"ISO8859-7" | "ISO-8859-7" => Ok((decode_iso(aff, ISO_8859_7_HIGH), decode_iso(dic, ISO_8859_7_HIGH))),
|
||
"ISO8859-15" | "ISO-8859-15" => Ok((decode_iso(aff, ISO_8859_15_HIGH), decode_iso(dic, ISO_8859_15_HIGH))),
|
||
"ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
|
||
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
|
||
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
|
||
other => Err(format!("unsupported encoding {other}")),
|
||
}
|
||
}
|
||
|
||
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
|
||
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
|
||
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
|
||
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
|
||
let (mut set, mut flag) = (None, None);
|
||
for l in aff.split(|&b| b == b'\n') {
|
||
if set.is_some() && flag.is_some() {
|
||
break;
|
||
}
|
||
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
|
||
let slot = match l.first() {
|
||
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
|
||
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
|
||
_ => continue,
|
||
};
|
||
let arg = l
|
||
.strip_prefix(slot.1)
|
||
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
|
||
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
|
||
if let Some(a) = arg {
|
||
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
|
||
}
|
||
}
|
||
(set, flag)
|
||
}
|
||
|
||
/// Directives whose one argument is a flag.
|
||
const FLAG_DIRECTIVES: &[&str] = &[
|
||
"CIRCUMFIX",
|
||
"COMPOUNDBEGIN",
|
||
"COMPOUNDEND",
|
||
"COMPOUNDFLAG",
|
||
"COMPOUNDFORBIDFLAG",
|
||
"COMPOUNDLAST",
|
||
"COMPOUNDFIRST",
|
||
"COMPOUNDMIDDLE",
|
||
"COMPOUNDPERMITFLAG",
|
||
"COMPOUNDROOT",
|
||
"FORBIDDENWORD",
|
||
"FORCEUCASE",
|
||
"GENERATE",
|
||
"HU_KOTOHANGZO",
|
||
"KEEPCASE",
|
||
"LEMMA_PRESENT",
|
||
"NEEDAFFIX",
|
||
"NOSUGGEST",
|
||
"ONLYINCOMPOUND",
|
||
"ONLYROOT",
|
||
"PSEUDOROOT",
|
||
"SUBSTANDARD",
|
||
"WARN",
|
||
];
|
||
|
||
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
|
||
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
|
||
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
|
||
|
||
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
|
||
/// names the line in an error.
|
||
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
|
||
let mut slash = None;
|
||
let mut i = 0;
|
||
while let Some(&b) = field.get(i) {
|
||
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
|
||
i += 2;
|
||
continue;
|
||
}
|
||
if b == b'/' {
|
||
slash = Some(i);
|
||
break;
|
||
}
|
||
i += 1;
|
||
}
|
||
let (text, flags) = match slash {
|
||
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
|
||
None => (field, &[][..]),
|
||
};
|
||
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
|
||
out.push_str(&decode_latin1(flags));
|
||
Ok(out)
|
||
}
|
||
|
||
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
|
||
/// its parts is the original).
|
||
fn fields(line: &[u8]) -> Vec<&[u8]> {
|
||
let mut out = Vec::new();
|
||
let mut start = 0;
|
||
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
|
||
for (i, b) in line.iter().enumerate() {
|
||
let sep = matches!(b, b' ' | b'\t');
|
||
if sep != in_sep {
|
||
out.push(line.get(start..i).unwrap_or_default());
|
||
start = i;
|
||
in_sep = sep;
|
||
}
|
||
}
|
||
out.push(line.get(start..).unwrap_or_default());
|
||
out
|
||
}
|
||
|
||
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
|
||
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
|
||
let mut out = String::with_capacity(aff.len());
|
||
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
|
||
if n > 0 {
|
||
out.push('\n');
|
||
}
|
||
if let Ok(s) = std::str::from_utf8(line)
|
||
&& s.is_ascii()
|
||
{
|
||
out.push_str(s);
|
||
continue;
|
||
}
|
||
let parts = fields(line);
|
||
// Field index among the non-separator parts.
|
||
let words: Vec<usize> =
|
||
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
|
||
let dir = words
|
||
.first()
|
||
.and_then(|&i| parts.get(i))
|
||
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
|
||
.unwrap_or_default();
|
||
let what = || format!(".aff line {}", n + 1);
|
||
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
|
||
out.push_str(&String::from_utf8_lossy(line));
|
||
continue;
|
||
}
|
||
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
|
||
// a flag; from there on the line is kept as text. It can only start after the directive's
|
||
// own fields, since a field such as an affix condition may itself begin with `#`.
|
||
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
|
||
let first_free = comment_field(&dir, field_text);
|
||
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
|
||
// The field index of the next non-separator part.
|
||
let mut next_field = 0;
|
||
for (i, p) in parts.iter().enumerate() {
|
||
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
|
||
next_field += 1;
|
||
next_field - 1
|
||
});
|
||
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
|
||
let decoded = match (dir.as_str(), field) {
|
||
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
|
||
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
|
||
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
|
||
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
|
||
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
|
||
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
|
||
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
|
||
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
|
||
};
|
||
out.push_str(&decoded);
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
|
||
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
|
||
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
|
||
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
|
||
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
|
||
match dir {
|
||
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
|
||
"PFX" | "SFX" => 5,
|
||
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
|
||
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
|
||
{
|
||
2
|
||
}
|
||
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
|
||
"CHECKCOMPOUNDPATTERN" => 4,
|
||
_ => 2,
|
||
}
|
||
}
|
||
|
||
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
|
||
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
|
||
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
|
||
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
|
||
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
|
||
let blank = |b: &u8| matches!(b, b' ' | b'\t');
|
||
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
|
||
let tab = line.iter().position(|&b| b == b'\t');
|
||
let end = match (morph, tab) {
|
||
(Some(m), Some(t)) => m.min(t),
|
||
(m, t) => match m.or(t) {
|
||
Some(e) => e,
|
||
None => line.len(),
|
||
},
|
||
};
|
||
// Blanks before the morphological field or the line end aren't part of the entry.
|
||
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
|
||
}
|
||
|
||
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
|
||
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
|
||
let mut out = String::with_capacity(dic.len());
|
||
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
|
||
if n > 0 {
|
||
out.push('\n');
|
||
}
|
||
if let Ok(s) = std::str::from_utf8(line)
|
||
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
|
||
{
|
||
out.push_str(s);
|
||
continue;
|
||
}
|
||
let what = || format!(".dic line {}", n + 1);
|
||
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
|
||
// morphological fields after it are UTF-8 text.
|
||
let end = dic_entry_end(line);
|
||
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
|
||
let rest = line.get(end..).unwrap_or_default();
|
||
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// CodeRabbit on craft-libs#489: without LibreOffice extension names no drive is probed; with
|
||
/// them, only drives where LibreOffice is found give folders.
|
||
#[test]
|
||
fn libreoffice_drives_are_probed_only_when_wanted() {
|
||
let probes = std::cell::Cell::new(0);
|
||
let count = |_: &Path| {
|
||
probes.set(probes.get() + 1);
|
||
false
|
||
};
|
||
// craft-libs#495 review: the drives are probed once, not on every lookup.
|
||
let cache = std::sync::OnceLock::new();
|
||
let cached = || cache.get_or_init(|| lo_drive_bases(count)).as_slice();
|
||
assert!(lo_drive_dirs(None, cached).is_empty());
|
||
assert_eq!(probes.get(), 0);
|
||
let exts = ["dict-tr".to_string()];
|
||
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
|
||
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
|
||
assert_eq!(probes.get(), 48);
|
||
let found = lo_drive_bases(|p: &Path| p.to_string_lossy().starts_with(r"D:\Program Files\"));
|
||
assert_eq!(
|
||
lo_drive_dirs(Some(&exts), || found.as_slice()),
|
||
[PathBuf::from(r"D:\Program Files\LibreOffice\share\extensions").join("dict-tr")]
|
||
);
|
||
}
|
||
|
||
/// Another region's Language pack never beats the system's dictionary of the right region
|
||
/// (`pt-pt` pack vs system `pt_BR`; `de-ch` vs `de_DE`; `en-gb` vs `en_US`), but beats the
|
||
/// system's other names; the right region's pack beats the system; the dictionaries folder
|
||
/// beats everything.
|
||
#[test]
|
||
fn another_regions_pack_comes_after_the_systems_exact_region() {
|
||
let root = std::env::temp_dir().join(format!("wc-proof-order-{}", std::process::id()));
|
||
let _ = std::fs::remove_dir_all(&root);
|
||
let pair = |dir: &Path, name: &str| {
|
||
std::fs::create_dir_all(dir).unwrap();
|
||
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
|
||
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
|
||
dir.join(format!("{name}.dic"))
|
||
};
|
||
let (folder, system) = (root.join("folder"), root.join("system"));
|
||
let en: &[&str] = &["en_US", "en-US", "en", "en_CA", "en_GB", "en-GB", "en_AU"];
|
||
for (names, exact, other, cross) in [
|
||
(&["pt_BR", "pt-BR", "pt"][..], "pt_BR", "pt", "pt-pt"),
|
||
(&["de_DE", "de-DE", "de"][..], "de_DE", "de", "de-ch"),
|
||
(&["nl_NL", "nl-NL", "nl"][..], "nl_NL", "nl", "nl-be"),
|
||
(&["es_ES", "es-ES", "es_ANY", "es"][..], "es_ES", "es", "es-mx"),
|
||
(en, "en_US", "en_GB", "en-gb"),
|
||
] {
|
||
let sys_exact = pair(&system, exact);
|
||
let sys_other = pair(&system, other);
|
||
let cross_pack = pair(&root.join("packs").join(cross), crate::packs::SPELLING);
|
||
let dics = |v: Vec<(PathBuf, PathBuf)>| v.into_iter().map(|(_, d)| d).collect::<Vec<_>>();
|
||
let got =
|
||
dics(ordered(names, &[], &[], std::slice::from_ref(&system), std::slice::from_ref(&cross_pack.parent().unwrap().to_path_buf())));
|
||
assert_eq!(got, [sys_exact.clone(), cross_pack.clone(), sys_other.clone()], "{names:?}");
|
||
// The region's own pack first, then the system's exact region.
|
||
let region_pack = pair(&root.join("packs").join(format!("{}-region", names[0])), crate::packs::SPELLING);
|
||
let got = dics(ordered(names, &[], &[region_pack.parent().unwrap().to_path_buf()], std::slice::from_ref(&system), &[]));
|
||
assert_eq!(got.first(), Some(®ion_pack), "{names:?}");
|
||
assert_eq!(got.get(1), Some(&sys_exact), "{names:?}");
|
||
// The dictionaries folder before all of them, whatever name.
|
||
let mine = pair(&folder, other);
|
||
let got = dics(ordered(
|
||
names,
|
||
std::slice::from_ref(&folder),
|
||
&[region_pack.parent().unwrap().to_path_buf()],
|
||
std::slice::from_ref(&system),
|
||
&[],
|
||
));
|
||
assert_eq!(got.first(), Some(&mine), "{names:?}");
|
||
assert_eq!(find_in(&folder, &[other]).map(|(_, d)| d), Some(mine));
|
||
let _ = std::fs::remove_dir_all(&root);
|
||
}
|
||
}
|
||
|
||
/// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link
|
||
/// (swapped in after the pack scan) is never a candidate; one in a system folder still is.
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn pack_files_are_never_read_through_a_link() {
|
||
let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id()));
|
||
let _ = std::fs::remove_dir_all(&root);
|
||
let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere"));
|
||
for d in [&pack, &system, &elsewhere] {
|
||
std::fs::create_dir_all(d).unwrap();
|
||
}
|
||
std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap();
|
||
std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap();
|
||
std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap();
|
||
std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap();
|
||
std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap();
|
||
std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap();
|
||
let got: Vec<PathBuf> =
|
||
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
|
||
assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never");
|
||
std::fs::remove_file(pack.join("spelling.dic")).unwrap();
|
||
std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap();
|
||
let got: Vec<PathBuf> =
|
||
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
|
||
assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]);
|
||
let _ = std::fs::remove_dir_all(&root);
|
||
}
|
||
|
||
/// WordCraft's own dictionaries folder under the user's application data is searched on every
|
||
/// desktop system, after the setting and before the shared system locations (#1063).
|
||
#[test]
|
||
fn the_app_folder_comes_before_the_system_locations() {
|
||
if cfg!(target_arch = "wasm32") {
|
||
return;
|
||
}
|
||
let user = if cfg!(windows) { "LOCALAPPDATA" } else { "HOME" };
|
||
if std::env::var_os(user).filter(|v| !v.is_empty()).is_none() {
|
||
return; // no user folder to derive it from
|
||
}
|
||
let dirs = search_dirs(None);
|
||
let at = |suffix: &str| dirs.iter().position(|d| d.ends_with(suffix));
|
||
let (app, shared) = if cfg!(windows) {
|
||
("WordCraft/dictionaries", "hunspell")
|
||
} else if cfg!(target_os = "macos") {
|
||
("Application Support/WordCraft/dictionaries", "/Library/Spelling")
|
||
} else {
|
||
("wordcraft/dictionaries", "/usr/share/hunspell")
|
||
};
|
||
assert!(matches!((at(app), at(shared)), (Some(app), Some(shared)) if app < shared), "{dirs:?}");
|
||
}
|
||
|
||
/// `$XDG_DATA_HOME` counts only as an absolute path; otherwise (unset, empty or relative) the
|
||
/// spec's default `~/.local/share` applies, and with no home there is no folder.
|
||
#[test]
|
||
fn xdg_data_home_follows_the_spec() {
|
||
let home = Path::new("/home/a");
|
||
assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data")));
|
||
assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data")));
|
||
for xdg in [None, Some(""), Some("relative/data")] {
|
||
assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}");
|
||
assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn decodes_latin2_and_utf8() {
|
||
let aff = b"SET ISO8859-2\nTRY a\n";
|
||
let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap();
|
||
assert!(a.contains("TRY a"));
|
||
assert_eq!(d, "2\nżółw\n");
|
||
let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
|
||
assert_eq!(d, "żółw");
|
||
let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap();
|
||
assert_eq!(d, "ĞİŞğışÖç");
|
||
assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
|
||
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
|
||
}
|
||
|
||
/// #2830: a dictionary that is there but can't be read leaves its reason behind; a good
|
||
/// load or no file clears it.
|
||
#[test]
|
||
fn load_error_is_remembered_until_the_next_good_load() {
|
||
let key = "test_unreadable_2830";
|
||
assert_eq!(load_error(key), None);
|
||
let dic = Path::new("/dicts/test.dic");
|
||
assert!(note_load(key, dic, decode(b"SET SHIFT_JIS\n", b"")).is_err());
|
||
let (file, reason) = load_error(key).expect("error kept");
|
||
assert_eq!(file, dic);
|
||
assert_eq!(reason, "unsupported encoding SHIFT_JIS");
|
||
assert!(note_load(key, dic, decode(b"SET UTF-8\n", b"")).is_ok());
|
||
assert_eq!(load_error(key), None);
|
||
let _ = note_load::<()>(key, dic, Err("too large".into()));
|
||
clear_load_error(key);
|
||
assert_eq!(load_error(key), None);
|
||
}
|
||
|
||
/// Every table covers its 96 (ISO 8859) or 128 (KOI8-R, Windows-1251) high bytes.
|
||
#[test]
|
||
fn tables_are_complete() {
|
||
for t in [ISO_8859_2_HIGH, ISO_8859_7_HIGH, ISO_8859_15_HIGH] {
|
||
assert_eq!(t.chars().count(), 96, "{t}");
|
||
}
|
||
for t in [KOI8_R_HIGH, CP1251_HIGH] {
|
||
assert_eq!(t.chars().count(), 128, "{t}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn decodes_greek_and_latin9() {
|
||
// `σπίτι νερό` in ISO 8859-7, and a flag after the slash stays ASCII.
|
||
let (a, d) = decode(b"SET ISO8859-7\nTRY \xe1\n", b"2\n\xf3\xf0\xdf\xf4\xe9/A\n\xed\xe5\xf1\xfc\n").unwrap();
|
||
assert!(a.contains("TRY α"), "{a}");
|
||
assert_eq!(d, "2\nσπίτι/A\nνερό\n");
|
||
// Unassigned bytes and the 2003 additions.
|
||
let (_, d) = decode(b"SET ISO-8859-7\n", b"\xae\xd2\xff\xa4\xa5\xaa\xa2").unwrap();
|
||
assert_eq!(d, "\u{fffd}\u{fffd}\u{fffd}€₯ͺ’");
|
||
// `šokolaad õun` in ISO 8859-15, and its eight differences from ISO 8859-1.
|
||
let (_, d) = decode(b"SET ISO8859-15\n", b"\xa8okolaad \xf5un").unwrap();
|
||
assert_eq!(d, "šokolaad õun");
|
||
let (_, d) = decode(b"SET ISO-8859-15\n", b"\xa4\xa6\xa8\xb4\xb8\xbc\xbd\xbe\xe4").unwrap();
|
||
assert_eq!(d, "€ŠšŽžŒœŸä");
|
||
}
|
||
|
||
#[test]
|
||
fn decodes_latin1() {
|
||
// `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1.
|
||
let (a, d) = decode(b"SET ISO8859-1\nTRY \xe7\n", b"1\nor\xe7amento/A\nesper\xe1vamos\n").unwrap();
|
||
assert!(a.contains("TRY ç"), "{a}");
|
||
assert_eq!(d, "1\norçamento/A\nesperávamos\n");
|
||
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
|
||
}
|
||
|
||
#[test]
|
||
fn utf8_dictionaries_keep_single_byte_flags() {
|
||
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
|
||
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
|
||
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
|
||
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
|
||
let (a, d) = decode(aff, dic).unwrap();
|
||
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
|
||
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
|
||
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
|
||
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
|
||
assert!(!dict.check("házban"));
|
||
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
|
||
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||
assert_eq!(d, "kert/\u{e9}\n");
|
||
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
|
||
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
|
||
}
|
||
|
||
#[test]
|
||
fn single_flag_fields_keep_their_first_byte() {
|
||
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
|
||
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
|
||
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
|
||
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
|
||
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
|
||
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
|
||
assert!(d.contains("kert/\u{c3}\u{a9}"));
|
||
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
|
||
assert!(dict.check("kert") && dict.check("kertok"));
|
||
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
|
||
}
|
||
|
||
#[test]
|
||
fn numeric_and_long_flags_stay_text() {
|
||
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
|
||
// number with a non-ASCII rule or comment keeps every digit.
|
||
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
|
||
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
|
||
assert_eq!(a, aff);
|
||
assert_eq!(d, "1\nkitap/1234\n");
|
||
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
|
||
assert!(tr.contains("kitapları") && tr.contains("kitap"));
|
||
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
|
||
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
|
||
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
|
||
// `FLAG DEFAULT` is the single-byte default.
|
||
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
|
||
assert_eq!(d, "1\nkert/\u{e9}\n");
|
||
}
|
||
|
||
#[test]
|
||
fn flag_utf8_counts_anywhere_in_the_aff() {
|
||
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
|
||
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
|
||
for _ in 0..400 {
|
||
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
|
||
}
|
||
aff.extend_from_slice(b"FLAG UTF-8\r\n");
|
||
assert!(aff.len() > 8192);
|
||
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
|
||
assert_eq!(d, "kert/\u{e9}\n");
|
||
}
|
||
|
||
#[test]
|
||
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
|
||
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||
assert_eq!(d, "kert/\u{e9}\n");
|
||
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
|
||
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
|
||
}
|
||
|
||
#[test]
|
||
fn decodes_russian_single_byte_encodings() {
|
||
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
|
||
let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap();
|
||
assert!(a.contains("TRY ое"), "{a}");
|
||
assert_eq!(d, "1\nёлка/A\n");
|
||
let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap();
|
||
assert_eq!(d, "1\nёлка Дом\n");
|
||
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
|
||
}
|
||
|
||
#[test]
|
||
fn comments_start_after_the_directive_fields() {
|
||
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
|
||
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
|
||
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
|
||
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
|
||
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
|
||
assert_eq!(comment_field("REP", f("REP 12")), 2);
|
||
assert_eq!(comment_field("REP", f("REP # x")), 3);
|
||
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
|
||
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
|
||
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
|
||
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
|
||
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
|
||
}
|
||
|
||
#[test]
|
||
fn dic_entries_end_where_hunspell_ends_them() {
|
||
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
|
||
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
|
||
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
|
||
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
|
||
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
|
||
assert_eq!(end("ad hoc/B "), "ad hoc/B");
|
||
assert_eq!(end("word"), "word");
|
||
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
|
||
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
|
||
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
|
||
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
|
||
// Without a tab or a morphological field the whole line is the entry: its byte flags
|
||
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
|
||
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
|
||
assert_eq!(d, "1\nad hoc/\u{e9}\n");
|
||
}
|
||
|
||
#[test]
|
||
fn libreoffice_extension_dirs_follow_the_language() {
|
||
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese
|
||
// `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried.
|
||
assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]);
|
||
assert_eq!(super::lo_ext_dirs("id_ID"), ["dict-id", "dict-id-ID"]);
|
||
assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);
|
||
assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]);
|
||
assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]);
|
||
let dirs = search_dirs(Some(&super::lo_ext_dirs("pt_BR")));
|
||
assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}");
|
||
assert!(search_dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess");
|
||
}
|
||
|
||
/// craft-libs#482 re-review: a name is looked for in every folder before the next name, so
|
||
/// an `en_US` in the dictionaries folder doesn't beat a system `en_GB` for British English.
|
||
#[test]
|
||
fn candidates_go_name_by_name_across_the_folders() {
|
||
let base = std::env::temp_dir().join(format!("wordcraft-candidates-{}", std::process::id()));
|
||
let (folder, system) = (base.join("folder"), base.join("system"));
|
||
for (dir, name) in [(&folder, "en_US"), (&system, "en_GB"), (&folder, "en-GB")] {
|
||
std::fs::create_dir_all(dir).unwrap();
|
||
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
|
||
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
|
||
}
|
||
let dirs = [folder.clone(), system.clone()];
|
||
let dics: Vec<PathBuf> = candidates_in(&dirs, &["en_GB", "en-GB"]).into_iter().map(|(_, d)| d).collect();
|
||
assert_eq!(dics, vec![system.join("en_GB.dic"), folder.join("en-GB.dic")]);
|
||
let us: Vec<PathBuf> = candidates_in(&dirs, &["en_US", "en_GB"]).into_iter().map(|(_, d)| d).collect();
|
||
assert_eq!(us, vec![folder.join("en_US.dic"), system.join("en_GB.dic")]);
|
||
assert!(candidates_in(&dirs, &["en_CA"]).is_empty());
|
||
let _ = std::fs::remove_dir_all(&base);
|
||
}
|
||
|
||
#[test]
|
||
fn finds_and_loads_a_pair_in_a_directory() {
|
||
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-{}", std::process::id()));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found");
|
||
std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap();
|
||
assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary");
|
||
std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap();
|
||
let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap();
|
||
let d = load(&aff, &dic).unwrap();
|
||
assert!(d.check("kot") && d.check("koty") && !d.check("kotx"));
|
||
assert!(load(&dir.join("missing.aff"), &dic).is_err());
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
#[test]
|
||
fn reads_are_bounded_and_regular_files_only() {
|
||
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id()));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
let aff = dir.join("xx.aff");
|
||
std::fs::write(&aff, "SET UTF-8\n").unwrap();
|
||
// One byte over the cap (sparse: nothing is written) is refused without reading it.
|
||
let big = dir.join("big.dic");
|
||
std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap();
|
||
assert!(load(&aff, &big).unwrap_err().contains("too large"));
|
||
// A directory, and on Unix an endless device, are not dictionary files. (No clock: a
|
||
// reader that read `/dev/zero` would never return, so the error itself is the proof.)
|
||
assert!(load(&aff, &dir).unwrap_err().contains("not a regular file"));
|
||
#[cfg(unix)]
|
||
assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file"));
|
||
// Turkish reads through the same reader.
|
||
let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
|
||
std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap();
|
||
std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap();
|
||
assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar"));
|
||
assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large"));
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
/// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from
|
||
/// disk through the capped reader and decoded per its `SET` line: its words with Turkish
|
||
/// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the
|
||
/// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not.
|
||
#[test]
|
||
fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() {
|
||
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id()));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
|
||
std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap();
|
||
// ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş.
|
||
let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n";
|
||
std::fs::write(&dic, words).unwrap();
|
||
assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8");
|
||
let d = crate::tr::load(&aff, &dic).unwrap();
|
||
for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] {
|
||
assert!(d.contains(w), "{w}");
|
||
}
|
||
// The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those.
|
||
assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir"));
|
||
assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them");
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
}
|