From 74b0eab93a6bb0421e3ec0702fb511c0ead6756e Mon Sep 17 00:00:00 2001 From: Ufuk Efe Date: Wed, 14 Oct 2026 21:45:30 +0300 Subject: [PATCH] proof: Turkish spelling, suggestions, grammar and hyphenation (storytold/wordcraft#1527) * proof: Turkish spelling, suggestions, grammar and hyphenation * Turkish: dictionary tests skip when no Turkish Hunspell dictionary is installed Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Brandon Thomas Co-authored-by: Claude Opus 5.5 --- crates/proof/src/hunspell.rs | 10 + crates/proof/src/hyphen.rs | 3 + crates/proof/src/lang.rs | 19 +- crates/proof/src/lib.rs | 23 ++- crates/proof/src/tr/dict.rs | 129 ++++++++++++ crates/proof/src/tr/grammar.rs | 352 +++++++++++++++++++++++++++++++++ crates/proof/src/tr/hyphen.rs | 98 +++++++++ crates/proof/src/tr/mod.rs | 156 +++++++++++++++ crates/proof/src/tr/suggest.rs | 298 ++++++++++++++++++++++++++++ 9 files changed, 1079 insertions(+), 9 deletions(-) create mode 100644 crates/proof/src/tr/dict.rs create mode 100644 crates/proof/src/tr/grammar.rs create mode 100644 crates/proof/src/tr/hyphen.rs create mode 100644 crates/proof/src/tr/mod.rs create mode 100644 crates/proof/src/tr/suggest.rs diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 113420843..c596d4607 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -66,6 +66,16 @@ pub fn search_dirs(lo_exts: Option<&[String]>) -> Vec { dirs.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy())); } } + // If LibreOffice is installed on another drive (e.g. D:\Program Files\LibreOffice), check drives C..Z + for drive in b'C'..=b'Z' { + let drive_char = drive as char; + for pf in ["Program Files", "Program Files (x86)"] { + let lo_base = PathBuf::from(format!(r"{drive_char}:\{pf}\LibreOffice\share\extensions")); + if lo_base.is_dir() { + dirs.extend(lo_dirs(&lo_base.to_string_lossy())); + } + } + } } else if cfg!(target_os = "macos") { if let Some(h) = &home { dirs.push(h.join("Library/Spelling")); diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index c8a17d32f..913017b6d 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -127,6 +127,9 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) { return (crate::bg::points(core), Source::Rules); } + if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) { + return (crate::tr::hyphen::points(core), Source::Rules); + } if lang == ProofLang::Pt { return core_breaks_pt(core); } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 48ef51aa8..62075c9ed 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -26,6 +26,9 @@ pub enum ProofLang { It, /// Croatian (`hr`, `hr-HR`) Hr, + /// Turkish (`tr`, `tr-TR`, `tr-CY`): the installed Hunspell dictionary (when there is one), + /// Turkish grammar rules, spelling suggestions and Turkish syllable hyphenation rules. + Tr, /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without /// the language's proofing tools: no spelling or grammar marks, so its words are never /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before @@ -35,7 +38,7 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 5] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr]; + pub const PROOFED: [ProofLang; 6] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr]; /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag /// counts. No tag (or an empty one) is English, WordCraft's default editing language. @@ -54,12 +57,14 @@ impl ProofLang { ProofLang::It } else if primary.eq_ignore_ascii_case("hr") || primary.eq_ignore_ascii_case("hrv") { ProofLang::Hr + } else if primary.eq_ignore_ascii_case("tr") || primary.eq_ignore_ascii_case("tur") { + ProofLang::Tr } else { ProofLang::Other } } - /// The primary language subtag (`en`, `pl`, `bg`); empty for [`ProofLang::Other`]. + /// The primary language subtag (`en`, `pl`, `bg`, `tr`); empty for [`ProofLang::Other`]. pub fn code(self) -> &'static str { match self { ProofLang::En => "en", @@ -68,6 +73,7 @@ impl ProofLang { ProofLang::Pt => "pt", ProofLang::It => "it", ProofLang::Hr => "hr", + ProofLang::Tr => "tr", ProofLang::Other => "", } } @@ -153,7 +159,10 @@ mod tests { for t in ["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Bg, "{t}"); } - for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "e", "-"] { + for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}"); + } + for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "e", "-"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); } assert_eq!(ProofLang::from_tag(None), ProofLang::En); @@ -164,10 +173,14 @@ mod tests { assert!(!ProofLang::Bg.is_proofed()); assert_eq!(ProofLang::Bg.code(), "bg"); assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed()); + assert!(ProofLang::Tr.is_proofed()); assert_eq!(ProofLang::Pl.code(), "pl"); assert_eq!(ProofLang::Pt.code(), "pt"); assert!(ProofLang::It.is_proofed()); assert_eq!(ProofLang::It.code(), "it"); + assert!(ProofLang::Hr.is_proofed()); + assert_eq!(ProofLang::Hr.code(), "hr"); + assert_eq!(ProofLang::Tr.code(), "tr"); assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed"); } } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 6b34a4a5d..d644b62f9 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -42,6 +42,7 @@ pub mod lang; pub mod patterns; pub mod pl; pub mod pt; +pub mod tr; use std::collections::HashSet; use std::ops::Range; @@ -129,9 +130,10 @@ pub fn dictionary_generation() -> u64 { DICTIONARY_GENERATION.load(Ordering::Relaxed) } -/// Is `lower` (a lower-case word) in the user dictionary? +/// Is `lower` (a lower-case word) in the user dictionary or ignored? pub(crate) fn in_user_dictionary(lower: &str) -> bool { user_words().read().unwrap_or_else(|e| e.into_inner()).contains(lower) + || ignored_words().read().unwrap_or_else(|e| e.into_inner()).contains(lower) } /// Words in the user dictionary. @@ -533,6 +535,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option ProofLang::Hr => "Moguća pravopisna pogreška", ProofLang::Pt => "Possível erro de ortografia", ProofLang::It => "Possibile errore di ortografia", + ProofLang::Tr => "Olası yazım hatası", _ => "Possible spelling mistake", }; Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) @@ -551,6 +554,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct(word), ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct(word), ProofLang::It => accepted_by_options(word, opts) || it::is_correct(word), + ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word), ProofLang::Bg | ProofLang::Other => true, } } @@ -563,13 +567,14 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { ProofLang::Hr => hr::suggest(word, max), ProofLang::Pt => pt::suggest(word, max), ProofLang::It => it::suggest(word, max), + ProofLang::Tr => tr::suggest(word, max), ProofLang::Bg | ProofLang::Other => Vec::new(), } } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones /// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]), -/// or none for a language WordCraft doesn't proof. +/// the Turkish ones ([`tr::grammar`]), or none for a language WordCraft doesn't proof. pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { match lang { ProofLang::En => check_grammar(text), @@ -577,6 +582,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { ProofLang::Hr => hr::grammar::check(text), ProofLang::Pt => pt::grammar::check(text), ProofLang::It => it::grammar::check(text), + ProofLang::Tr => tr::grammar::check(text), ProofLang::Bg | ProofLang::Other => Vec::new(), } } @@ -1162,7 +1168,7 @@ mod tests { let _ = has_dictionary(Some(&s)); let _ = is_correct_for(&s, Some(&s)); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); - for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Other] { + for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Other] { let _ = check_spelling_in(&s, lang); let _ = check_grammar_in(&s, lang); let _ = is_correct_in(&s, lang); @@ -1183,9 +1189,14 @@ mod tests { } #[test] - fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..4), 0..6)) { - let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::Other]; - let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % 4])).collect(); + fn turkish_suggestions_never_panic(s in "[a-zçğıöşüA-ZÇĞİÖŞÜ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::Tr); + } + + #[test] + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..5), 0..6)) { + let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::Tr, ProofLang::Other]; + let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % 5])).collect(); for i in check_text(&s, &spans) { proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); } diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs new file mode 100644 index 000000000..ed57cbca7 --- /dev/null +++ b/crates/proof/src/tr/dict.rs @@ -0,0 +1,129 @@ +//! Turkish Hunspell dictionary parser with `FLAG num` support. +//! +//! Turkish uses `FLAG num`, storing thousands of numerical suffix flags (0..6465). +//! This engine parses `/usr/share/hunspell/tr_TR.aff` and `.dic` safely without panics. + +use std::collections::HashMap; + +/// A suffix rule for Turkish. +#[derive(Clone, Debug)] +pub struct TrSuffixRule { + pub strip: Box, + pub add: Box, + pub cond: Box, +} + +/// Dedicated Turkish Hunspell dictionary engine. +#[derive(Clone, Debug, Default)] +pub struct TrDict { + /// Stem -> list of numerical flags it carries. + stems: HashMap>, + /// Flag number -> list of suffix rules. + rules: HashMap>, + /// Suffix 'add' string -> list of (flag, rule_index). + suffix_index: HashMap>, + /// Longest suffix 'add' string in characters. + max_suffix_len: usize, +} + +impl TrDict { + /// Returns true if empty (no dictionary loaded). + pub fn is_empty(&self) -> bool { + self.stems.is_empty() + } + + /// Load from `.aff` and `.dic` contents. + pub fn from_hunspell(aff: &str, dic: &str) -> Result { + let mut rules: HashMap> = HashMap::new(); + let mut max_suffix_len = 0; + + for line in aff.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + // SFX [] + if parts.len() >= 4 && parts[0] == "SFX" { + let Ok(flag) = parts[1].parse::() else { continue }; + // Skip header line: `SFX Y/N ` + if parts.len() == 4 && (parts[2] == "Y" || parts[2] == "N") && parts[3].parse::().is_ok() { + continue; + } + let strip = if parts[2] == "0" { "" } else { parts[2] }; + let add = if parts[3] == "0" { "" } else { parts[3] }; + let cond = if parts.len() > 4 { parts[4] } else { "." }; + + max_suffix_len = max_suffix_len.max(add.chars().count()); + + rules.entry(flag).or_default().push(TrSuffixRule { strip: strip.into(), add: add.into(), cond: cond.into() }); + } + } + + let mut suffix_index: HashMap> = HashMap::new(); + for (&flag, r_list) in &rules { + for (idx, r) in r_list.iter().enumerate() { + if let Ok(idx_u16) = u16::try_from(idx) { + suffix_index.entry(r.add.to_string()).or_default().push((flag, idx_u16)); + } + } + } + + let mut stems: HashMap> = HashMap::new(); + for (i, line) in dic.lines().enumerate() { + if i == 0 && line.trim().parse::().is_ok() { + continue; + } + let line = line.trim(); + if line.is_empty() { + continue; + } + if let Some((word, flags_part)) = line.split_once('/') { + let flags: Vec = flags_part.split(',').filter_map(|f| f.parse::().ok()).collect(); + stems.insert(word.to_string(), flags.into_boxed_slice()); + } else { + stems.insert(line.to_string(), Box::new([])); + } + } + + Ok(TrDict { stems, rules, suffix_index, max_suffix_len }) + } + + /// Check if word is correct (stem or affixed). + pub fn contains(&self, word: &str) -> bool { + if self.is_empty() || word.is_empty() { + return false; + } + + // Direct stem hit + if self.stems.contains_key(word) { + return true; + } + + // Try stripping suffixes matching the tail + let word_len = word.chars().count(); + let char_indices: Vec<(usize, char)> = word.char_indices().collect(); + + // Check each possible suffix length up to max_suffix_len + for cut_chars in 0..=self.max_suffix_len.min(word_len) { + let byte_idx = if cut_chars == 0 { word.len() } else { char_indices[word_len - cut_chars].0 }; + + let add = &word[byte_idx..]; + let base = &word[..byte_idx]; + + if let Some(matches) = self.suffix_index.get(add) { + for &(flag, rule_idx) in matches { + let Some(rule_list) = self.rules.get(&flag) else { continue }; + let Some(rule) = rule_list.get(rule_idx as usize) else { continue }; + + // Check condition on root + let mut candidate = String::with_capacity(base.len() + rule.strip.len()); + candidate.push_str(base); + candidate.push_str(&rule.strip); + + if self.stems.get(&candidate).is_some_and(|f_list| f_list.contains(&flag)) { + return true; + } + } + } + } + + false + } +} diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs new file mode 100644 index 000000000..f0598a17d --- /dev/null +++ b/crates/proof/src/tr/grammar.rs @@ -0,0 +1,352 @@ +//! Turkish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure. +//! Messages are in Turkish. +//! +//! - **Repeated word** (`çok çok`), except idiomatic reduplications and doublings (`ALLOWED_REPEATS`: +//! `yavaş yavaş`, `tıkır tıkır`, `adım adım`). +//! - **Space before punctuation** (`ev ,`) and **double spaces**. +//! - **Capital letter at the start of a sentence**, knowing Turkish abbreviations (`vb.`, `vs.`, +//! `prof.`, `dr.`, `mad.`, `yy.`, `s.`…) after which lowercase or numbers may continue. +//! - **No comma before/after conjunctions**: `ve`, `veya`, `yahut`, `ile`, `de`, `da`, `hem ... hem`, +//! `ne ... ne` conjunctions should not have commas immediately before or after them. +//! - **Doubled punctuation** (`..`, `,,`, `;;`). + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Turkish reduplications and emphatic word pairings that are intentionally repeated. +const ALLOWED_REPEATS: &[&str] = &[ + "yavaş", + "hızlı", + "koşa", + "güle", + "adım", + "kara", + "tıkır", + "şakır", + "akın", + "ağır", + "açık", + "ayrı", + "bire", + "bir", + "boş", + "bölük", + "buram", + "cana", + "can", + "cılk", + "çifter", + "çit", + "dalga", + "damla", + "deste", + "didik", + "dizi", + "dolu", + "deste", + "döne", + "eğri", + "enine", + "epey", + "fıkır", + "fırıl", + "gece", + "gıcır", + "gide", + "günü", + "güzel", + "halk", + "harıl", + "hık", + "hım", + "horul", + "ılık", + "ışıl", + "için", + "içten", + "iğne", + "iri", + "iyiden", + "iyi", + "kaba", + "kaç", + "kat", + "kaynar", + "kelli", + "kese", + "kıtır", + "kıvrım", + "koca", + "kol", + "kucak", + "kütür", + "mırıl", + "öbek", + "parça", + "patır", + "pıtır", + "sabah", + "salına", + "salkım", + "sap", + "sara", + "sarı", + "serin", + "sıra", + "sırım", + "sıkı", + "sırılsıklam", + "tane", + "teker", + "tek", + "tıpış", + "top", + "tutam", + "uzun", + "üçer", + "üst", + "yalan", + "yana", + "yan", + "yarım", + "yer", + "yudum", + "yığın", + "yürekten", + "yüz", + "zaman", + "zor", +]; + +/// Turkish abbreviations that end with a period and after which lower case may continue. +const ABBREVIATIONS: &[&str] = &[ + "vb", "vs", "vd", "bkz", "bknz", "ör", "örnek", "mad", "md", "s", "sf", "sn", "dk", "sa", "yy", "m", "km", "cm", "mm", "gr", "kg", "mg", "dr", + "prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "a.ş", "ltd", "şti", "t.c", + "tbmm", "meb", "möt", "mö", "ms", +]; + +/// Conjunctions that must not be preceded by a comma. +const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"]; + +/// Conjunctions that must not be followed by a comma. +const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "ne", "ya", "gerek", "ister", "de", "da"]; + +/// Check grammar and punctuation in a Turkish text block. +pub fn check(text: &str) -> Vec { + let mut v = Vec::new(); + let ws = words(text); + + // 1. Repeated words (`kelime kelime`) + for pair in ws.windows(2) { + let (w1, w2) = match (text.get(pair[0].0..pair[0].1), text.get(pair[1].0..pair[1].1)) { + (Some(a), Some(b)) => (a, b), + _ => continue, + }; + let gap = match text.get(pair[0].1..pair[1].0) { + Some(g) => g, + None => continue, + }; + let only_spaces = gap.chars().all(|c| c == ' ' || c == '\t'); + let lower1 = w1.to_lowercase(); + let lower2 = w2.to_lowercase(); + if only_spaces && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) { + v.push(Issue { + start: pair[1].0, + end: pair[1].1, + kind: IssueKind::Grammar, + message: "Yinelenen sözcük".into(), + suggestions: Vec::new(), + lang: ProofLang::Tr, + }); + } + } + + // 2. Spaces before punctuation and double spaces + let chars: Vec<(usize, char)> = text.char_indices().collect(); + for i in 0..chars.len() { + let (idx, c) = chars[i]; + if matches!(c, ',' | '.' | '?' | '!' | ';' | ':') && i > 0 && chars[i - 1].1 == ' ' { + let prev_sp = chars[i - 1].0; + // Leave emoticons and ellipses alone + let is_dot_in_dots = c == '.' && (i + 1 < chars.len() && chars[i + 1].1 == '.'); + if !is_dot_in_dots { + v.push(Issue { + start: prev_sp, + end: idx + c.len_utf8(), + kind: IssueKind::Grammar, + message: "Noktalama işaretinden önce boşluk bırakılmaz".into(), + suggestions: vec![c.to_string()], + lang: ProofLang::Tr, + }); + } + } + // Double space + if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { + let mut end = idx + 1; + while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { + end += 1; + } + v.push(Issue { + start: idx, + end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } + } + + // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) + for (i, &(w_start, w_end)) in ws.iter().enumerate() { + let word = match text.get(w_start..w_end) { + Some(w) => w.to_lowercase(), + None => continue, + }; + // Check comma before conjunction + if CONJUNCTIONS_NO_PRE_COMMA.contains(&word.as_str()) + && i > 0 + && let Some(between) = text.get(ws[i - 1].1..w_start) + && let Some(comma_pos) = between.find(',') + { + let actual_pos = ws[i - 1].1 + comma_pos; + v.push(Issue { + start: actual_pos, + end: actual_pos + 1, + kind: IssueKind::Grammar, + message: format!("'{word}' bağlacından önce virgül konulmaz"), + suggestions: vec![String::new()], + lang: ProofLang::Tr, + }); + } + // Check comma after conjunction + if CONJUNCTIONS_NO_POST_COMMA.contains(&word.as_str()) + && i + 1 < ws.len() + && let Some(between) = text.get(w_end..ws[i + 1].0) + && let Some(comma_pos) = between.find(',') + { + let actual_pos = w_end + comma_pos; + v.push(Issue { + start: actual_pos, + end: actual_pos + 1, + kind: IssueKind::Grammar, + message: format!("'{word}' bağlacından sonra virgül konulmaz"), + suggestions: vec![String::new()], + lang: ProofLang::Tr, + }); + } + } + + // 4. Sentence capitalization knowing Turkish abbreviations + for (i, &(w_start, w_end)) in ws.iter().enumerate() { + if i == 0 { + // First word in paragraph + if let Some(w) = text.get(w_start..w_end) + && w.chars().next().is_some_and(char::is_lowercase) + { + let cap = turkish_capitalize(w); + v.push(Issue { + start: w_start, + end: w_end, + kind: IssueKind::Grammar, + message: "Cümle büyük harfle başlar".into(), + suggestions: vec![cap], + lang: ProofLang::Tr, + }); + } + continue; + } + + // Check if previous character sequence ended a sentence + let prev_end = ws[i - 1].1; + let prev_word = text.get(ws[i - 1].0..prev_end).unwrap_or(""); + if let Some(gap) = text.get(prev_end..w_start) { + let trimmed = gap.trim(); + if (trimmed.ends_with('.') || trimmed.ends_with('?') || trimmed.ends_with('!')) + && !trimmed.contains("...") + && !ABBREVIATIONS.contains(&prev_word.to_lowercase().as_str()) + && let Some(w) = text.get(w_start..w_end) + && w.chars().next().is_some_and(char::is_lowercase) + { + let cap = turkish_capitalize(w); + v.push(Issue { + start: w_start, + end: w_end, + kind: IssueKind::Grammar, + message: "Cümle büyük harfle başlar".into(), + suggestions: vec![cap], + lang: ProofLang::Tr, + }); + } + } + } + + v.sort_by_key(|issue| (issue.start, issue.end)); + v +} + +/// Capitalize according to Turkish rules (`i` -> `İ`, `ı` -> `I`). +pub fn turkish_capitalize(s: &str) -> String { + let mut chars = s.chars(); + let Some(first) = chars.next() else { return String::new() }; + let first_cap = match first { + 'i' => 'İ', + 'ı' => 'I', + other => other.to_uppercase().next().unwrap_or(other), + }; + let mut out = String::with_capacity(s.len() + 1); + out.push(first_cap); + out.push_str(chars.as_str()); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clean_turkish_text_produces_no_issues() { + let text = "Bu bir test cümlesidir. Kitap ve defter aldım. Çocuk yavaş yavaş yürüdü."; + assert!(check(text).is_empty()); + } + + #[test] + fn repeated_words_flagged_except_allowed() { + let text = "Gitti gitti ama bulamadı."; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert_eq!(issues[0].message, "Yinelenen sözcük"); + + // Allowed repeat + let text2 = "Çocuk yavaş yavaş ilerliyordu."; + assert!(check(text2).is_empty()); + } + + #[test] + fn space_before_punctuation_flagged() { + let text = "Merhaba , dünya!"; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert_eq!(issues[0].message, "Noktalama işaretinden önce boşluk bırakılmaz"); + } + + #[test] + fn comma_around_conjunctions_flagged() { + let text = "Elma, ve armut."; + let issues = check(text); + assert_eq!(issues.len(), 1); + assert!(issues[0].message.contains("virgül konulmaz")); + + let text2 = "Bana kalırsa bu mesele de, haklı olan taraf bizdik."; + let issues2 = check(text2); + assert_eq!(issues2.len(), 1); + assert!(issues2[0].message.contains("'de' bağlacından sonra virgül konulmaz")); + } + + #[test] + fn sentence_capitalization_respects_abbreviations() { + let text = "Dr. ahmet geldi. Fakat geç kaldı."; + let issues = check(text); + // "Dr. ahmet" does not flag "ahmet" because "Dr." is in ABBREVIATIONS. + assert!(issues.is_empty()); + } +} diff --git a/crates/proof/src/tr/hyphen.rs b/crates/proof/src/tr/hyphen.rs new file mode 100644 index 000000000..a8a481088 --- /dev/null +++ b/crates/proof/src/tr/hyphen.rs @@ -0,0 +1,98 @@ +//! Turkish hyphenation by rule ([`ProofLang::Tr`](crate::ProofLang::Tr)). +//! +//! Turkish words are split at syllables according to Turkish Language Association (TDK) rules. +//! Because Turkish syllable boundaries are fully regular, they can be determined directly by rule +//! without needing external pattern files or word lists: +//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü` (both lowercase and uppercase). +//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `ö-ğren-ci`). +//! - Of two consonants between two vowels, the first stays with the preceding syllable and the +//! second joins the next (`alt-lık`, `sev-gi`, `tür-kçe`). +//! - Of three consonants between two vowels, the first two stay with the preceding syllable and the +//! third joins the next (`türk-çe`, `sürp-riz`). +//! - In compound words, syllable division follows pronunciation across the boundary (`ba-şöğ-ret-men`, +//! `il-ko-kul`), which matches the general vowel/consonant distribution rules. +//! +//! A single letter left at the end or start of a line is ruled out by the paragraph's +//! [`Limits`](crate::hyphen::Limits). + +/// Is `c` a Turkish vowel? +pub fn is_vowel(c: char) -> bool { + matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü') +} + +/// Is `c` a Turkish letter? +pub(crate) fn is_letter(c: char) -> bool { + c.is_alphabetic() +} + +/// Break points of a Turkish word (char indices; a hyphen goes before that char). +/// Empty for words with fewer than two vowels or containing non-alphabetic characters. +pub fn points(word: &[char]) -> Vec { + if word.len() < 2 || !word.iter().all(|&c| is_letter(c)) { + return Vec::new(); + } + let vowels: Vec = (0..word.len()).filter(|&i| is_vowel(word[i])).collect(); + if vowels.len() < 2 { + return Vec::new(); + } + let mut out = Vec::new(); + for pair in vowels.windows(2) { + let (v1, v2) = (pair[0], pair[1]); + if v2 <= v1 { + continue; + } + let num_consonants = v2 - v1 - 1; + let pt = match num_consonants { + 0 => v1 + 1, + 1 => v2 - 1, + 2 => v1 + 2, + _ => v2 - 1, + }; + if pt > v1 && pt <= v2 { + out.push(pt); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn split(w: &str) -> String { + let chars: Vec = w.chars().collect(); + let pts = points(&chars); + let mut out = String::new(); + for (i, c) in chars.iter().enumerate() { + if pts.contains(&i) { + out.push('-'); + } + out.push(*c); + } + out + } + + #[test] + fn syllables_follow_turkish_rules() { + assert_eq!(split("balık"), "ba-lık"); + assert_eq!(split("araba"), "a-ra-ba"); + assert_eq!(split("öğrenci"), "öğ-ren-ci"); + assert_eq!(split("altlık"), "alt-lık"); + assert_eq!(split("sevgi"), "sev-gi"); + assert_eq!(split("türkçe"), "türk-çe"); + assert_eq!(split("saat"), "sa-at"); + assert_eq!(split("şiir"), "şi-ir"); + assert_eq!(split("ilkokul"), "il-ko-kul"); + assert_eq!(split("başöğretmen"), "ba-şöğ-ret-men"); + } + + #[test] + fn short_or_non_words_stay_unsplit() { + assert_eq!(split(""), ""); + assert_eq!(split("a"), "a"); + assert_eq!(split("ve"), "ve"); + assert_eq!(split("Türk"), "Türk"); + assert_eq!(split("123"), "123"); + assert_eq!(split("araba-ev"), "araba-ev"); + } +} diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs new file mode 100644 index 000000000..e0b6fab9c --- /dev/null +++ b/crates/proof/src/tr/mod.rs @@ -0,0 +1,156 @@ +//! Turkish proofing. +//! +//! - **Spelling** against the Hunspell `tr_TR` dictionary installed on the system +//! ([`dictionary`], found by [`crate::hunspell`]: `tr_TR.aff`/`tr_TR.dic` from the OS, a +//! package manager or LibreOffice; it is not bundled). Without it Turkish text gets no spelling +//! marks. Capitalized stems are proper nouns (`Ankara`); foreign names inflected with an apostrophe +//! are accepted (`WordCraft'ta`), and the user dictionary applies as in English. +//! - **Suggestions** tuned for Turkish ([`suggest`]): ASCII-diacritic restoration (`turkce` -> `Türkçe`), +//! common confusions (`yalnış` -> `yanlış`), detached particles (`geldimi` -> `geldi mi`), and typos. +//! - **Grammar** ([`grammar`]): high-precision rules (reduplications, comma around conjunctions, +//! sentence start with abbreviations). +//! - **Hyphenation** ([`hyphen`]): syllable division by Turkish Language Association (TDK) rules. + +pub mod dict; +pub mod grammar; +pub mod hyphen; +pub mod suggest; + +use std::sync::{Arc, RwLock}; + +pub use dict::TrDict; +pub use suggest::suggest; + +/// The Turkish dictionary: `None` until first use, then the installed one (or an empty one). +static DICT: RwLock>> = RwLock::new(None); + +/// File names of a Turkish Hunspell dictionary. +const NAMES: &[&str] = &["tr_TR", "tr-TR", "tr"]; + +/// The Turkish dictionary: the installed Hunspell `tr_TR` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available. +pub fn dictionary() -> Arc { + if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + return d.clone(); + } + let loaded = Arc::new(load_installed()); + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + slot.get_or_insert(loaded).clone() +} + +fn load_installed() -> TrDict { + let Some((aff, dic)) = crate::hunspell::find(NAMES) else { + log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); + return TrDict::default(); + }; + let read = |p: &std::path::Path| std::fs::read_to_string(p); + match (read(&aff), read(&dic)) { + (Ok(aff_text), Ok(dic_text)) => TrDict::from_hunspell(&aff_text, &dic_text).unwrap_or_else(|e| { + log::warn!("Turkish spelling: {e}"); + TrDict::default() + }), + _ => TrDict::default(), + } +} + +/// Use this Turkish dictionary from now on. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + let d = TrDict::from_hunspell(aff, dic)?; + *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + Ok(()) +} + +/// Reload the dictionary on next check. +pub fn reload_dictionary() { + *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; +} + +/// Is a Turkish dictionary available? +pub fn has_dictionary() -> bool { + !dictionary().is_empty() +} + +/// Is `word` spelled correctly in Turkish? +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] against a given dictionary. +pub fn is_correct_with(d: &TrDict, word: &str) -> bool { + if d.is_empty() { + return true; // No Turkish dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + return true; + } + // Check user dictionary first + if crate::in_user_dictionary(&suggest::turkish_to_lower(w)) || crate::in_user_dictionary(&w.to_lowercase()) { + return true; + } + + // Direct lookup or lowercase lookup + if d.contains(w) { + return true; + } + let lower = suggest::turkish_to_lower(w); + if d.contains(&lower) { + return true; + } + + // Handle apostrophe for proper nouns (e.g., `Ankara'da`, `WordCraft'ta`) + if let Some((stem, _suffix)) = w.split_once(['\'', '’']) + && (d.contains(stem) || d.contains(&suggest::turkish_to_lower(stem))) + { + return true; + } + + // Compound words separated by hyphens (e.g., `afyon-karahisar`) + if w.contains('-') && w.split('-').all(|part| part.is_empty() || is_correct_with(d, part)) { + return true; + } + + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn without_dictionary_everything_is_correct() { + let empty = TrDict::default(); + assert!(is_correct_with(&empty, "herhangi")); + assert!(is_correct_with(&empty, "xyzabc")); + } + + #[test] + fn apostrophe_and_compounds_are_handled() { + let empty = TrDict::default(); + assert!(is_correct_with(&empty, "Ankara'da")); + assert!(is_correct_with(&empty, "açık-mavi")); + } + + #[test] + fn installed_dictionary_checks_words_if_present() { + // Machines without a Turkish Hunspell dictionary (CI, most desktops) have nothing to check. + if !has_dictionary() { + return; + } + assert!(is_correct("kitap")); + assert!(is_correct("kitaplar")); + assert!(is_correct("evde")); + assert!(is_correct("olacaklardan")); + assert!(is_correct("sonra")); + assert!(!is_correct("sorna")); + assert!(!is_correct("kitapppp")); + assert!(!is_correct("evdekişşş")); + + // Test user dictionary + assert!(!is_correct("benimbilmedigimkelime")); + crate::add_word("benimbilmedigimkelime"); + assert!(is_correct("benimbilmedigimkelime")); + } +} diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs new file mode 100644 index 000000000..dedc1bb26 --- /dev/null +++ b/crates/proof/src/tr/suggest.rs @@ -0,0 +1,298 @@ +//! Turkish spelling suggestions, ranked by likelihood. +//! +//! Candidates are generated and ranked by cost: +//! 1. Capitalization / case adjustments (especially dotted/dotless I: `istanbul` -> `İstanbul`). +//! 2. Missing Turkish diacritics / ASCII substitutions (most frequent Turkish typing slip): +//! `c` <-> `ç`, `g` <-> `ğ`, `i` <-> `ı`, `o` <-> `ö`, `s` <-> `ş`, `u` <-> `ü` (`turkce` -> `Türkçe`). +//! 3. Common spelling mistakes and phonetic confusions (`REP` table): +//! `yalnış` -> `yanlış`, `yanlız` -> `yalnız`, `herşey` -> `her şey`, `bir kaç` -> `birkaç`. +//! 4. Missing space for detached particles and conjunctions (`bende` -> `ben de`, `geldinmi` -> `geldin mi`). +//! 5. Single typo (Damerau-Levenshtein distance <= 2). + +use std::collections::HashSet; + +use crate::tr::dict::TrDict; + +/// Common misspellings and their canonical replacements in Turkish. +const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ + ("yalnış", "yanlış"), + ("yanlız", "yalnız"), + ("herşey", "her şey"), + ("herkez", "herkes"), + ("bir kaç", "birkaç"), + ("birşey", "bir şey"), + ("tabiki", "tabii ki"), + ("orjinal", "orijinal"), + ("kiprik", "kirpik"), + ("şöför", "şoför"), + ("sarmısak", "sarımsak"), + ("makina", "makine"), + ("ünvan", "unvan"), + ("rastlantı", "rastlantı"), + ("labaratuvar", "laboratuvar"), + ("laboratuar", "laboratuvar"), + ("kravat", "kravat"), + ("gardolap", "gardırop"), + ("egzoz", "egzoz"), + ("egzozt", "egzoz"), + ("egsoz", "egzoz"), + ("gelcen", "geleceksin"), + ("gelecen", "geleceksin"), + ("gitcen", "gideceksin"), + ("gidecen", "gideceksin"), + ("yapcan", "yapacaksın"), + ("yapacan", "yapacaksın"), + ("edecen", "edeceksin"), + ("deyil", "değil"), +]; + +/// Produce spelling suggestions for a misspelled Turkish word. +pub fn suggest(word: &str, max: usize) -> Vec { + let dict = super::dictionary(); + suggest_with(&dict, word, max) +} + +/// Suggest with a given dictionary. +pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { + if word.is_empty() || max == 0 { + return Vec::new(); + } + + let mut suggestions = Vec::new(); + let mut seen = HashSet::new(); + + let lower = turkish_to_lower(word); + + // 1. Check direct common replacement table + for &(wrong, right) in COMMON_REPLACEMENTS { + if lower == wrong && seen.insert(right.to_string()) { + suggestions.push(right.to_string()); + if suggestions.len() >= max { + return suggestions; + } + } + } + + // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`) + if !d.is_empty() { + let variants = generate_diacritic_variants(&lower); + for cand in variants { + if cand != lower && d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + + // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { + let stem = &lower[..lower.len() - suffix.len()]; + if d.is_empty() || d.contains(stem) { + let cand = format!("{stem} {suffix}"); + if seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 4. Single typo (transposition, deletion, insertion, substitution) + if !d.is_empty() { + let chars: Vec = lower.chars().collect(); + let n = chars.len(); + + // 4a. Transposition (adjacent swap: `sorna` -> `sonra`) + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut swapped = chars.clone(); + swapped.swap(i, i + 1); + let cand: String = swapped.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + + // Turkish alphabet for insertions and substitutions + const TR_ALPHABET: &[char] = &[ + 'a', 'b', 'c', 'ç', 'd', 'e', 'f', 'g', 'ğ', 'h', 'ı', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'ö', 'p', 'r', 's', 'ş', 't', 'u', 'ü', 'v', + 'y', 'z', + ]; + + // 4b. Deletion (one character left out: `kitapp` -> `kitap`) + for i in 0..n { + let mut deleted = chars.clone(); + deleted.remove(i); + let cand: String = deleted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + + // 4c. Substitution (one wrong letter: `yolculuk` -> ...) + for i in 0..n { + for &c in TR_ALPHABET { + if chars.get(i) != Some(&c) { + let mut substituted = chars.clone(); + if let Some(target) = substituted.get_mut(i) { + *target = c; + } + let cand: String = substituted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 4d. Insertion (one missing letter: `kitp` -> `kitap`) + for i in 0..=n { + for &c in TR_ALPHABET { + let mut inserted = chars.clone(); + inserted.insert(i, c); + let cand: String = inserted.into_iter().collect(); + if d.contains(&cand) && seen.insert(cand.clone()) { + suggestions.push(match_casing(word, &cand)); + if suggestions.len() >= max { + return suggestions; + } + } + } + } + } + + // 5. Fallback: capitalization + let cap = super::grammar::turkish_capitalize(&lower); + if cap != word && (d.is_empty() || d.contains(&cap)) && seen.insert(cap.clone()) { + suggestions.push(cap); + } + + suggestions +} + +/// Convert ASCII letters to Turkish diacritic equivalents. +fn generate_diacritic_variants(s: &str) -> Vec { + let mut results = vec![String::new()]; + for c in s.chars() { + let (alt1, alt2) = match c { + 'c' => ('c', Some('ç')), + 'ç' => ('ç', Some('c')), + 'g' => ('g', Some('ğ')), + 'ğ' => ('ğ', Some('g')), + 'i' => ('i', Some('ı')), + 'ı' => ('ı', Some('i')), + 'o' => ('o', Some('ö')), + 'ö' => ('ö', Some('o')), + 's' => ('s', Some('ş')), + 'ş' => ('ş', Some('s')), + 'u' => ('u', Some('ü')), + 'ü' => ('ü', Some('u')), + other => (other, None), + }; + + let branch_mult = if alt2.is_some() { 2 } else { 1 }; + if results.len() * branch_mult > 1024 { + // Cap search branch explosion + for res in &mut results { + res.push(c); + } + continue; + } + + let mut next_gen = Vec::with_capacity(results.len() * branch_mult); + for existing in &results { + let mut s1 = existing.clone(); + s1.push(alt1); + next_gen.push(s1); + if let Some(c2) = alt2 { + let mut s2 = existing.clone(); + s2.push(c2); + next_gen.push(s2); + } + } + results = next_gen; + } + results +} + +/// Turkish lowercase with dotted/dotless I support. +pub fn turkish_to_lower(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + 'I' => out.push('ı'), + 'İ' => out.push('i'), + other => { + for lc in other.to_lowercase() { + out.push(lc); + } + } + } + } + out +} + +/// Match original casing on suggestion. +fn match_casing(orig: &str, cand: &str) -> String { + if orig.chars().all(char::is_uppercase) { + cand.to_uppercase() + } else if orig.chars().next().is_some_and(char::is_uppercase) { + super::grammar::turkish_capitalize(cand) + } else { + cand.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn common_replacements_are_suggested() { + let sugs = suggest("yalnış", 3); + assert!(sugs.contains(&"yanlış".to_string())); + + let sugs2 = suggest("herkez", 3); + assert!(sugs2.contains(&"herkes".to_string())); + } + + #[test] + fn detached_suffix_is_suggested() { + let sugs = suggest("geldinmi", 3); + assert!(sugs.contains(&"geldin mi".to_string())); + } + + #[test] + fn turkish_to_lower_handles_i() { + assert_eq!(turkish_to_lower("IŞIK"), "ışık"); + assert_eq!(turkish_to_lower("İSTANBUL"), "istanbul"); + } + + #[test] + fn typo_transposition_is_suggested() { + // Suggestions come from the installed Turkish dictionary; without one there are none. + if !super::super::has_dictionary() { + return; + } + let sugs = suggest("sorna", 3); + assert!(sugs.contains(&"sonra".to_string()), "sorna should suggest sonra, got {sugs:?}"); + + let sugs_gelcen = suggest("gelcen", 3); + assert!(sugs_gelcen.contains(&"geleceksin".to_string()), "gelcen should suggest geleceksin, got {sugs_gelcen:?}"); + } +}