diff --git a/crates/i18n/src/fold.rs b/crates/i18n/src/fold.rs index 8d2e256b..959c6425 100644 --- a/crates/i18n/src/fold.rs +++ b/crates/i18n/src/fold.rs @@ -191,6 +191,23 @@ fn continues_syllable(prev: HangulSyllableType, next: HangulSyllableType) -> boo ) } +/// For each prefix of `n`, the length of its longest proper prefix that is also its suffix: the +/// Knuth-Morris-Pratt failure links. +fn prefix_links(n: &[u8]) -> Vec { + let mut links = vec![0; n.len()]; + let mut k = 0; + for i in 1..n.len() { + while k > 0 && n[k] != n[i] { + k = links[k - 1]; + } + if n[k] == n[i] { + k += 1; + } + links[i] = k; + } + links +} + /// Folded text and where each part came from. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Folded { @@ -244,8 +261,13 @@ impl Folded { let inside = self.starts.binary_search_by_key(&at, |&(f, _)| f).is_err(); inside && self.text[..at].chars().next_back().is_some_and(|p| continues_syllable(hangul.get(p), hangul.get(c))) }; - let n = needle.text.as_str(); - let mut from = 0; + // Every occurrence, overlapping ones included, is found in one left-to-right pass, so a + // run of occurrences turned down at a mark doesn't search the needle again from each + // one. Both texts are UTF-8, so a byte match starts and ends on character boundaries. + let n = needle.text.as_bytes(); + let hay = self.text.as_bytes(); + let links = prefix_links(n); + let (mut at, mut k) = (0, 0); let mut done = false; std::iter::from_fn(move || { if done { @@ -255,14 +277,22 @@ impl Folded { done = true; return Some(0..0); } - while let Some(i) = self.text.get(from..).and_then(|t| t.find(n)) { - let start = from + i; - let end = start + n.len(); - if !starts_mark(start) && !starts_mark(end) { - from = end; - return Some(self.source_range(start..end)); + while let Some(&b) = hay.get(at) { + while k > 0 && n[k] != b { + k = links[k - 1]; + } + if n[k] == b { + k += 1; + } + at += 1; + if k == n.len() { + let (start, end) = (at - k, at); + if !starts_mark(start) && !starts_mark(end) { + k = 0; + return Some(self.source_range(start..end)); + } + k = links[k - 1]; } - from = start + self.text[start..].chars().next().map_or(1, char::len_utf8); } done = true; None @@ -351,6 +381,23 @@ mod tests { let all: Vec<_> = f.fold("Ab ab AB").matches(&f.fold("ab")).collect(); assert_eq!(all, [0..2, 3..5, 6..8]); + let all: Vec<_> = f.fold("aaaaa").matches(&f.fold("aa")).collect(); + assert_eq!(all, [0..2, 2..4]); + assert_eq!(find("abababc", "ababc"), Some("ababc")); + } + + #[test] + fn occurrences_turned_down_at_a_mark_are_not_searched_again() { + // Every occurrence of the needle ends just before an accent, except the one at the end: + // each is turned down, and the next one overlaps it. + let f = Folder::new("en").keep_accents(); + let haystack = format!("{}a", "a\u{301}".repeat(100_000)); + let needle = format!("{}a", "a\u{301}".repeat(50_000)); + let folded = f.fold(&haystack); + let start = std::time::Instant::now(); + let found = folded.find(&f.fold(&needle)); + assert_eq!(found, Some(haystack.len() - needle.len()..haystack.len())); + assert!(start.elapsed() < std::time::Duration::from_secs(5), "{:?}", start.elapsed()); } #[test] diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 17b9eaa3..b6d6984f 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -471,18 +471,24 @@ fn wrap(initiator: char, value: &str) -> String { out.push(initiator); // One entry per open initiator: true for an isolate, false for an embedding or override. let mut open: Vec = Vec::new(); + // How many of `open` are isolates, so a PDI needn't search the stack for one. + let mut isolates = 0usize; for c in value.chars() { match c { - LRI | RLI | FSI => open.push(true), + LRI | RLI | FSI => { + open.push(true); + isolates += 1; + } LRE | RLE | LRO | RLO => open.push(false), PDF if open.last() == Some(&false) => { open.pop(); } PDF => continue, - PDI if open.contains(&true) => { + PDI if isolates > 0 => { while open.pop() == Some(false) { out.push(PDF); } + isolates -= 1; } PDI => continue, _ => {} @@ -580,8 +586,17 @@ fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String { /// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. A region that /// implies a script (CLDR likely subtags) counts as that script after the tag as written: /// `pa-PK` → `pa-pk`, `pa-arab-pk`, `pa-arab`. +/// A locale tag is no longer than this: a language, script, region and a few variants or a +/// glibc encoding and modifier fit with room to spare. A longer one is no locale, and isn't +/// split into a fallback for every subtag. +const MAX_TAG_LEN: usize = 128; + fn candidates(tag: &str) -> Vec { - let tag = tag.trim().to_ascii_lowercase(); + let tag = tag.trim(); + if tag.len() > MAX_TAG_LEN { + return Vec::new(); + } + let tag = tag.to_ascii_lowercase(); let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); let mut parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); let Some(&primary) = parts.first() else { @@ -843,6 +858,12 @@ mod tests { assert_eq!(fmt("ש {n}", &[("n", &one)]), format!("ש {one}")); // Two isolates side by side are not one, so they get wrapped. assert_eq!(fmt("ש{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "ש\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); + // A PDI closes the embeddings opened after its isolate, and one with no isolate open is + // dropped, however many embeddings are open. + assert_eq!(isolate("\u{2066}\u{202A}a\u{2069}b"), "\u{2068}\u{2066}\u{202A}a\u{202C}\u{2069}b\u{2069}"); + let embeddings = "\u{202A}".repeat(50_000); + let wrapped = isolate(&format!("{embeddings}{}", "\u{2069}".repeat(50_000))); + assert_eq!(wrapped, format!("\u{2068}{embeddings}{}\u{2069}", "\u{202C}".repeat(50_000))); } /// `text` in display order, as one line in a left-to-right paragraph, without bidi controls. @@ -1033,6 +1054,9 @@ mod tests { assert_eq!(candidates("kk-CN"), ["kk-cn", "kk-arab-cn", "kk-arab"]); assert_eq!(candidates("kk-KZ"), ["kk-kz", "kk"]); assert_eq!(candidates("ug-KZ"), ["ug-kz", "ug-cyrl-kz", "ug-cyrl"]); + // A tag too long to be a locale has no fallbacks, rather than one per subtag. + assert!(candidates(&"x-".repeat(500_000)).is_empty()); + assert_eq!(candidates(&format!("en-{}", "x-".repeat(60))).len(), 61); } #[test]