diff --git a/crates/text/examples/hyphgen.rs b/crates/text/examples/hyphgen.rs index 520cee91..9a494b95 100644 --- a/crates/text/examples/hyphgen.rs +++ b/crates/text/examples/hyphgen.rs @@ -100,7 +100,7 @@ fn main() { println!("{entries} entries → {} words (+{} from compounds)", list.len(), extra.len()); list.extend(extra); let dict = Dictionary::from_entries(list); - let bytes = dict.to_bytes(); + let bytes = dict.to_bytes().expect("dictionary within the inflate limit"); std::fs::create_dir_all(&out_dir).expect("mkdir"); std::fs::write(out_dir.join("en-us.dic"), &bytes).expect("write en-us.dic"); let back = Dictionary::from_bytes(&bytes).expect("roundtrip"); diff --git a/crates/text/src/hyphen/dict.rs b/crates/text/src/hyphen/dict.rs index 28b71a77..ac198b50 100644 --- a/crates/text/src/hyphen/dict.rs +++ b/crates/text/src/hyphen/dict.rs @@ -49,7 +49,8 @@ impl Dictionary { /// Deserialize data written by `Dictionary::to_bytes`. pub(crate) fn decode(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, MAX_INFLATED) + .map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); @@ -90,7 +91,7 @@ impl Dictionary { } /// Serialize (deflate-compressed). - pub fn to_bytes(&self) -> Vec { + pub fn to_bytes(&self) -> Result, String> { let mut raw = MAGIC.to_vec(); let mut prev: Vec = Vec::new(); for (w, m) in self.iter() { @@ -101,7 +102,11 @@ impl Dictionary { raw.push(b'\n'); prev = h; } - miniz_oxide::deflate::compress_to_vec(&raw, 10) + // decode refuses anything past MAX_INFLATED, so refuse to write it + if raw.len() > MAX_INFLATED { + return Err(format!("hyphenation dictionary: {} bytes exceeds the {MAX_INFLATED}-byte limit", raw.len())); + } + Ok(miniz_oxide::deflate::compress_to_vec(&raw, 10)) } /// Deserialize data written by [`Dictionary::to_bytes`]. @@ -110,6 +115,11 @@ impl Dictionary { } } +/// The most bytes a dictionary may inflate to. The bundled US English dictionary inflates to +/// under 2 MiB; a small stream must not be able to ask for the whole of memory before the +/// header is checked. +const MAX_INFLATED: usize = 64 << 20; + /// `word` with `-` before every char whose bit is set in `mask`. #[cfg(any(test, feature = "generator"))] pub fn hyphenated(word: &str, mask: u64) -> String { @@ -149,11 +159,18 @@ mod tests { fn roundtrip() { let e: Vec<(String, u64)> = ["ty-pog-ra-phy", "cat", "hap-pen", "o'clock", "naïve-té", "naïve"].iter().map(|s| parse_hyphenated(s)).collect(); let d = Dictionary::from_entries(e); - let back = Dictionary::from_bytes(&d.to_bytes()).unwrap(); + let back = Dictionary::from_bytes(&d.to_bytes().unwrap()).unwrap(); assert_eq!(back.len(), 6); assert_eq!(back.get("typography").map(|m| hyphenated("typography", m)), Some("ty-pog-ra-phy".to_string())); assert_eq!(back.get("cat"), Some(0)); assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string())); assert!(back.get("dog").is_none()); } + + #[test] + fn an_oversized_stream_is_refused_before_parsing() { + let bomb = miniz_oxide::deflate::compress_to_vec(&vec![b'a'; MAX_INFLATED + 1], 10); + assert!(bomb.len() < 1 << 20, "{} bytes compressed", bomb.len()); + assert!(Dictionary::from_bytes(&bomb).unwrap_err().contains("inflate failed")); + } } diff --git a/crates/text/src/hyphen/mod.rs b/crates/text/src/hyphen/mod.rs index 76d1416f..fa0ec4e9 100644 --- a/crates/text/src/hyphen/mod.rs +++ b/crates/text/src/hyphen/mod.rs @@ -427,10 +427,12 @@ fn char_points(word: &str, lim: &Limits, lang: Lang) -> Vec { /// `word` with `-` at every hyphenation point (tests and diagnostics). pub fn hyphenate_word(word: &str, lim: &Limits) -> String { - let pts = hyphen_points(word, lim); + let mut pts = hyphen_points(word, lim); + pts.sort_unstable(); + let mut next = pts.iter().peekable(); let mut s = String::with_capacity(word.len() + pts.len()); for (i, ch) in word.char_indices() { - if pts.contains(&i) { + if next.next_if_eq(&&i).is_some() { s.push('-'); } s.push(ch); diff --git a/crates/text/src/hyphen/patgen.rs b/crates/text/src/hyphen/patgen.rs index eea88b2a..84d9bb36 100644 --- a/crates/text/src/hyphen/patgen.rs +++ b/crates/text/src/hyphen/patgen.rs @@ -178,8 +178,16 @@ pub fn score(pats: &Patterns, words: &[(Vec, Vec)]) -> Score { let Some(pts) = pats.points(w) else { continue }; s.words += 1; s.breaks += br.len(); + // Indexed once per word: `pts` are in `1..w.len()`, so no expected break past the word + // can be found anyway. + let mut expected = vec![false; w.len() + 1]; + for &b in br { + if let Some(e) = expected.get_mut(b) { + *e = true; + } + } for p in &pts { - if br.contains(p) { + if expected.get(*p).copied().unwrap_or(false) { s.found += 1; } else { s.wrong += 1; diff --git a/crates/text/src/hyphen/patterns.rs b/crates/text/src/hyphen/patterns.rs index a5dff954..21e4aff4 100644 --- a/crates/text/src/hyphen/patterns.rs +++ b/crates/text/src/hyphen/patterns.rs @@ -7,6 +7,10 @@ use std::collections::HashMap; use std::hash::{BuildHasherDefault, Hasher}; +/// The most patterns one set holds; later ones are ignored. The bundled US English set has +/// about 7,000. +pub(crate) const MAX_PATTERNS: usize = 1 << 20; + /// Word-boundary marker in patterns. pub(crate) const DOT: char = '.'; /// Longest pattern the packed key supports. @@ -17,9 +21,6 @@ pub(crate) const MAX_PATTERN_LEN: usize = 9; pub(crate) struct FxHasher(u64); impl Hasher for FxHasher { - fn finish(&self) -> u64 { - self.0 - } fn write(&mut self, bytes: &[u8]) { for &b in bytes { self.write_u64(b as u64); @@ -28,6 +29,12 @@ impl Hasher for FxHasher { fn write_u64(&mut self, i: u64) { self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95); } + fn finish(&self) -> u64 { + // The table indexes by the low bits, which a multiplication leaves depending on the + // low bits of the key alone: packed patterns that share their first letters would + // share a bucket. Rotating brings the well-mixed high bits down (as rustc-hash 2 does). + self.0.rotate_left(26) + } } pub(crate) type FxMap = HashMap>; @@ -123,8 +130,12 @@ impl Patterns { if codes.is_empty() || codes.len() > MAX_PATTERN_LEN { return false; } + let key = pack(&codes); + if self.map.len() >= MAX_PATTERNS && !self.map.contains_key(&key) { + return false; + } self.max_len = self.max_len.max(codes.len()); - self.map.insert(pack(&codes), vals.into_boxed_slice()); + self.map.insert(key, vals.into_boxed_slice()); true } @@ -238,4 +249,27 @@ mod tests { assert!(tex.contains(&"hen5at".to_string())); assert!(p.values(&['h', 'é']).is_none()); } + + #[test] + fn packed_keys_sharing_a_prefix_spread_over_the_buckets() { + // 100,000 patterns whose packed keys differ in the last letter codes only. With the low + // bits of a multiplicative hash left as they are, they probe the same buckets. + let letters: Vec = ('a'..='z').collect(); + let mut text = String::new(); + for i in 0..100_000u32 { + let mut n = i; + text.push_str("p1r"); + for _ in 0..4 { + text.push(letters[(n % 26) as usize]); + n /= 26; + } + text.push(' '); + } + let started = std::time::Instant::now(); + let p = Patterns::parse(&text); + assert_eq!(p.map.len(), 100_000); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + let w: Vec = "praaaa".chars().collect(); + assert_eq!(p.points(&w).unwrap(), vec![1]); + } } diff --git a/crates/text/src/linebreak/mod.rs b/crates/text/src/linebreak/mod.rs index d34d50b2..9d20b064 100644 --- a/crates/text/src/linebreak/mod.rs +++ b/crates/text/src/linebreak/mod.rs @@ -419,6 +419,9 @@ struct Sums<'a> { z: Vec, over: Vec, fill: Vec, + /// After item `k`: the first box or forced break, or the item count. Looked up for a break + /// at `k`, so it is found once per item rather than by a scan per candidate. + next: Vec, } impl<'a> Sums<'a> { @@ -450,7 +453,12 @@ impl<'a> Sums<'a> { z.push(sz); over.push(so); fill.push(sf); - Sums { items, w, y, z, over, fill } + let mut next = vec![m; m + 1]; + for b in (0..m).rev() { + let i = b + 1; + next[b] = if items.get(i).is_some_and(|it| it.kind == Kind::Box || it.is_forced()) { i } else { next[i] }; + } + Sums { items, w, y, z, over, fill, next } } fn len(&self) -> usize { @@ -481,14 +489,7 @@ impl<'a> Sums<'a> { /// First item of the line after a break at `b`: glue and penalties are discarded up to the next /// box or forced break. fn next_start(&self, b: usize) -> usize { - let mut i = b + 1; - while let Some(it) = self.items.get(i) { - if it.kind == Kind::Box || it.is_forced() { - break; - } - i += 1; - } - i.min(self.len()) + self.next.get(b).copied().unwrap_or(self.len()) } fn at(&self, k: usize) -> (f64, Tiers, Tiers, f64, u32) { diff --git a/crates/text/src/linebreak/tests.rs b/crates/text/src/linebreak/tests.rs index a9cc150b..d8163edd 100644 --- a/crates/text/src/linebreak/tests.rs +++ b/crates/text/src/linebreak/tests.rs @@ -474,3 +474,21 @@ fn overfull_lines_keep_the_hyphen_limit() { assert!(lines.windows(2).all(|w| !(w[0].flagged && w[1].flagged)), "{lines:?}"); assert_eq!(spans(&lines), [(0, 1), (2, 5)]); } + +#[test] +fn next_start_is_the_first_box_or_forced_break_after_each_item() { + // Boxes, glue, penalties and forced breaks in a fixed mix; the table matches a scan. + let items: Vec = (0..2_000) + .map(|i| match i % 7 { + 0 | 3 => Item::boxed(CHAR), + 1 => Item::glue(SPACE, 1.0, 1.0), + 6 if i % 3 == 0 => Item::forced(), + _ => Item::penalty(HYPHEN, 50.0, true), + }) + .collect(); + let s = Sums::new(&items); + for b in 0..=items.len() { + let scan = (b + 1..items.len()).find(|&i| items[i].kind == Kind::Box || items[i].is_forced()).unwrap_or(items.len()); + assert_eq!(s.next_start(b), scan, "after {b}"); + } +}