text: keep hyphenation and line-breaking work linear and bounded (#560)

* text: keep hyphenation and line-breaking work linear and bounded - hyphen::dict: inflate with a 64 MiB limit (MAX_INFLATED). - hyphenate_word: walk the sorted break offsets with one iterator. - patgen::score: index expected breaks once per word. - patterns: FxHasher's finish rotates the multiplied bits down so packed keys sharing low bits spread over buckets; at most MAX_PATTERNS (1M). - linebreak: Sums precomputes each item's next box or forced break, so next_start is a lookup instead of a scan per candidate. * address review: refuse to serialize hyphenation dictionaries past the inflate limit
2026-11-07 12:13:20 +00:00 · 2026-11-03 23:04:30 -05:00
parent cb82b90580
commit d8568d6cfc
7 changed files with 101 additions and 21 deletions
@@ -100,7 +100,7 @@ fn main() {
    println!("{entries} entries → {} words (+{} from compounds)", list.len(), extra.len());
    list.extend(extra);
    let dict = Dictionary::from_entries(list);
-    let bytes = dict.to_bytes();
+    let bytes = dict.to_bytes().expect("dictionary within the inflate limit");
    std::fs::create_dir_all(&out_dir).expect("mkdir");
    std::fs::write(out_dir.join("en-us.dic"), &bytes).expect("write en-us.dic");
    let back = Dictionary::from_bytes(&bytes).expect("roundtrip");
@@ -49,7 +49,8 @@ impl Dictionary {

    /// Deserialize data written by `Dictionary::to_bytes`.
    pub(crate) fn decode(data: &[u8]) -> Result<Dictionary, String> {
-        let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?;
+        let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, MAX_INFLATED)
+            .map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?;
        let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?;
        let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() };
        let mut cur: Vec<u8> = Vec::new();
@@ -90,7 +91,7 @@ impl Dictionary {
    }

    /// Serialize (deflate-compressed).
-    pub fn to_bytes(&self) -> Vec<u8> {
+    pub fn to_bytes(&self) -> Result<Vec<u8>, String> {
        let mut raw = MAGIC.to_vec();
        let mut prev: Vec<u8> = Vec::new();
        for (w, m) in self.iter() {
@@ -101,7 +102,11 @@ impl Dictionary {
            raw.push(b'\n');
            prev = h;
        }
-        miniz_oxide::deflate::compress_to_vec(&raw, 10)
+        // decode refuses anything past MAX_INFLATED, so refuse to write it
+        if raw.len() > MAX_INFLATED {
+            return Err(format!("hyphenation dictionary: {} bytes exceeds the {MAX_INFLATED}-byte limit", raw.len()));
+        }
+        Ok(miniz_oxide::deflate::compress_to_vec(&raw, 10))
    }

    /// Deserialize data written by [`Dictionary::to_bytes`].
@@ -110,6 +115,11 @@ impl Dictionary {
    }
 }

+/// The most bytes a dictionary may inflate to. The bundled US English dictionary inflates to
+/// under 2 MiB; a small stream must not be able to ask for the whole of memory before the
+/// header is checked.
+const MAX_INFLATED: usize = 64 << 20;
+
 /// `word` with `-` before every char whose bit is set in `mask`.
 #[cfg(any(test, feature = "generator"))]
 pub fn hyphenated(word: &str, mask: u64) -> String {
@@ -149,11 +159,18 @@ mod tests {
    fn roundtrip() {
        let e: Vec<(String, u64)> = ["ty-pog-ra-phy", "cat", "hap-pen", "o'clock", "naïve-té", "naïve"].iter().map(|s| parse_hyphenated(s)).collect();
        let d = Dictionary::from_entries(e);
-        let back = Dictionary::from_bytes(&d.to_bytes()).unwrap();
+        let back = Dictionary::from_bytes(&d.to_bytes().unwrap()).unwrap();
        assert_eq!(back.len(), 6);
        assert_eq!(back.get("typography").map(|m| hyphenated("typography", m)), Some("ty-pog-ra-phy".to_string()));
        assert_eq!(back.get("cat"), Some(0));
        assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string()));
        assert!(back.get("dog").is_none());
    }
+
+    #[test]
+    fn an_oversized_stream_is_refused_before_parsing() {
+        let bomb = miniz_oxide::deflate::compress_to_vec(&vec![b'a'; MAX_INFLATED + 1], 10);
+        assert!(bomb.len() < 1 << 20, "{} bytes compressed", bomb.len());
+        assert!(Dictionary::from_bytes(&bomb).unwrap_err().contains("inflate failed"));
+    }
 }
@@ -427,10 +427,12 @@ fn char_points(word: &str, lim: &Limits, lang: Lang) -> Vec<usize> {

 /// `word` with `-` at every hyphenation point (tests and diagnostics).
 pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
-    let pts = hyphen_points(word, lim);
+    let mut pts = hyphen_points(word, lim);
+    pts.sort_unstable();
+    let mut next = pts.iter().peekable();
    let mut s = String::with_capacity(word.len() + pts.len());
    for (i, ch) in word.char_indices() {
-        if pts.contains(&i) {
+        if next.next_if_eq(&&i).is_some() {
            s.push('-');
        }
        s.push(ch);
@@ -178,8 +178,16 @@ pub fn score(pats: &Patterns, words: &[(Vec<char>, Vec<usize>)]) -> Score {
        let Some(pts) = pats.points(w) else { continue };
        s.words += 1;
        s.breaks += br.len();
+        // Indexed once per word: `pts` are in `1..w.len()`, so no expected break past the word
+        // can be found anyway.
+        let mut expected = vec![false; w.len() + 1];
+        for &b in br {
+            if let Some(e) = expected.get_mut(b) {
+                *e = true;
+            }
+        }
        for p in &pts {
-            if br.contains(p) {
+            if expected.get(*p).copied().unwrap_or(false) {
                s.found += 1;
            } else {
                s.wrong += 1;
@@ -7,6 +7,10 @@
 use std::collections::HashMap;
 use std::hash::{BuildHasherDefault, Hasher};

+/// The most patterns one set holds; later ones are ignored. The bundled US English set has
+/// about 7,000.
+pub(crate) const MAX_PATTERNS: usize = 1 << 20;
+
 /// Word-boundary marker in patterns.
 pub(crate) const DOT: char = '.';
 /// Longest pattern the packed key supports.
@@ -17,9 +21,6 @@ pub(crate) const MAX_PATTERN_LEN: usize = 9;
 pub(crate) struct FxHasher(u64);

 impl Hasher for FxHasher {
-    fn finish(&self) -> u64 {
-        self.0
-    }
    fn write(&mut self, bytes: &[u8]) {
        for &b in bytes {
            self.write_u64(b as u64);
@@ -28,6 +29,12 @@ impl Hasher for FxHasher {
    fn write_u64(&mut self, i: u64) {
        self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95);
    }
+    fn finish(&self) -> u64 {
+        // The table indexes by the low bits, which a multiplication leaves depending on the
+        // low bits of the key alone: packed patterns that share their first letters would
+        // share a bucket. Rotating brings the well-mixed high bits down (as rustc-hash 2 does).
+        self.0.rotate_left(26)
+    }
 }

 pub(crate) type FxMap<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher>>;
@@ -123,8 +130,12 @@ impl Patterns {
        if codes.is_empty() || codes.len() > MAX_PATTERN_LEN {
            return false;
        }
+        let key = pack(&codes);
+        if self.map.len() >= MAX_PATTERNS && !self.map.contains_key(&key) {
+            return false;
+        }
        self.max_len = self.max_len.max(codes.len());
-        self.map.insert(pack(&codes), vals.into_boxed_slice());
+        self.map.insert(key, vals.into_boxed_slice());
        true
    }

@@ -238,4 +249,27 @@ mod tests {
        assert!(tex.contains(&"hen5at".to_string()));
        assert!(p.values(&['h', 'é']).is_none());
    }
+
+    #[test]
+    fn packed_keys_sharing_a_prefix_spread_over_the_buckets() {
+        // 100,000 patterns whose packed keys differ in the last letter codes only. With the low
+        // bits of a multiplicative hash left as they are, they probe the same buckets.
+        let letters: Vec<char> = ('a'..='z').collect();
+        let mut text = String::new();
+        for i in 0..100_000u32 {
+            let mut n = i;
+            text.push_str("p1r");
+            for _ in 0..4 {
+                text.push(letters[(n % 26) as usize]);
+                n /= 26;
+            }
+            text.push(' ');
+        }
+        let started = std::time::Instant::now();
+        let p = Patterns::parse(&text);
+        assert_eq!(p.map.len(), 100_000);
+        assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed());
+        let w: Vec<char> = "praaaa".chars().collect();
+        assert_eq!(p.points(&w).unwrap(), vec![1]);
+    }
 }
@@ -419,6 +419,9 @@ struct Sums<'a> {
    z: Vec<Tiers>,
    over: Vec<f64>,
    fill: Vec<u32>,
+    /// After item `k`: the first box or forced break, or the item count. Looked up for a break
+    /// at `k`, so it is found once per item rather than by a scan per candidate.
+    next: Vec<usize>,
 }

 impl<'a> Sums<'a> {
@@ -450,7 +453,12 @@ impl<'a> Sums<'a> {
        z.push(sz);
        over.push(so);
        fill.push(sf);
-        Sums { items, w, y, z, over, fill }
+        let mut next = vec![m; m + 1];
+        for b in (0..m).rev() {
+            let i = b + 1;
+            next[b] = if items.get(i).is_some_and(|it| it.kind == Kind::Box || it.is_forced()) { i } else { next[i] };
+        }
+        Sums { items, w, y, z, over, fill, next }
    }

    fn len(&self) -> usize {
@@ -481,14 +489,7 @@ impl<'a> Sums<'a> {
    /// First item of the line after a break at `b`: glue and penalties are discarded up to the next
    /// box or forced break.
    fn next_start(&self, b: usize) -> usize {
-        let mut i = b + 1;
-        while let Some(it) = self.items.get(i) {
-            if it.kind == Kind::Box || it.is_forced() {
-                break;
-            }
-            i += 1;
-        }
-        i.min(self.len())
+        self.next.get(b).copied().unwrap_or(self.len())
    }

    fn at(&self, k: usize) -> (f64, Tiers, Tiers, f64, u32) {
@@ -474,3 +474,21 @@ fn overfull_lines_keep_the_hyphen_limit() {
    assert!(lines.windows(2).all(|w| !(w[0].flagged && w[1].flagged)), "{lines:?}");
    assert_eq!(spans(&lines), [(0, 1), (2, 5)]);
 }
+
+#[test]
+fn next_start_is_the_first_box_or_forced_break_after_each_item() {
+    // Boxes, glue, penalties and forced breaks in a fixed mix; the table matches a scan.
+    let items: Vec<Item> = (0..2_000)
+        .map(|i| match i % 7 {
+            0 | 3 => Item::boxed(CHAR),
+            1 => Item::glue(SPACE, 1.0, 1.0),
+            6 if i % 3 == 0 => Item::forced(),
+            _ => Item::penalty(HYPHEN, 50.0, true),
+        })
+        .collect();
+    let s = Sums::new(&items);
+    for b in 0..=items.len() {
+        let scan = (b + 1..items.len()).find(|&i| items[i].kind == Kind::Box || items[i].is_forced()).unwrap_or(items.len());
+        assert_eq!(s.next_start(b), scan, "after {b}");
+    }
+}