diff --git a/crates/render/src/text.rs b/crates/render/src/text.rs index aa66a4b683..bc132466b1 100644 --- a/crates/render/src/text.rs +++ b/crates/render/src/text.rs @@ -131,7 +131,9 @@ impl PageText { } /// Case-insensitive search. Returns glyph ranges of every match (words may span line breaks, - /// which count as a single space). + /// which count as a single space, or as nothing where the line runs on between two Chinese or + /// Japanese characters or ends in a soft hyphen: there a query with a space across the break + /// doesn't match, and one without does). pub fn find(&self, needle: &str) -> Vec> { self.find_opts(needle, false, false) } diff --git a/crates/render/src/text/search.rs b/crates/render/src/text/search.rs index c68660dbe9..71670cc271 100644 --- a/crates/render/src/text/search.rs +++ b/crates/render/src/text/search.rs @@ -1,4 +1,8 @@ //! Literal search in normalized page text, retaining the original glyph owners. +//! +//! Words and lines are separated by one space, except where a line of Chinese or Japanese runs on +//! to the next: a break between two such characters is no word break, so it reads as nothing and +//! "検知機構" is found when the line ends after "検知". use super::{PageText, fold_case}; use std::ops::Range; @@ -7,21 +11,51 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b return Vec::new(); } let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) }; - let needle: Vec = fold(query).split_whitespace().collect::>().join(" ").chars().collect(); + let query: String = fold(query).chars().filter(|&c| !is_invisible(c)).collect(); + let needle: Vec = query.split_whitespace().collect::>().join(" ").chars().collect(); if needle.is_empty() { return Vec::new(); } let mut chars: Vec<(char, usize)> = Vec::new(); + // Whether a skipped ZERO WIDTH SPACE came right before each character: it draws nothing, but + // it is a word boundary (UAX #29 breaks on both sides of it), which Thai and Khmer write between words + // with no visible gap, so whole-word search still sees two words there. + let mut zero_width_space: Vec = Vec::new(); + let mut pending_zero_width_space = false; + // The last character pushed other than a variation selector, kept as characters are pushed so + // a long run of selectors costs no rescan. + let mut last_base: Option = None; for (i, glyph) in page.glyphs.iter().enumerate() { + let text = fold(&glyph.text); let line_break = i.checked_sub(1).and_then(|previous| page.line_of.get(previous)).zip(page.line_of.get(i)).is_some_and(|(a, b)| a != b); - let gap = page.space_before.get(i).copied().unwrap_or(false) || line_break; + // A soft hyphen marks where a word may be divided at a line end (ISO 32000-2 §14.8.2.3), so + // a line that ends in one runs on to the next with no word break between them. + // Other invisible characters after the soft hyphen don't hide it. + let soft_break = line_break && last_kept(page, i) == Some('\u{AD}'); + // So does a line of Chinese or Japanese that runs on to the next. The characters on both + // sides of the break are the visible ones: invisible characters are skipped below, and a + // variation selector after the last one only picks its glyph. + let joined = line_break && last_base.is_some_and(unspaced) && first_visible(page, i).is_some_and(unspaced) && wraps(page, i); + let gap = page.space_before.get(i).copied().unwrap_or(false) || (line_break && !(joined || soft_break)); if i > 0 && gap && chars.last().is_some_and(|c| c.0 != ' ') { chars.push((' ', i)); + last_base = Some(' '); + zero_width_space.push(false); + pending_zero_width_space = false; } - for c in fold(&glyph.text).chars() { + for c in text.chars() { + if is_invisible(c) { + pending_zero_width_space |= c == '\u{200B}'; + continue; + } let c = if c.is_whitespace() { ' ' } else { c }; if c != ' ' || chars.last().is_none_or(|last| last.0 != ' ') { chars.push((c, i)); + if !is_variation_selector(c) { + last_base = Some(c); + } + zero_width_space.push(pending_zero_width_space && c != ' '); + pending_zero_width_space = false; } } } @@ -48,7 +82,9 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b } let end = i.saturating_add(1); let start = end.saturating_sub(needle.len()); - let bounded = !whole_words || (!word(start.checked_sub(1)) && !word(Some(end))); + // A skipped zero width space before the first character or after the last is a boundary. + let after_break = |i: usize| zero_width_space.get(i).copied().unwrap_or(false); + let bounded = !whole_words || ((after_break(start) || !word(start.checked_sub(1))) && (after_break(end) || !word(Some(end)))); if bounded { if let Some((_, first)) = chars.get(start) && let Some(last) = owner.checked_add(1) @@ -68,6 +104,100 @@ pub(super) fn find(page: &PageText, query: &str, sensitive: bool, whole_words: b hits } +/// A character that draws nothing inside a word and only tells layout where a line may break or +/// letters may join: SOFT HYPHEN, ZERO WIDTH SPACE, ZERO WIDTH NON-JOINER and JOINER, WORD JOINER +/// and ZERO WIDTH NO-BREAK SPACE. All six are Unicode default-ignorable format characters, so +/// find skips them in the page text and in the query alike: "Mitarbeiterinnen" matches a word +/// stored with soft hyphens. A visible word gap is its own space in the page text, so skipping a +/// zero width space next to one still keeps the two words apart. +fn is_invisible(c: char) -> bool { + matches!(c, '\u{00AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}') +} + +/// A variation selector (VS1-VS16 and VS17-VS256): it picks a variant glyph of the character +/// before it and is no character of its own. +fn is_variation_selector(c: char) -> bool { + matches!(c as u32, 0xFE00..=0xFE0F | 0xE0100..=0xE01EF) +} + +/// The last character before glyph `i` on the line that glyph `i - 1` ends, other than the +/// invisible characters find skips; a soft hyphen counts, since it marks a divided word. Looks +/// back no further than that line's start, so each line is read once. +fn last_kept(page: &PageText, i: usize) -> Option { + let end = i.checked_sub(1)?; + let line = page.line_of.get(end)?; + (0..=end) + .rev() + .take_while(|&k| page.line_of.get(k) == Some(line)) + .filter_map(|k| page.glyphs.get(k)) + .flat_map(|g| g.text.chars().rev()) + .find(|&c| c == '\u{AD}' || !is_invisible(c)) +} + +/// Whether `c` is set solid, with no spaces between words, in text that a line break may divide +/// anywhere: East Asian Width Wide, Fullwidth or Halfwidth (Unicode Standard Annex #11) Han, +/// kana, bopomofo, CJK symbols and punctuation and fullwidth forms other than Latin letters and +/// digits. Hangul is left out: Korean separates its words with spaces, so a line end there is a +/// word break. A line break between two such characters is the CSS segment-break rule's case for removing the break, not turning +/// it into a space (CSS Text Module, segment break transformation). +fn unspaced(c: char) -> bool { + matches!(c as u32, + 0x2E80..=0x2FFF // CJK radicals, Kangxi radicals, ideographic description characters + | 0x3001..=0x303E // CJK symbols and punctuation (U+3000 is a space; U+303F is narrow) + | 0x3041..=0x30FF // Hiragana, Katakana + | 0x3105..=0x312F // Bopomofo + | 0x3190..=0x31FF // Kanbun, Bopomofo extended, CJK strokes, Katakana phonetic extensions + | 0x3200..=0x4DBF // Enclosed CJK letters, CJK compatibility, Han extension A + | 0x4E00..=0x9FFF // Han + | 0xF900..=0xFAFF // CJK compatibility ideographs + | 0xFE10..=0xFE19 | 0xFE30..=0xFE6F // vertical, compatibility and small forms + // Fullwidth forms, less the fullwidth digits and Latin letters (U+FF10-U+FF19, U+FF21-U+FF3A, + // U+FF41-U+FF5A): those are Latin text set wide, whose words still end at a line end. + | 0xFF01..=0xFF0F | 0xFF1A..=0xFF20 | 0xFF3B..=0xFF40 | 0xFF5B..=0xFF60 | 0xFFE0..=0xFFE6 + | 0xFF61..=0xFF9F // halfwidth CJK punctuation and Katakana (not halfwidth Hangul) + | 0x16FE0..=0x16FFF | 0x1AFF0..=0x1B2FF // ideographic symbols, kana supplements and extensions + | 0x20000..=0x3FFFD // Han extensions B and later, compatibility supplement + ) && !matches!(c as u32, 0x3200..=0x321E | 0x3260..=0x327E) // parenthesised and circled Hangul +} + +/// Whether glyph `i` starts the line that the text of glyph `i - 1` runs on to: both run the same +/// way, it sits past that glyph across the lines, by more than half the glyph's line height and +/// at most three, and starts no further along the line than half a line height past it, as a +/// wrapped line does (a line of one character starts level with the one before). Text set side by +/// side on one line (table cells, labels), a cell set lower and further along the row, a column +/// that starts back at the top, a block further down the page and a flow in another direction +/// keep their word break. +fn wraps(page: &PageText, i: usize) -> bool { + // Measured on the glyphs that own the visible characters either side of the break: an + // invisible glyph or a variation selector of its own (drawn with no height) says nothing about + // the lines. Each search stays within one line. + let Some(end) = i.checked_sub(1) else { return false }; + let visible = |k: &usize| page.glyphs.get(*k).is_some_and(|g| g.text.chars().any(|c| !is_invisible(c) && !is_variation_selector(c))); + let same_line = |k: &usize, of: usize| page.line_of.get(*k).is_some() && page.line_of.get(*k) == page.line_of.get(of); + let before = (0..=end).rev().take_while(|k| same_line(k, end)).find(visible).unwrap_or(end); + let after = (i..page.glyphs.len()).take_while(|k| same_line(k, i)).find(visible).unwrap_or(i); + let Some((before, after)) = page.glyphs.get(before).zip(page.glyphs.get(after)) else { return false }; + // Unit directions: the same flow has a dot product of 1. + let same_flow = before.direction[0] * after.direction[0] + before.direction[1] * after.direction[1] > 0.99; + let centre = |q: &[[f32; 2]; 4]| [q.iter().map(|p| p[0]).sum::() / 4.0, q.iter().map(|p| p[1]).sum::() / 4.0]; + let (a, b) = (centre(&before.quad), centre(&after.quad)); + // Towards the next line: the writing direction turned a quarter clockwise in y-down view space. + let [dx, dy] = before.direction; + let across = (b[0] - a[0]) * -dy + (b[1] - a[1]) * dx; + let along = (b[0] - a[0]) * dx + (b[1] - a[1]) * dy; + let [near, far] = [before.quad[0], before.quad[3]]; + let height = (near[0] - far[0]).hypot(near[1] - far[1]); + // NaN compares false: no join. + same_flow && across > height * 0.5 && across <= height * 3.0 && along < height * 0.5 +} + +/// The first visible character on the line that glyph `i` starts, skipping the invisible characters +/// that find drops, so a line that opens with a zero width character still joins on its first letter. +fn first_visible(page: &PageText, i: usize) -> Option { + let line = page.line_of.get(i)?; + page.glyphs.iter().zip(&page.line_of).skip(i).take_while(|(_, l)| *l == line).flat_map(|(g, _)| g.text.chars()).find(|&c| !is_invisible(c)) +} + /// Extend a prefix, falling back through strictly shorter proper prefixes on a mismatch. fn advance(needle: &[char], prefix: &[usize], mut matched: usize, c: char) -> usize { while matched > 0 && needle.get(matched) != Some(&c) { diff --git a/crates/render/tests/it/search.rs b/crates/render/tests/it/search.rs index 44fbe49350..5b2a1e4c46 100644 --- a/crates/render/tests/it/search.rs +++ b/crates/render/tests/it/search.rs @@ -12,28 +12,53 @@ fn page(parts: &[&str]) -> PageText { // The original straightforward search is an independent oracle for valid page metadata. fn reference(page: &PageText, query: &str, sensitive: bool, whole: bool, limit: usize) -> Vec> { let fold = |s: &str| if sensitive { s.to_owned() } else { fold_case(s) }; - let needle: Vec = fold(query).split_whitespace().collect::>().join(" ").chars().collect(); + // Soft hyphens and zero width characters draw nothing and are skipped on both sides. + let visible = + |s: String| s.chars().filter(|c| !matches!(c, '\u{AD}' | '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}')).collect::(); + let needle: Vec = visible(fold(query)).split_whitespace().collect::>().join(" ").chars().collect(); if needle.is_empty() || limit == 0 { return Vec::new(); } - let mut chars = Vec::new(); + // Each character, its glyph, and whether a skipped zero width space came right before it. + let mut chars: Vec<(char, usize, bool)> = Vec::new(); + let mut after_zwsp = false; for (i, glyph) in page.glyphs.iter().enumerate() { - if i > 0 && (page.space_before[i] || page.line_of[i] != page.line_of[i - 1]) && chars.last().is_some_and(|&(c, _)| c != ' ') { - chars.push((' ', i)); + // A line that ends in a soft hyphen, past any empty or other invisible glyphs and + // characters after it on that line, runs on to the next. + let ends_in_soft_hyphen = |end: usize| { + (0..=end) + .rev() + .take_while(|&k| page.line_of[k] == page.line_of[end]) + .flat_map(|k| page.glyphs[k].text.chars().rev()) + .find(|&c| c == '\u{AD}' || !visible(c.to_string()).is_empty()) + == Some('\u{AD}') + }; + let line_break = i > 0 && page.line_of[i] != page.line_of[i - 1] && !ends_in_soft_hyphen(i - 1); + if i > 0 && (page.space_before[i] || line_break) && chars.last().is_some_and(|&(c, ..)| c != ' ') { + chars.push((' ', i, false)); + after_zwsp = false; } for c in fold(&glyph.text).chars() { + after_zwsp |= c == '\u{200B}'; + if visible(c.to_string()).is_empty() { + continue; + } let c = if c.is_whitespace() { ' ' } else { c }; - if c != ' ' || chars.last().is_none_or(|&(last, _)| last != ' ') { - chars.push((c, i)); + if c != ' ' || chars.last().is_none_or(|&(last, ..)| last != ' ') { + chars.push((c, i, after_zwsp && c != ' ')); + after_zwsp = false; } } } - let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|&(c, _)| c.is_alphanumeric()); + let word = |i: Option| i.and_then(|i| chars.get(i)).is_some_and(|&(c, ..)| c.is_alphanumeric()); + // A word ends at a skipped zero width space as it does at a space. + let edge = |i: usize| chars.get(i).is_some_and(|&(.., zwsp)| zwsp); let mut hits = Vec::new(); let mut start = 0; while start + needle.len() <= chars.len() { let end = start + needle.len(); - if (!whole || (!word(start.checked_sub(1)) && !word(Some(end)))) && chars[start..end].iter().map(|&(c, _)| c).eq(needle.iter().copied()) { + let bounded = (edge(start) || !word(start.checked_sub(1))) && (edge(end) || !word(Some(end))); + if (!whole || bounded) && chars[start..end].iter().map(|&(c, ..)| c).eq(needle.iter().copied()) { hits.push(chars[start].1..chars[end - 1].1 + 1); if hits.len() == limit { break; @@ -69,7 +94,7 @@ fn whole_word_rejection_keeps_overlapping_candidates() { #[test] fn search_matches_reference_for_unicode_gaps_limits_and_word_boundaries() { - let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "e\u{301}", "العربية"]; + let alphabet = ["a", "b", " ", "-", "İ", "ı", "ς", "Σ", "\t", "\n", "ffi", "", "界", "e\u{301}", "العربية", "\u{AD}", "a\u{200B}"]; let mut seed = 1234567u64; let mut next = || { seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); @@ -110,3 +135,486 @@ fn long_repeated_prefix_search_preserves_the_last_match_and_limit() { assert_eq!(text.find_opts_limited(&query, false, false, 1), vec![126977..131073]); assert!(text.find(&format!("{}c", "a".repeat(4095))).is_empty()); } + +/// pdf.js issue 13226 stores "Mitarbeiterinnen" with soft hyphens at its hyphenation points, and +/// issue 9186 puts zero width spaces next to the gaps between words. Find skips both characters +/// (and the other invisible joiners) in the page and in the query, and a match still starts and +/// ends on visible letters. +#[test] +fn find_skips_soft_hyphens_and_zero_width_characters() { + let word = page(&["M", "i", "t", "\u{AD}", "a", "r", "b", "e", "i", "\u{AD}", "t", "e", "r", "i", "n", "n", "e", "n"]); + assert_eq!(word.find("Mitarbeiterinnen"), vec![0..18]); + assert_eq!(word.find_opts("Mitarbeiterinnen", true, true), vec![0..18]); + assert_eq!(word.find("arbei"), vec![4..9]); + // Inside one word, a piece of it is not a whole word. + assert!(word.find_opts("terinnen", false, true).is_empty()); + // A query that carries the soft hyphens still matches. + assert_eq!(word.find("Mit\u{AD}arbei\u{AD}terinnen"), vec![0..18]); + + let mut gaps = page(&["is", "\u{200B}", "\u{200B}", "made", "\u{2060}", "as"]); + gaps.space_before = vec![false, false, true, false, false, true]; + assert_eq!(gaps.find("is made"), vec![0..4]); + assert_eq!(gaps.find_opts("made as", false, true), vec![3..6]); + for joiner in ['\u{200C}', '\u{200D}', '\u{2060}', '\u{FEFF}'] { + assert_eq!(page(&["ab", &joiner.to_string(), "cd"]).find("abcd"), vec![0..3], "{joiner:?}"); + // The same character typed in the query is skipped too. + assert_eq!(page(&["ab", "cd"]).find_opts(&format!("ab{joiner}cd"), true, false), vec![0..2], "{joiner:?}"); + } + assert!(page(&["\u{200B}"]).find("\u{200B}").is_empty()); +} + +/// A soft hyphen at a line end is where the word was divided (ISO 32000-2 §14.8.2.3): the break +/// after it is no word break, so "Mitar\u{AD}" ending one line and "beiterinnen" starting the next +/// are found as one word, highlighted on both lines. A line break after any other glyph still +/// separates words. +#[test] +fn find_joins_a_word_divided_at_a_line_end_soft_hyphen() { + let parts: Vec = "Mitar\u{AD}beiterinnen".chars().map(String::from).collect(); + let mut text = page(&parts.iter().map(String::as_str).collect::>()); + for (i, glyph) in text.glyphs.iter_mut().enumerate() { + let x = i as f32 * 10.0; + glyph.rect = [x, 0.0, x + 10.0, 10.0]; + } + // "Mitar" and the soft hyphen on the first line, "beiterinnen" on the second. + for line in text.line_of.iter_mut().skip(6) { + *line = 1; + } + let hits = text.find("Mitarbeiterinnen"); + assert_eq!(hits, vec![0..17]); + assert_eq!(text.line_rects(hits[0].clone()).len(), 2); + assert_eq!(text.find_opts("Mitarbeiterinnen", true, true), vec![0..17]); + assert!(text.find("Mitar beiterinnen").is_empty()); + + // The soft hyphen as the last character of a longer glyph works the same way. + let mut text = page(&["Mitar\u{AD}", "beiterinnen"]); + text.line_of = vec![0, 1]; + assert_eq!(text.find("Mitarbeiterinnen"), vec![0..2]); + + // Without the soft hyphen the line break is a word break. + let mut text = page(&["Mitar", "beiterinnen"]); + text.line_of = vec![0, 1]; + assert!(text.find("Mitarbeiterinnen").is_empty()); + assert_eq!(text.find("Mitar beiterinnen"), vec![0..2]); +} + +/// Horizontal glyphs of `size` points, one per character, each line `lead` points below the +/// last and starting at `x`: (x, first line's top, text) per line. +fn laid_out(lines: &[(f32, f32, &str)], size: f32) -> PageText { + let mut glyphs = Vec::new(); + for &(x0, top, text) in lines { + for (k, c) in text.chars().enumerate() { + let x = x0 + size * k as f32; + let (y0, y1) = (top, top + size); + glyphs.push(TextGlyph { + text: c.into(), + rect: [x, y0, x + size, y1], + quad: [[x, y1], [x + size, y1], [x + size, y0], [x, y0]], + direction: [1.0, 0.0], + }); + } + } + pdfcraft_render::text::layout(glyphs) +} + +#[test] +fn a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines() { + // "検知機構" wraps after "検知" (pdf.js #15340); "ソレノイド" after "ソレ". + let text = laid_out(&[(10.0, 10.0, "装置の検知"), (10.0, 25.0, "機構です"), (10.0, 40.0, "ソレ"), (10.0, 55.0, "ノイド")], 10.0); + assert_eq!(text.line_of.iter().copied().max(), Some(3)); + let hits = text.find("検知機構"); + assert_eq!(hits, vec![3..7]); + // One highlight per line, the end of the first line and the start of the second. + assert_eq!(text.line_rects(hits[0].clone()), vec![[40.0, 10.0, 60.0, 20.0], [10.0, 25.0, 30.0, 35.0]]); + assert_eq!(text.find("ソレノイド"), vec![9..14]); + // The query typed with a space still finds nothing across the joined break. + assert!(text.find("検知 機構").is_empty()); + // Copied text keeps the line break. + assert_eq!(text.text_of(3..7), "検知\n機構"); +} + +#[test] +fn a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text() { + // Latin and Korean separate words with spaces: the line end is one. + let latin = laid_out(&[(10.0, 10.0, "abc"), (10.0, 25.0, "def")], 10.0); + assert_eq!(latin.find("abc def"), vec![0..6]); + assert!(latin.find("abcdef").is_empty()); + let mixed = laid_out(&[(10.0, 10.0, "PDF"), (10.0, 25.0, "検知")], 10.0); + assert_eq!(mixed.find("PDF 検知"), vec![0..5]); + assert!(mixed.find("PDF検知").is_empty()); + let hangul = laid_out(&[(10.0, 10.0, "한국"), (10.0, 25.0, "어")], 10.0); + assert_eq!(hangul.find("한국 어"), vec![0..3]); + assert!(hangul.find("한국어").is_empty()); + // Two cells on one row read as two lines, but the second does not continue the first. + let row = laid_out(&[(10.0, 10.0, "東京"), (200.0, 10.0, "大阪")], 10.0); + assert_eq!(row.line_of, vec![0, 0, 1, 1]); + assert!(row.find("東京大阪").is_empty()); + assert_eq!(row.find("東京 大阪"), vec![0..4]); + // Nor does a cell set lower and further along the row: a wrapped line starts back behind the + // end of the line before it. + let staggered = laid_out(&[(10.0, 10.0, "東京"), (200.0, 17.0, "大阪")], 10.0); + assert_eq!(staggered.line_of, vec![0, 0, 1, 1]); + assert!(staggered.find("東京大阪").is_empty()); + assert_eq!(staggered.find("東京 大阪"), vec![0..4]); + // Fullwidth Latin letters and digits are Latin text set wide: their line end is a word break. + let wide = laid_out(&[(10.0, 10.0, "AB"), (10.0, 25.0, "CD"), (10.0, 40.0, "12"), (10.0, 55.0, "34")], 10.0); + assert!(wide.find_opts("ABCD", true, false).is_empty()); + assert_eq!(wide.find_opts("AB CD", true, false), vec![0..4]); + assert!(wide.find("1234").is_empty()); + assert_eq!(wide.find("12 34"), vec![4..8]); +} + +/// Vertical glyphs (WMode 1) of `size` points, one per character, written downwards: each +/// column starts at `top` with its left edge at `x`: (x, top, text) per column, in reading order. +/// Each column is one line, as the text device groups a vertical flow. +fn columns(cols: &[(f32, f32, &str)], size: f32) -> PageText { + let mut text = PageText::default(); + for (line, &(x, top, chars)) in (0u32..).zip(cols) { + for (k, c) in chars.chars().enumerate() { + let y = top + size * k as f32; + // Written downwards: the em box's edge towards the next line is its left one. + text.glyphs.push(TextGlyph { + text: c.into(), + rect: [x, y, x + size, y + size], + quad: [[x, y], [x, y + size], [x + size, y + size], [x + size, y]], + direction: [0.0, 1.0], + }); + text.line_of.push(line); + text.space_before.push(false); + } + } + text +} + +#[test] +fn a_word_divided_between_vertical_columns_is_found_in_both() { + // Japanese set vertically: columns run top to bottom and follow each other to the left. + // "検知機構" breaks after "検知" at the foot of the right column. + let text = columns(&[(40.0, 10.0, "装置の検知"), (25.0, 10.0, "機構です")], 10.0); + let hits = text.find("検知機構"); + assert_eq!(hits, vec![3..7]); + // One highlight per column: the foot of the right one and the head of the left one. + assert_eq!(text.line_rects(hits[0].clone()), vec![[40.0, 40.0, 50.0, 60.0], [25.0, 10.0, 35.0, 30.0]]); + assert_eq!( + text.text_of(3..7), + "検知 +機構" + ); + + // A column that stands to the right of the one before does not continue it. + let back = columns(&[(40.0, 10.0, "装置の検知"), (55.0, 10.0, "機構です")], 10.0); + assert!(back.find("検知機構").is_empty()); + assert_eq!(back.find("検知 機構"), vec![3..7]); +} + +/// A zero width space draws nothing and is skipped, but UAX #29 breaks words on both sides of it: +/// Thai and Khmer write it between words with no visible gap, so each word is still a whole word. +/// The joiners and the soft hyphen join, so they are no boundary. +#[test] +fn a_zero_width_space_is_a_word_boundary_for_whole_words() { + let thai = page(&["สวัสดี", "\u{200B}", "ครับ"]); + assert_eq!(thai.find_opts("ครับ", false, true), vec![2..3]); + assert_eq!(thai.find_opts("สวัสดี", false, true), vec![0..1]); + // The words read together, as typed with no space, still match. + assert_eq!(thai.find_opts("สวัสดีครับ", false, true), vec![0..3]); + // Inside one glyph's text too. + assert_eq!(page(&["ab\u{200B}cd"]).find_opts("cd", false, true), vec![0..1]); + for joiner in ["\u{AD}", "\u{200C}", "\u{200D}", "\u{2060}", "\u{FEFF}"] { + let text = page(&["ab", joiner, "cd"]); + assert!(text.find_opts("cd", false, true).is_empty(), "{joiner:?}"); + assert!(text.find_opts("ab", false, true).is_empty(), "{joiner:?}"); + assert_eq!(text.find_opts("abcd", false, true), vec![0..3], "{joiner:?}"); + } +} + +/// Every range `unspaced` lists joins a line it ends to the next one, on either side of the break; +/// Hangul, parenthesised and circled Hangul, fullwidth Latin and digits, Yi and ASCII keep a space. +#[test] +fn each_unspaced_range_joins_and_spaced_scripts_do_not() { + let joined = [ + '\u{2E80}', + '\u{2F00}', // CJK radicals, Kangxi radicals + '\u{3001}', + '\u{3006}', // CJK symbols and punctuation + '\u{3042}', + '\u{30A2}', // Hiragana, Katakana + '\u{3105}', // Bopomofo + '\u{3190}', + '\u{31A0}', + '\u{31C0}', + '\u{31F0}', // Kanbun, Bopomofo extended, strokes, Katakana extensions + '\u{3220}', + '\u{3300}', + '\u{3400}', // enclosed ideograph, CJK compatibility, Han extension A + '\u{4E2D}', // Han + '\u{F900}', // CJK compatibility ideographs + '\u{FE10}', + '\u{FE30}', + '\u{FE50}', // vertical, compatibility and small forms + '\u{FF01}', + '\u{FF1A}', + '\u{FF3B}', + '\u{FF5B}', + '\u{FFE0}', // fullwidth punctuation and signs + '\u{FF61}', + '\u{FF71}', // halfwidth CJK punctuation, halfwidth Katakana + '\u{16FE0}', + '\u{1AFF0}', + '\u{1B000}', // ideographic symbols, kana extended and supplement + '\u{20000}', + '\u{2F800}', + '\u{30000}', // Han extension B, compatibility supplement, extension G + ]; + for c in joined { + let text = laid_out(&[(10.0, 10.0, &format!("中{c}")), (10.0, 25.0, &format!("{c}中"))], 10.0); + assert_eq!(text.find_opts(&format!("中{c}{c}中"), true, false), vec![0..4], "U+{:04X}", c as u32); + } + let spaced = ['\u{AC00}', '\u{3200}', '\u{3260}', '\u{FF10}', '\u{FF21}', '\u{FF41}', '\u{FFA1}', '\u{A000}', 'a']; + for c in spaced { + let text = laid_out(&[(10.0, 10.0, &format!("中{c}")), (10.0, 25.0, &format!("{c}中"))], 10.0); + assert!(text.find_opts(&format!("中{c}{c}中"), true, false).is_empty(), "U+{:04X}", c as u32); + assert_eq!(text.find_opts(&format!("中{c} {c}中"), true, false), vec![0..4], "U+{:04X}", c as u32); + } +} + +/// The two line-end rules meet: a soft hyphen after a Chinese or Japanese character, a line of +/// Chinese that runs on, a German word divided by a soft hyphen, and a line that opens with a zero +/// width space, all on one page. Each break reads as nothing, and plain Latin line ends stay spaces. +#[test] +fn a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine() { + let text = laid_out( + &[ + (10.0, 10.0, "装置の検\u{AD}"), + (10.0, 25.0, "知機構の"), + (10.0, 40.0, "ソレ"), + (10.0, 55.0, "\u{200B}ノイド"), + (10.0, 70.0, "Mitar\u{AD}"), + (10.0, 85.0, "beiterinnen"), + (10.0, 100.0, "abc"), + (10.0, 115.0, "def"), + ], + 10.0, + ); + // Both rules fire at the first break: one hit, highlighted on both lines. + let hits = text.find("検知機構"); + assert_eq!(hits, vec![3..8]); + assert_eq!(text.line_rects(hits[0].clone()).len(), 2); + assert!(text.find("検 知機構").is_empty()); + // A line of Chinese or Japanese with no soft hyphen, and one that opens with a zero width space. + assert_eq!(text.find("機構のソレノイド"), vec![6..15]); + // A soft hyphen between Latin letters. + assert_eq!(text.find_opts("Mitarbeiterinnen", true, true), vec![15..32]); + // Latin with neither: still a word break. + assert_eq!(text.find("abc def"), vec![32..38]); + assert!(text.find("abcdef").is_empty()); +} + +/// Glyphs of `size` points as [`laid_out`] sets them, each line `lead` below the last, with the +/// writing direction turned to `direction` about the first glyph's corner (a unit vector in +/// y-down view space; `[1, 0]` is [`laid_out`]). +fn turned(lines: &[&str], size: f32, lead: f32, direction: [f32; 2]) -> PageText { + let [dx, dy] = direction; + // Towards the next line: the writing direction turned a quarter clockwise in view space. + let [nx, ny] = [-dy, dx]; + let at = |along: f32, across: f32| [100.0 + along * dx + across * nx, 100.0 + along * dy + across * ny]; + let mut glyphs = Vec::new(); + for (l, line) in lines.iter().enumerate() { + for (k, c) in line.chars().enumerate() { + let (a, b) = (size * k as f32, lead * l as f32); + // Baseline edge first, in the writing direction; then the far edge back. + let quad = [at(a, b + size), at(a + size, b + size), at(a + size, b), at(a, b)]; + let xs = quad.map(|p| p[0]); + let ys = quad.map(|p| p[1]); + let rect = [ + xs.iter().copied().fold(f32::MAX, f32::min), + ys.iter().copied().fold(f32::MAX, f32::min), + xs.iter().copied().fold(f32::MIN, f32::max), + ys.iter().copied().fold(f32::MIN, f32::max), + ]; + glyphs.push(TextGlyph { text: c.into(), rect, quad, direction }); + } + } + let mut text = PageText::default(); + for (l, line) in (0u32..).zip(lines) { + for _ in line.chars() { + text.line_of.push(l); + text.space_before.push(false); + } + } + text.glyphs = glyphs; + text +} + +/// Text turned a quarter either way, or upside down, runs on between lines as horizontal text +/// does: the line after is the next one across the turned lines, starting back at the turned +/// line's head. +#[test] +fn a_chinese_or_japanese_word_divided_in_turned_text_is_found_on_both_lines() { + for direction in [[0.0, -1.0], [0.0, 1.0], [-1.0, 0.0]] { + let text = turned(&["装置の検知", "機構です"], 10.0, 15.0, direction); + assert_eq!(text.find("検知機構"), vec![3..7], "{direction:?}"); + assert!(text.find("検知 機構").is_empty(), "{direction:?}"); + } +} + +/// The line end after a soft hyphen is found past other invisible characters that follow it, in +/// the same glyph or in glyphs of their own. +#[test] +fn a_soft_hyphen_line_end_joins_past_invisible_characters() { + let hyphen = laid_out(&[(10.0, 10.0, "Mitar\u{AD}\u{200B}"), (10.0, 25.0, "beiterinnen")], 10.0); + assert_eq!(hyphen.find("Mitarbeiterinnen"), vec![0..18]); + let mut text = page(&["Mitar\u{AD}", "\u{2060}", "beiterinnen"]); + text.line_of = vec![0, 0, 1]; + assert_eq!(text.find("Mitarbeiterinnen"), vec![0..3]); +} + +/// A variation selector after the last Chinese or Japanese character of a line only picks its +/// glyph: the line still runs on to the next. +#[test] +fn a_variation_selector_at_a_line_end_keeps_the_line_running_on() { + let selector = laid_out(&[(10.0, 10.0, "葛\u{E0100}"), (10.0, 25.0, "飾区")], 10.0); + assert_eq!(selector.find("葛\u{E0100}飾区"), vec![0..4]); + assert!(selector.find("葛\u{E0100} 飾区").is_empty()); +} + +/// A wrapped line is near the one before: a block far down the page keeps its word break. +#[test] +fn a_line_end_does_not_join_a_block_far_below() { + let far = laid_out(&[(10.0, 10.0, "検知"), (10.0, 200.0, "機構")], 10.0); + assert!(far.find("検知機構").is_empty()); + assert_eq!(far.find("検知 機構"), vec![0..4]); +} + +/// A wrapped line runs the same way as the one before: a line in another writing direction keeps +/// its word break, wherever it sits. +#[test] +fn a_line_end_does_not_join_a_line_in_another_direction() { + let mut across = laid_out(&[(10.0, 10.0, "検知"), (10.0, 25.0, "機構")], 10.0); + for g in across.glyphs.iter_mut().skip(2) { + g.direction = [0.0, 1.0]; + } + assert!(across.find("検知機構").is_empty()); + assert_eq!(across.find("検知 機構"), vec![0..4]); +} + +/// A line of one character starts level with the one before, not behind it, and still runs on. +#[test] +fn lines_of_one_character_run_on() { + let single = laid_out(&[(10.0, 10.0, "検"), (10.0, 25.0, "知"), (10.0, 40.0, "機")], 10.0); + assert_eq!(single.find("検知機"), vec![0..3]); +} + +/// A minimal PDF of `objects` (numbered from 1), each with its xref entry. +fn pdf(objects: &[String]) -> Vec { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes()); + } + let xref = bytes.len(); + bytes.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes()); + for offset in offsets { + bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes()); + bytes +} + +/// The same vertical Japanese as [`a_word_divided_between_vertical_columns_is_found_in_both`], +/// drawn by a page in an Identity-V font: extracted text runs on from the foot of the right +/// column to the head of the left one. +#[test] +fn a_word_divided_between_vertical_columns_of_a_page_is_found_in_both() { + use pdfcraft_render::{PageRenderer, RenderConfig, RenderRequest, RequestKind}; + let chars: Vec = "装置の検知機構です".chars().collect(); + let pairs: String = (1u32..).zip(&chars).map(|(code, c)| format!("<{code:04X}> <{:04X}> ", *c as u32)).collect(); + let to_unicode = format!( + "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def /CMapName /TestUnicode def /CMapType 2 def 1 begincodespacerange <0000> endcodespacerange {} beginbfchar {pairs}endbfchar endcmap CMapName currentdict /CMap defineresource pop end end", + chars.len() + ); + let content = "BT /F1 10 Tf 1 0 0 1 300 700 Tm <0001000200030004 0005> Tj 1 0 0 1 285 700 Tm <0006000700080009> Tj ET"; + let stream = |dict: &str, data: &str| format!("<< {dict} /Length {} >>\nstream\n{data}\nendstream", data.len()); + let bytes = pdf(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 800] /Resources << /Font << /F1 4 0 R >> >> /Contents 7 0 R >>".into(), + "<< /Type /Font /Subtype /Type0 /BaseFont /TestMincho /Encoding /Identity-V /DescendantFonts [5 0 R] /ToUnicode 6 0 R >>".into(), + "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /TestMincho /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 /CIDToGIDMap /Identity >>".into(), + stream("", &to_unicode), + stream("", content), + ]); + let mut renderer = PageRenderer::new(std::sync::Arc::new(bytes), RenderConfig::default()); + let result = renderer.render(RenderRequest { page: 0, kind: RequestKind::Text, ..Default::default() }); + assert!(result.error.is_none(), "{:?}", result.error); + let text = result.text.expect("text"); + assert_eq!(text.find("検知機構"), vec![3..7], "{:?}", text.plain_text()); + assert_eq!(text.line_rects(3..7).len(), 2, "one highlight per column"); + assert!(text.find("検知 機構").is_empty()); +} + +/// A line of Chinese or Japanese that ends in a zero width space drawn with no height still runs +/// on: the break is measured on the glyphs that draw the characters either side of it. +#[test] +fn a_line_end_after_an_invisible_glyph_with_no_height_still_joins() { + let mut text = laid_out(&[(10.0, 10.0, "検知\u{200B}"), (10.0, 25.0, "機構")], 10.0); + if let Some(g) = text.glyphs.get_mut(2) { + g.rect = [30.0, 15.0, 30.0, 15.0]; + g.quad = [[30.0, 15.0]; 4]; + } + assert_eq!(text.find("検知機構"), vec![0..5]); +} + +/// A long run of variation selectors, each a glyph of its own, is searched in time linear in the +/// page: the line-end check doesn't rescan the selectors before every glyph. +#[test] +fn a_long_run_of_variation_selectors_is_searched_quickly() { + let selectors: String = std::iter::once('葛').chain(std::iter::repeat_n('\u{FE00}', 200_000)).collect(); + let text = laid_out(&[(10.0, 10.0, &selectors)], 0.001); + let started = std::time::Instant::now(); + assert!(text.find("飾").is_empty()); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); +} + +/// A variation selector drawn as a glyph of its own with no height at a line end stays in the +/// searchable text, and the line still runs on: the break is measured on the letter it follows. +#[test] +fn a_line_end_after_a_zero_height_variation_selector_glyph_still_joins() { + let mut text = laid_out(&[(10.0, 10.0, "葛\u{E0100}"), (10.0, 25.0, "飾区")], 10.0); + if let Some(g) = text.glyphs.get_mut(1) { + g.rect = [20.0, 15.0, 20.0, 15.0]; + g.quad = [[20.0, 15.0]; 4]; + } + assert_eq!(text.find("葛\u{E0100}飾区"), vec![0..4]); +} + +/// The visible glyphs either side of a break are found however many invisible glyphs stand +/// between them and the break. +#[test] +fn a_line_end_after_many_invisible_glyphs_still_joins() { + let line: String = std::iter::once('検').chain(std::iter::once('知')).chain(std::iter::repeat_n('\u{200B}', 40)).collect(); + let mut text = laid_out(&[(10.0, 10.0, &line), (10.0, 25.0, "機構")], 10.0); + for g in text.glyphs.iter_mut().skip(2).take(40) { + let [x, y] = [g.rect[0], g.rect[1]]; + g.rect = [x, y, x, y]; + g.quad = [[x, y]; 4]; + } + assert_eq!(text.find("検知機構"), vec![0..44]); +} + +/// The reference search and `find` agree on a soft hyphen followed, before the line end, by an +/// empty glyph and an invisible one: both join the word. +#[test] +fn reference_and_find_join_past_empty_and_invisible_glyphs_after_a_soft_hyphen() { + let mut text = page(&["Mitar\u{AD}", "", "\u{200B}", "beiterinnen"]); + text.line_of = vec![0, 0, 0, 1]; + assert_eq!(text.find("Mitarbeiterinnen"), vec![0..4]); + assert_eq!(reference(&text, "Mitarbeiterinnen", false, false, usize::MAX), vec![0..4]); + + // A visible glyph after the soft hyphen ends the hyphenated word there: the line end stays a space. + let mut text = page(&["Mitar\u{AD}", "x", "", "\u{200B}", "beiterinnen"]); + text.line_of = vec![0, 0, 0, 0, 1]; + assert!(text.find("Mitarbeiterinnen").is_empty()); + assert!(reference(&text, "Mitarbeiterinnen", false, false, usize::MAX).is_empty()); + assert_eq!(text.find("Mitarx beiterinnen"), vec![0..5]); + assert_eq!(reference(&text, "Mitarx beiterinnen", false, false, usize::MAX), vec![0..5]); +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 7d3249ddb7..d45e64e6d5 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1406,8 +1406,8 @@ milestone = "M3" status = "shipped" commands = ["edit.find"] tools = ["text_find"] -evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable"] -notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Complex interleaved flows still use a geometric reading-order heuristic." +evidence = ["crates/ui-egui/tests/it/ui.rs::find_counts_matches_across_pages", "crates/ui-egui/tests/it/ui.rs::find_reports_no_matches", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/text/tests.rs::identity_v_columns_are_searchable_in_reading_order", "crates/render/src/text/tests.rs::toc_leaders_do_not_hide_word_gaps", "crates/render/src/text/tests.rs::short_tracked_ellipses_remain_searchable", "crates/render/tests/it/search.rs::find_skips_soft_hyphens_and_zero_width_characters", "crates/render/tests/it/search.rs::find_joins_a_word_divided_at_a_line_end_soft_hyphen", "crates/render/tests/it/search.rs::a_zero_width_space_is_a_word_boundary_for_whole_words", "crates/render/tests/it/search.rs::a_chinese_or_japanese_word_divided_by_a_line_end_is_found_on_both_lines", "crates/render/tests/it/search.rs::a_line_break_stays_a_space_beside_latin_or_hangul_and_between_side_by_side_text", "crates/render/tests/it/search.rs::a_word_divided_between_vertical_columns_is_found_in_both", "crates/render/tests/it/search.rs::each_unspaced_range_joins_and_spaced_scripts_do_not", "crates/render/tests/it/search.rs::a_soft_hyphen_and_a_chinese_or_japanese_line_end_combine"] +notes = "Horizontal and WMode 1 vertical text are grouped independently, including a horizontal heading above right-to-left vertical columns. Dotted TOC leaders preserve word spaces for phrase search (#3086), while short tracked ellipses remain intact and searchable; sparse lines and condensed OCR gaps below the existing height-based threshold remain limitations. Complex interleaved flows still use a geometric reading-order heuristic. Soft hyphens, zero width spaces and joiners, word joiners and U+FEFF inside a word are skipped, in the page and in the query; a skipped zero width space still ends a word for Whole words only. A line break reads as no word break when the line ends in a soft hyphen, or when it runs on between two Chinese or Japanese characters (East Asian Wide, Fullwidth or Halfwidth, not Hangul and not fullwidth Latin letters or digits), so a divided word is found, in horizontal lines and in vertical columns that continue to the left; the break stays a space beside Latin or Korean text and between text set side by side on one line or in staggered cells." [[feature]] id = "view.find-next-previous"