From 8e03cd016d42032b867fef6bad86bc69bf2c1920 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:30:09 +0100 Subject: [PATCH] craft-pdf: content streams (parse, serialize, matrices) from PDFCraft (#1852) Adds the content feature and cos::content: PDFCraft's crates/content, moved here so other apps can read and write page content streams. The parser keeps operator spans, skips and counts unreadable bytes, and keeps inline images whole; tests move with it. Refs #778 --- crates/pdf/Cargo.toml | 3 + crates/pdf/README.md | 6 + crates/pdf/src/cos.rs | 3 + crates/pdf/src/cos/content.rs | 785 ++++++++++++++++++++++++++++ crates/pdf/src/cos/content/tests.rs | 589 +++++++++++++++++++++ 5 files changed, 1386 insertions(+) create mode 100644 crates/pdf/src/cos/content.rs create mode 100644 crates/pdf/src/cos/content/tests.rs diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml index 266297f42..a04f030c1 100644 --- a/crates/pdf/Cargo.toml +++ b/crates/pdf/Cargo.toml @@ -23,6 +23,9 @@ write-raster = ["dep:flate2", "dep:roxmltree"] write-simple-text = ["write", "krilla/simple-text"] filters = ["dep:craft-stream-codecs", "dep:flate2", "dep:thiserror"] cos = ["crypt", "filters", "dep:chrono", "dep:js-sys", "dep:wasm-bindgen"] +# Content streams on the object layer (`cos::content`); inline images end where hayro-syntax's +# tokenizer ends them. +content = ["cos", "dep:hayro-syntax"] crypt = ["dep:aes", "dep:md-5", "dep:sha2", "dep:stringprep", "dep:zeroize", "dep:thiserror"] [dependencies] diff --git a/crates/pdf/README.md b/crates/pdf/README.md index 210b79849..b9e0129d3 100644 --- a/crates/pdf/README.md +++ b/crates/pdf/README.md @@ -16,6 +16,7 @@ No feature is enabled by default. | `filters` | The `/Filter` chain for editing and writing PDF files: decode and encode, with limits and accounting. No reader or writer dependency | | `crypt` | The standard security handler: decrypt, encrypt and create encryption. No reader or writer dependency | | `cos` | `crypt` and `filters` plus the object layer: open, edit and save PDF files (incremental and full saves). No reader or writer dependency | +| `content` | `cos` plus `cos::content`: parse content streams into operators with source spans, write them back, and the matrix arithmetic. Uses hayro-syntax's tokenizer for inline images | ## Reading and rendering @@ -133,6 +134,11 @@ let bytes = write_incremental(&doc, &SaveOptions::default())?; // append a revis let bytes = write_full(&doc, &SaveOptions::default())?; // rewrite what the trailer reaches ``` +With `content`, `cos::content` reads and writes page content streams. `parse` keeps each +operator's byte span and skips (and counts) what it cannot read instead of failing; inline images +stay whole, so `parse` → `serialize_ops` round-trips them. `Matrix` holds the arithmetic content +interpretation needs. + - Reads every cross-reference form (tables, streams, hybrid `/XRefStm`) and object streams. A file whose cross-reference data is wrong or missing is rebuilt by scanning for object headers, and each repair is recorded (`repair_log`). Malformed input gives `CosError`, never a panic. diff --git a/crates/pdf/src/cos.rs b/crates/pdf/src/cos.rs index 1084c8bf4..d67543041 100644 --- a/crates/pdf/src/cos.rs +++ b/crates/pdf/src/cos.rs @@ -13,6 +13,9 @@ mod budget; mod bytes; +/// Content streams: operators with source spans, inline images, the serializer, matrices. +#[cfg(feature = "content")] +pub mod content; mod document; mod object; pub mod page_labels; diff --git a/crates/pdf/src/cos/content.rs b/crates/pdf/src/cos/content.rs new file mode 100644 index 000000000..a3087972d --- /dev/null +++ b/crates/pdf/src/cos/content.rs @@ -0,0 +1,785 @@ +//! Content streams (ISO 32000-2 §7.8.2, §8, §9): parse a stream into operators that keep their +//! source spans, write operators back, and the matrix arithmetic content interpretation needs. +//! +//! Feature `content`. The parser is tolerant: bytes it cannot read are skipped (and counted in +//! [`Parsed::skipped`]) instead of failing the whole stream, and it never panics. Inline images +//! (`BI … ID … EI`) are kept whole, so a parse → serialize round trip preserves them. Where an +//! inline image ends is decided by hayro-syntax's tokenizer, the one the `raster` renderer reads +//! content with ("the renderer" below). + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use crate::cos::{Dict, Lexer, Object, PdfString, serialize}; + +#[cfg(test)] +mod tests; + +/// One operator with its operands. +#[derive(Clone, Debug, PartialEq)] +pub struct Op { + /// The operator keyword (`Tj`, `re`, `BI` for inline images…). + pub op: Vec, + pub operands: Vec, + /// Inline images: the image dictionary (`operands` is empty) and the data between `ID` and + /// `EI`. + pub inline: Option<(Dict, Vec)>, + /// The bytes of the source stream this operator came from (operands included). + pub span: std::ops::Range, +} + +impl Op { + pub fn new(op: &str, operands: Vec) -> Self { + Op { op: op.as_bytes().to_vec(), operands, inline: None, span: 0..0 } + } + + pub fn is(&self, op: &str) -> bool { + self.op == op.as_bytes() + } + + /// Operand `i` as a number. + pub fn num(&self, i: usize) -> Option { + self.operands.get(i).and_then(Object::as_f64) + } + + /// The last `n` operands as numbers (operators take their operands from the end). + pub fn nums(&self) -> Option<[f64; N]> { + let k = self.operands.len().checked_sub(N)?; + let mut out = [0.0; N]; + for (o, v) in out.iter_mut().zip(&self.operands[k..]) { + *o = v.as_f64()?; + } + Some(out) + } + + pub fn name(&self, i: usize) -> Option<&[u8]> { + self.operands.get(i).and_then(Object::as_name) + } + + /// The last operand as a name: the one `Do` and the other one-operand operators act on, since + /// the renderer takes operands from the end and ignores any surplus before them. + pub fn last_name(&self) -> Option<&[u8]> { + self.operands.last().and_then(Object::as_name) + } + + /// Operand `k` counted from the end (0 is the last), as the renderer reads an operator's + /// operands: `(x) /OC /MC0 BDC` marks with `/OC /MC0`, and `(x) /F1 10 Tf` selects `/F1`. + pub fn back(&self, k: usize) -> Option<&Object> { + let i = self.operands.len().checked_sub(k)?.checked_sub(1)?; + self.operands.get(i) + } + + /// [`Op::back`] as a name. + pub fn name_back(&self, k: usize) -> Option<&[u8]> { + self.back(k).and_then(Object::as_name) + } + + /// [`Op::back`] as a number. + pub fn num_back(&self, k: usize) -> Option { + self.back(k).and_then(Object::as_f64) + } +} + +/// A parsed stream. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Parsed { + pub ops: Vec, + /// Bytes that could not be read as operands or operators. + pub skipped: usize, + /// Inline images the renderer's tokenizer reads no image for (it stops, or finds no `EI` it + /// accepts), so their end here is a guess and the operators after them may not be the ones + /// the page draws; and images it reads whose parameters this parser can't, which are missing + /// from `ops` although the page draws them. + pub ambiguous_images: usize, + /// For [`Pieces::parse`]: per piece, whether a comment runs into it from the piece before + /// and whether one runs out of it into the next. Only boundaries are recorded, so the size + /// depends on the number of pieces, not of comments. + comment_edges: Vec<(bool, bool)>, +} + +fn is_ws(b: u8) -> bool { + matches!(b, b'\0' | b'\t' | b'\n' | b'\x0C' | b'\r' | b' ') +} + +fn is_delim(b: u8) -> bool { + matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') +} + +/// Longest unreadable number token retried from its next byte. Real numbers are far shorter. +const MAX_RETRIED_TOKEN: usize = 64; + +/// Parse a (decoded) content stream. +pub fn parse(data: &[u8]) -> Parsed { + parse_pieces(data, &[]) +} + +/// The operators of a (decoded) content stream, read one at a time, as [`parse`] reads them. A +/// caller that stops early or keeps only a summary never holds every operator of the stream. +pub fn ops(data: &[u8]) -> Ops<'_> { + Ops::new(data, &[]) +} + +/// Parse `data`, recording which of the piece boundaries `ends` (as in [`Pieces`]) a comment +/// crosses. +fn parse_pieces(data: &[u8], ends: &[usize]) -> Parsed { + let mut reader = Ops::new(data, ends); + let ops = reader.by_ref().collect(); + Parsed { ops, skipped: reader.skipped, ambiguous_images: reader.ambiguous_images, comment_edges: reader.comment_edges } +} + +/// The iterator [`ops`] returns. +pub struct Ops<'a> { + data: &'a [u8], + lx: Lexer<'a>, + operands: Vec, + start: Option, + skipped: usize, + ambiguous_images: usize, + ends: &'a [usize], + boundary: usize, + comment_edges: Vec<(bool, bool)>, +} + +impl<'a> Ops<'a> { + fn new(data: &'a [u8], ends: &'a [usize]) -> Self { + Ops { + data, + lx: Lexer::new(data, 0), + operands: Vec::new(), + start: None, + skipped: 0, + ambiguous_images: 0, + ends, + boundary: 0, + comment_edges: vec![(false, false); ends.len()], + } + } + + /// Bytes read so far that could not be read as operands or operators. + pub fn skipped(&self) -> usize { + self.skipped + } + + /// Inline images read so far that are ambiguous (as in [`Parsed::ambiguous_images`]): either + /// their end is this parser's guess, or the renderer reads them and this parser cannot. + pub fn ambiguous_images(&self) -> usize { + self.ambiguous_images + } +} + +impl Iterator for Ops<'_> { + type Item = Op; + + fn next(&mut self) -> Option { + let Ops { data, lx, operands, start, skipped, ambiguous_images, ends, boundary, comment_edges: edges } = self; + let (data, ends): (&[u8], &[usize]) = (data, ends); + loop { + let trivia = lx.pos; + lx.skip_ws(); + // Only the whitespace the lexer skips here holds top-level comments: a `%` inside a + // string, a name, an array or dictionary operand or inline image data is not one. + if *boundary < ends.len() { + comment_edges(data, trivia..lx.pos, ends, boundary, edges); + } + let Some(&b) = data.get(lx.pos) else { + // Operands left without an operator. + *skipped += operands.len(); + operands.clear(); + return None; + }; + let at = lx.pos; + let operand_start = matches!(b, b'/' | b'(' | b'<' | b'[' | b'+' | b'-' | b'.') || b.is_ascii_digit(); + if operand_start { + match lx.object() { + Ok(o) => { + start.get_or_insert(at); + operands.push(o); + } + Err(_) if b == b'(' => { + // An unterminated string runs to the end of the stream. + *skipped += data.len() - at; + lx.pos = data.len(); + } + Err(_) => { + let read = lx.pos.saturating_sub(at); + #[cfg(test)] + tests::RESCANNED.with(|c| c.set(c.get() + read)); + let number = matches!(b, b'+' | b'-' | b'.') || b.is_ascii_digit(); + if number && read > MAX_RETRIED_TOKEN { + // A long unreadable number is skipped whole: retrying from each of its + // bytes would read the rest of it again every time. + *skipped += read; + } else { + // Skip the offending byte and carry on. + lx.pos = at + 1; + *skipped += 1; + } + } + } + continue; + } + if is_delim(b) { + // A stray `)`, `>`, `]`, `{`, `}`. + lx.pos += 1; + *skipped += 1; + continue; + } + let kw = lx.token(); + if kw.is_empty() { + lx.pos = at + 1; + *skipped += 1; + continue; + } + match kw { + b"true" => { + start.get_or_insert(at); + operands.push(Object::Bool(true)); + } + b"false" => { + start.get_or_insert(at); + operands.push(Object::Bool(false)); + } + b"null" => { + start.get_or_insert(at); + operands.push(Object::Null); + } + b"BI" => { + let image = inline_image(data, at, lx.pos); + *ambiguous_images += usize::from(image.ambiguous); + let end = image.end; + lx.pos = end; + operands.clear(); + match image.image { + Some((dict, img)) => { + return Some(Op { + op: b"BI".to_vec(), + operands: Vec::new(), + inline: Some((dict, img)), + span: start.take().unwrap_or(at)..end, + }); + } + None => { + *skipped += end.saturating_sub(at); + *start = None; + } + } + } + _ => { + let s = start.take().unwrap_or(at); + return Some(Op { op: kw.to_vec(), operands: std::mem::take(operands), inline: None, span: s..lx.pos }); + } + } + } + } +} + +/// Record the piece boundaries crossed by the comments in `trivia` (skipped whitespace and +/// comments). `boundary` is the first boundary not yet passed; boundaries only move forward. +fn comment_edges(data: &[u8], trivia: std::ops::Range, ends: &[usize], boundary: &mut usize, edges: &mut [(bool, bool)]) { + let mut at = trivia.start; + while at < trivia.end { + if data.get(at) != Some(&b'%') { + at += 1; + continue; + } + let comment = at; + while at < trivia.end && !matches!(data.get(at), Some(b'\r' | b'\n')) { + at += 1; + } + // The separator of piece `boundary` is the last byte before its end. + while let Some(&end) = ends.get(*boundary) { + let separator = end.saturating_sub(1); + if separator >= at { + break; + } + if separator >= comment { + if let Some(edge) = edges.get_mut(*boundary) { + edge.1 = true; + } + // Includes a comment whose end of line is the next piece's first byte. + if let Some(edge) = edges.get_mut(boundary.saturating_add(1)) { + edge.0 = true; + } + } + *boundary += 1; + } + } +} + +/// An inline image as [`inline_image`] reads it. +struct Inline { + /// Its parameters and data; `None` when the parameters can't be read. + image: Option<(Dict, Vec)>, + /// The position after its `EI` (or the end of the stream). + end: usize, + /// The renderer's tokenizer reads no image here, so `end` is this parser's guess; or it reads + /// one whose parameters this parser can't, so `image` is `None` for an image the page draws. + ambiguous: bool, +} + +/// Read the inline image whose `BI` keyword starts at `bi` (its parameters start at `pos`). +/// +/// Filtered image data has no length, and `EI` can occur inside it, so where it ends is a +/// heuristic choice among `EI` candidates. The renderer's choice decides which operators the +/// page draws after the image, and redaction must see the same ones, so the end is taken from +/// the renderer's tokenizer (hayro-syntax) rather than worked out here by another rule. Only +/// when that tokenizer reads no image does this parser fall back to its own rules, and the +/// image is marked ambiguous. +fn inline_image(data: &[u8], bi: usize, pos: usize) -> Inline { + let mut lx = Lexer::new(data, pos); + let mut dict = Dict::new(); + let mut readable = true; + let mut stop = pos; + loop { + lx.skip_ws(); + if lx.eat_keyword(b"ID") { + break; + } + let at = lx.pos; + let entry = match lx.object() { + Ok(Object::Name(k)) => lx.object().ok().map(|v| (k, v)), + _ => None, + }; + let Some((k, v)) = entry else { + // A stream that ends inside the parameters is cut short, not unreadable: keep what was read. + if at >= data.len() { + return Inline { image: Some((dict, Vec::new())), end: data.len(), ambiguous: false }; + } + readable = false; + stop = at; + break; + }; + dict.set(k, v); + } + if let Some((start, end_data, end)) = renderer_image_end(data, bi) { + // Where the renderer ends the data at an `EI` found by its heuristic, one whitespace byte + // before that `EI` separates it from the data. + let mut samples = data.get(start..end_data).unwrap_or_default(); + if data.get(end_data..).is_some_and(|rest| rest.starts_with(b"EI")) + && let Some((last, rest)) = samples.split_last() + && is_ws(*last) + { + samples = rest; + } + // The renderer draws an image whose parameters this parser can't read: it is not + // represented here, so a redaction can't inspect or remove it and must not trust the page. + return Inline { image: readable.then(|| (dict, samples.to_vec())), end, ambiguous: !readable }; + } + if !readable { + // Without an `EI` after it, the image runs to the end of the stream. + return Inline { image: None, end: end_of_image(data, stop, b"").map_or(data.len(), |(_, end)| end), ambiguous: true }; + } + // One whitespace byte separates ID from the data. + let begin = (lx.pos + 1).min(data.len()); + let end = unfiltered_image_end(data, begin, &dict) + .or_else(|| declared_image_end(data, begin, &dict)) + .or_else(|| image_data_end(data, begin, filter_eod(&dict))); + let (image, end) = match end { + Some((end_data, end)) => (data.get(begin..end_data).unwrap_or_default().to_vec(), end), + None => (data.get(begin..).unwrap_or_default().to_vec(), data.len()), + }; + Inline { image: Some((dict, image)), end, ambiguous: true } +} + +/// Where the renderer's tokenizer ends the inline image whose `BI` starts at `bi`: the start and +/// end of its data and the position after its `EI`. `None` when it reads no image there (its +/// parameters don't read, or no `EI` passes its checks). +fn renderer_image_end(data: &[u8], bi: usize) -> Option<(usize, usize, usize)> { + use hayro_syntax::content::TypedIter; + use hayro_syntax::content::ops::{InlineImage, TypedInstruction}; + let tail = data.get(bi..)?; + let mut iter = TypedIter::new(tail); + let (start, len) = match iter.next()? { + TypedInstruction::InlineImage(InlineImage(stream)) => match stream.raw_data() { + // The data is a slice of `tail`: its offset is where it starts. + std::borrow::Cow::Borrowed(samples) => ((samples.as_ptr() as usize).checked_sub(tail.as_ptr() as usize)?, samples.len()), + std::borrow::Cow::Owned(_) => return None, + }, + _ => return None, + }; + let end_data = start.checked_add(len)?; + // `committed` is past the whitespace after `EI`; the image ends at the `EI`. + let mut end = iter.committed(); + while end > end_data && tail.get(end - 1).is_some_and(|b| is_ws(*b)) { + end -= 1; + } + if end < end_data.checked_add(2)? || tail.get(end - 2..end) != Some(b"EI") { + return None; + } + Some((bi.checked_add(start)?, bi.checked_add(end_data)?, bi.checked_add(end)?)) +} + +/// The end-of-data marker (`>` for ASCIIHex, `~>` for ASCII85) of the outer stream filter, if any. +fn filter_eod(dict: &Dict) -> &'static [u8] { + for key in [b"Filter".as_slice(), b"F"] { + if let Some(filter) = dict.get(key) { + let name = match filter { + Object::Name(name) => Some(name.as_slice()), + Object::Array(filters) => filters.first().and_then(Object::as_name), + _ => None, + }; + return match name { + Some(b"ASCIIHexDecode" | b"AHx") => b">", + Some(b"ASCII85Decode" | b"A85") => b"~>", + _ => b"", + }; + } + } + b"" +} + +/// An unfiltered image's samples determine its length, including byte padding per row. +/// Binary samples may themselves contain whitespace + `EI` + whitespace, so use the known +/// length when the actual terminator follows it. Unknown or damaged layouts keep the heuristic. +fn unfiltered_image_end(data: &[u8], begin: usize, dict: &Dict) -> Option<(usize, usize)> { + for key in [b"Filter".as_slice(), b"F"] { + if let Some(filter) = dict.get(key) + && !matches!(filter, Object::Null) + && !matches!(filter, Object::Array(filters) if filters.is_empty()) + { + return None; + } + } + for (full, short) in + [(b"Width".as_slice(), b"W".as_slice()), (b"Height", b"H"), (b"ColorSpace", b"CS"), (b"BitsPerComponent", b"BPC"), (b"ImageMask", b"IM")] + { + if let (Some(full), Some(short)) = (dict.get(full), dict.get(short)) + && full != short + { + return None; + } + } + let get = |full: &[u8], short: &[u8]| dict.get(full).or_else(|| dict.get(short)); + let width = usize::try_from(get(b"Width", b"W")?.as_int()?).ok().filter(|n| *n > 0)?; + let height = usize::try_from(get(b"Height", b"H")?.as_int()?).ok().filter(|n| *n > 0)?; + let mask = match get(b"ImageMask", b"IM") { + None | Some(Object::Null | Object::Bool(false)) => false, + Some(Object::Bool(true)) => true, + _ => return None, + }; + let bits = match get(b"BitsPerComponent", b"BPC") { + None | Some(Object::Null) => None, + Some(value) => Some(value.as_int()?), + }; + let (components, bits) = if mask { + if bits.is_some_and(|bits| bits != 1) { + return None; + } + (1usize, 1usize) + } else { + let (components, max_bits) = match get(b"ColorSpace", b"CS")? { + Object::Name(name) => match name.as_slice() { + b"DeviceGray" | b"G" => (1, 16), + b"DeviceRGB" | b"RGB" => (3, 16), + b"DeviceCMYK" | b"CMYK" => (4, 16), + _ => return None, + }, + Object::Array(space) if space.len() == 4 && matches!(space.first().and_then(Object::as_name), Some(b"Indexed" | b"I")) => (1, 8), + _ => return None, + }; + let bits = match bits? { + bits @ (1 | 2 | 4 | 8 | 16) => bits as usize, + _ => return None, + }; + if bits > max_bits { + return None; + } + (components, bits) + }; + let row = width.checked_mul(components)?.checked_mul(bits)?.div_ceil(8); + let end_data = begin.checked_add(row.checked_mul(height)?)?; + let after = data.get(end_data..)?; + let separator = after.iter().position(|b| !is_ws(*b))?; + if separator == 0 && data.get(end_data.checked_sub(1)?).is_some_and(|b| is_ws(*b)) { + return None; + } + let ei = end_data.checked_add(separator)?; + let end = ei.checked_add(2)?; + if data.get(ei..end) == Some(b"EI") && data.get(end).is_none_or(|b| is_ws(*b) || is_delim(*b)) { Some((end_data, end)) } else { None } +} + +/// The length declared by `/L` (or `/Length`, ISO 32000-2 §8.9.7), used when `EI` follows the +/// declared bytes after optional whitespace. A declared length without that `EI` is ignored. +fn declared_image_end(data: &[u8], begin: usize, dict: &Dict) -> Option<(usize, usize)> { + let length = dict.get(b"L").or_else(|| dict.get(b"Length"))?.as_int()?; + let end_data = begin.checked_add(usize::try_from(length).ok()?)?; + let after = data.get(end_data..)?; + let ei = end_data.checked_add(after.iter().position(|b| !is_ws(*b))?)?; + let end = ei.checked_add(2)?; + if data.get(ei..end) == Some(b"EI") && data.get(end).is_none_or(|b| is_ws(*b) || is_delim(*b)) { Some((end_data, end)) } else { None } +} + +/// Where inline image data of unknown length ends. A candidate `EI` followed by binary bytes before +/// the next `EI` lies inside the (compressed) data, unless another inline image starts first. +fn image_data_end(data: &[u8], begin: usize, eod: &[u8]) -> Option<(usize, usize)> { + let mut candidate = end_of_image(data, begin, eod)?; + loop { + let (end_data, end) = candidate; + let Some(next) = end_of_image(data, end, eod) else { + return Some(candidate); + }; + let between = data.get(end..next.0).unwrap_or_default(); + if !between.iter().any(|b| !b.is_ascii()) || starts_inline_image(between) { + return Some((end_data, end)); + } + candidate = next; + } +} + +/// Whether `bytes` hold a whitespace-delimited `BI` operator. +fn starts_inline_image(bytes: &[u8]) -> bool { + bytes + .windows(2) + .enumerate() + .any(|(i, pair)| pair == b"BI" && (i == 0 || is_ws(bytes[i - 1])) && bytes.get(i + 2).is_none_or(|b| is_ws(*b) || is_delim(*b))) +} + +/// Where inline image data starting at `begin` ends: (end of the data, position after `EI`). The +/// data ends at whitespace (or `eod`: `>` for ASCIIHex, `~>` for ASCII85) + `EI` + (whitespace, +/// a delimiter or the end). +fn end_of_image(data: &[u8], begin: usize, eod: &[u8]) -> Option<(usize, usize)> { + let mut i = begin; + while i + 2 <= data.len() { + if data[i] == b'E' + && data.get(i + 1) == Some(&b'I') + && (i == begin || is_ws(data[i - 1]) || (!eod.is_empty() && data[begin..i].ends_with(eod))) + && data.get(i + 2).is_none_or(|b| is_ws(*b) || is_delim(*b)) + { + let end_data = if i > begin && is_ws(data[i - 1]) { i - 1 } else { i }; + return Some((end_data, i + 2)); + } + i += 1; + } + None +} + +/// Write one operator (with a trailing newline). +pub fn write_op(op: &Op, out: &mut Vec) { + if let Some((dict, img)) = &op.inline { + out.extend_from_slice(b"BI"); + for (k, v) in dict.iter() { + out.push(b' '); + serialize(&Object::Name(k.clone()), out); + out.push(b' '); + serialize(v, out); + } + out.extend_from_slice(b" ID "); + out.extend_from_slice(img); + out.extend_from_slice(b"\nEI\n"); + return; + } + for o in &op.operands { + serialize(o, out); + out.push(b' '); + } + out.extend_from_slice(&op.op); + out.push(b'\n'); +} + +/// Write operators back to a content stream. +pub fn serialize_ops(ops: &[Op]) -> Vec { + let mut out = Vec::new(); + for op in ops { + write_op(op, &mut out); + } + out +} + +/// A page's content streams joined into the one stream they are. Writers may split it between +/// any two tokens (ISO 32000-2 §7.8.2), so an operator's operands can end one piece and its +/// keyword start the next: parse the pieces joined ([`Pieces::parse`]) and map positions back +/// with [`Pieces::piece_of`]. A space separates the pieces, as renderers join them (hayro, +/// MuPDF and Poppler alike): tokens never run together, and a comment at the end of a piece +/// runs on into the next one, up to its first end of line. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Pieces { + data: Vec, + /// Where each piece ends in `data`, after its separator. + ends: Vec, +} + +impl Pieces { + pub fn join>(pieces: &[D]) -> Self { + let mut data = Vec::new(); + let mut ends = Vec::with_capacity(pieces.len()); + for p in pieces { + data.extend_from_slice(p.as_ref()); + data.push(b' '); + ends.push(data.len()); + } + Pieces { data, ends } + } + + /// The joined bytes (the spans of [`Pieces::parse`]'s operators index into them). + pub fn data(&self) -> &[u8] { + &self.data + } + + /// How many pieces there are. + pub fn len(&self) -> usize { + self.ends.len() + } + + pub fn is_empty(&self) -> bool { + self.ends.is_empty() + } + + /// Parse the pieces as one stream. + pub fn parse(&self) -> Parsed { + parse_pieces(&self.data, &self.ends) + } + + /// Per piece, from `parsed` (this piece set's [`Pieces::parse`]): whether a comment runs + /// into it from the piece before, and whether one runs out of it into the next. A writer + /// that rewrites a piece without its comments must keep both: end the incoming comment + /// before its first operator, and leave a comment open at its end. + pub fn comment_edges(&self, parsed: &Parsed) -> Vec<(bool, bool)> { + let mut edges = parsed.comment_edges.clone(); + edges.resize(self.len(), (false, false)); + edges + } + + /// The piece holding joined position `at` (the last piece for positions past the end). + pub fn piece_of(&self, at: usize) -> usize { + self.ends.partition_point(|&e| e <= at).min(self.ends.len().saturating_sub(1)) + } + + /// The pieces an operator spans: (the one its first token is in, the one its keyword is in). + pub fn pieces_of(&self, op: &Op) -> (usize, usize) { + (self.piece_of(op.span.start), self.piece_of(op.span.end.saturating_sub(1))) + } + + /// The joined range of piece `i`, without its separator. + fn range(&self, i: usize) -> std::ops::Range { + let start = match i.checked_sub(1) { + None => 0, + Some(p) => self.ends.get(p).copied().unwrap_or(self.data.len()), + }; + let end = self.ends.get(i).map_or(self.data.len(), |e| e.saturating_sub(1)).max(start); + start..end + } + + /// Rebuild every piece after replacing joined byte ranges: `edits` are (range, bytes) in + /// ascending, non-overlapping order. A replacement goes into the piece its range starts in, + /// and the rest of a range that runs on into later pieces is removed from them, so an + /// operator split across pieces is replaced or removed whole. Everything else is copied byte + /// for byte. Returns every piece's new bytes (unchanged pieces come back equal). + pub fn splice(&self, edits: impl IntoIterator, Vec)>) -> Vec> { + let mut out: Vec> = (0..self.len()).map(|i| Vec::with_capacity(self.range(i).len())).collect(); + // Only the pieces from the one holding `from` up to `to`: one call per edit must not + // visit every piece. + let copy = |out: &mut Vec>, from: usize, to: usize| { + if from >= to { + return; + } + let first = self.piece_of(from); + for (i, piece) in out.iter_mut().enumerate().skip(first) { + #[cfg(test)] + tests::PIECE_VISITS.with(|c| c.set(c.get() + 1)); + let r = self.range(i); + if r.start >= to { + break; + } + let (a, b) = (from.max(r.start), to.min(r.end)); + if a < b { + piece.extend_from_slice(self.data.get(a..b).unwrap_or_default()); + } + } + }; + let mut at = 0; + for (range, bytes) in edits { + let start = range.start.clamp(at, self.data.len()); + copy(&mut out, at, start); + if !bytes.is_empty() + && let Some(piece) = out.get_mut(self.piece_of(start)) + { + piece.extend_from_slice(&bytes); + } + at = range.end.clamp(start, self.data.len()); + } + copy(&mut out, at, self.data.len()); + out + } +} + +/// A number operand, written as an integer when it is one. +pub fn num(v: f64) -> Object { + if v.fract() == 0.0 && v.abs() < 1e15 { Object::Int(v as i64) } else { Object::Real((v * 10_000.0).round() / 10_000.0) } +} + +pub fn string(bytes: Vec) -> Object { + Object::String(PdfString::literal(bytes)) +} + +/// An affine matrix `[a b c d e f]` (PDF order: a point maps to `(a·x + c·y + e, b·x + d·y + f)`). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Matrix(pub [f64; 6]); + +impl Default for Matrix { + fn default() -> Self { + Matrix::IDENTITY + } +} + +impl Matrix { + pub const IDENTITY: Matrix = Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]); + + pub fn translate(x: f64, y: f64) -> Self { + Matrix([1.0, 0.0, 0.0, 1.0, x, y]) + } + + pub fn from_operands(v: &[Object]) -> Option { + if v.len() != 6 { + return None; + } + let mut m = [0.0; 6]; + for (o, x) in m.iter_mut().zip(v) { + *o = x.as_f64()?; + } + Some(Matrix(m)) + } + + /// `self` then `then` (`self × then` in PDF's row-vector convention). + pub fn then(&self, then: &Matrix) -> Matrix { + let [a, b, c, d, e, f] = self.0; + let [a2, b2, c2, d2, e2, f2] = then.0; + Matrix([a * a2 + b * c2, a * b2 + b * d2, c * a2 + d * c2, c * b2 + d * d2, e * a2 + f * c2 + e2, e * b2 + f * d2 + f2]) + } + + pub fn apply(&self, x: f64, y: f64) -> (f64, f64) { + let [a, b, c, d, e, f] = self.0; + (a * x + c * y + e, b * x + d * y + f) + } + + pub fn invert(&self) -> Option { + let [a, b, c, d, e, f] = self.0; + let det = a * d - b * c; + if det == 0.0 || !det.is_finite() || det.abs() <= 1e-12 * ((a * d).abs() + (b * c).abs()) { + return None; + } + let (ia, ib, ic, id) = (d / det, -b / det, -c / det, a / det); + let inverse = [ia, ib, ic, id, -(e * ia + f * ic), -(e * ib + f * id)]; + // A determinant that underflows toward zero can still pass the relative check above, + // and dividing by it overflows: such a matrix has no usable inverse. + inverse.iter().all(|v| v.is_finite()).then_some(Matrix(inverse)) + } + + /// The bounding box of a rectangle `[x0 y0 x1 y1]` after this transform. + pub fn bbox(&self, r: [f64; 4]) -> [f64; 4] { + let pts = [self.apply(r[0], r[1]), self.apply(r[2], r[1]), self.apply(r[0], r[3]), self.apply(r[2], r[3])]; + let mut b = [f64::MAX, f64::MAX, f64::MIN, f64::MIN]; + for (x, y) in pts { + b = [b[0].min(x), b[1].min(y), b[2].max(x), b[3].max(y)]; + } + b + } +} + +/// Do two rectangles overlap by more than `eps` in both directions? +pub fn overlaps(a: [f64; 4], b: [f64; 4], eps: f64) -> bool { + a[0].max(b[0]) + eps < a[2].min(b[2]) && a[1].max(b[1]) + eps < a[3].min(b[3]) +} + +/// Is `inner` inside `outer` (within `eps`)? +pub fn contains(outer: [f64; 4], inner: [f64; 4], eps: f64) -> bool { + inner[0] >= outer[0] - eps && inner[1] >= outer[1] - eps && inner[2] <= outer[2] + eps && inner[3] <= outer[3] + eps +} diff --git a/crates/pdf/src/cos/content/tests.rs b/crates/pdf/src/cos/content/tests.rs new file mode 100644 index 000000000..6a44d51b1 --- /dev/null +++ b/crates/pdf/src/cos/content/tests.rs @@ -0,0 +1,589 @@ +use super::*; + +thread_local! { + /// Bytes the lexer read before an operand failed, on this thread. + pub(crate) static RESCANNED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +thread_local! { + /// Pieces [`Pieces::splice`] looked at while copying, on this thread. + pub(crate) static PIECE_VISITS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[test] +fn operands_are_counted_from_the_end() { + let op = parse(b"(x) /OC /MC0 BDC").ops.remove(0); + assert_eq!((op.name_back(1), op.name_back(0)), (Some(&b"OC"[..]), Some(&b"MC0"[..]))); + assert_eq!(op.back(2).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"x".to_vec())); + assert_eq!((op.back(3), op.back(usize::MAX)), (None, None)); + let op = parse(b"/F1 10 Tf").ops.remove(0); + assert_eq!((op.name_back(1), op.num_back(0), op.num_back(1)), (Some(&b"F1"[..]), Some(10.0), None)); + assert_eq!(Op::new("Q", vec![]).back(0), None); +} + +#[test] +fn splicing_visits_each_piece_a_few_times() { + let n = 5_000; + let pieces: Vec = (0..n).map(|i| format!("({i}) Tj")).collect(); + let joined = Pieces::join(&pieces); + let parsed = joined.parse(); + let edits: Vec<_> = parsed.ops.iter().map(|op| (op.span.clone(), b"(x) Tj".to_vec())).collect(); + assert_eq!(edits.len(), n); + PIECE_VISITS.with(|c| c.set(0)); + let out = joined.splice(edits); + let visits = PIECE_VISITS.with(|c| c.get()); + assert!(visits <= 4 * n, "{visits} piece visits for {n} pieces"); + assert!(out.iter().all(|p| p == b"(x) Tj")); + // A replacement spanning two pieces stays whole in the first and leaves the rest of the second. + let two = Pieces::join(&["q (a)", "Tj Q"]); + let tj = two.parse().ops.into_iter().find(|o| o.is("Tj")).map(|o| o.span).unwrap_or_default(); + assert_eq!(two.splice([(tj, b"(b) Tj".to_vec())]), [b"q (b) Tj".to_vec(), b" Q".to_vec()]); +} + +#[test] +fn a_long_unreadable_number_is_read_once() { + for junk in ["+", "5"] { + let n = 20_000; + let src = format!("{}{} 1 0 0 1 0 0 cm", junk.repeat(n), if junk == "5" { "x" } else { "" }); + RESCANNED.with(|c| c.set(0)); + let p = parse(src.as_bytes()); + let rescanned = RESCANNED.with(|c| c.get()); + assert!(rescanned <= 2 * n, "{rescanned} bytes read for a {n}-byte token"); + assert_eq!(p.ops.last().and_then(|o| o.nums::<6>()), Some([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])); + } + // A short damaged number still recovers byte by byte. + let p = parse(b"-+5 0 Td"); + assert_eq!((p.ops.len(), p.skipped), (1, 1)); + assert_eq!(p.ops[0].nums::<2>(), Some([5.0, 0.0])); +} + +#[test] +fn operators_keep_operands_and_spans() { + let src = b"q 1 0 0 1 72 720 cm BT /F1 12 Tf (Hello \\(world\\)) Tj [(A) -120 (B)] TJ ET Q"; + let p = parse(src); + assert_eq!(p.skipped, 0); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..], b"cm", b"BT", b"Tf", b"Tj", b"TJ", b"ET", b"Q"]); + let cm = &p.ops[1]; + assert_eq!(cm.nums::<6>(), Some([1.0, 0.0, 0.0, 1.0, 72.0, 720.0])); + assert_eq!(&src[cm.span.clone()], b"1 0 0 1 72 720 cm"); + assert_eq!(p.ops[4].operands[0].as_string().unwrap().bytes, b"Hello (world)"); + assert_eq!(p.ops[5].operands[0].as_array().unwrap().len(), 3); + assert_eq!(p.ops[3].name(0), Some(&b"F1"[..])); +} + +#[test] +fn round_trips_through_the_serializer() { + let src = b"0.5 g 10 10 100 50 re f /GS1 gs true false null 3 d0 [3 2] 0 d (x) ' 1 2 (y) \""; + let a = parse(src); + let b = parse(&serialize_ops(&a.ops)); + assert_eq!(a.ops.iter().map(|o| (&o.op, &o.operands)).collect::>(), b.ops.iter().map(|o| (&o.op, &o.operands)).collect::>()); +} + +#[test] +fn inline_images_stay_whole() { + let mut src = b"q 10 0 0 10 0 0 cm BI /W 2 /H 2 /CS /G /BPC 8 ID ".to_vec(); + src.extend_from_slice(&[0, b'E', b'I', 255]); + src.extend_from_slice(b"\nEI Q 1 g"); + let p = parse(&src); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..], b"cm", b"BI", b"Q", b"g"]); + let (d, data) = p.ops[2].inline.clone().unwrap(); + assert_eq!(d.int(b"W"), Some(2)); + assert_eq!(data, [0, b'E', b'I', 255], "an EI inside the data without whitespace before it is data"); + let again = parse(&serialize_ops(&p.ops)); + assert_eq!(again.ops[2].inline, p.ops[2].inline); +} + +#[test] +fn unfiltered_inline_image_keeps_whitespace_bounded_ei_pixels() { + let src = b"q 4 0 0 1 0 0 cm BI /W 4 /H 1 /BPC 8 /CS /G ID EI \nEI Q"; + let parsed = parse(src); + assert_eq!(parsed.ops[2].inline.as_ref().unwrap().1, b" EI "); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), [b"q".as_slice(), b"cm", b"BI", b"Q"]); + let rewritten = serialize_ops(&parsed.ops); + assert_eq!(parse(&rewritten).ops[2].inline, parsed.ops[2].inline); +} + +fn inline_pixels(parameters: &[u8], pixels: &[u8]) -> Parsed { + let mut src = b"q BI ".to_vec(); + src.extend_from_slice(parameters); + src.extend_from_slice(b" ID "); + src.extend_from_slice(pixels); + src.extend_from_slice(b"\nEI Q"); + parse(&src) +} + +#[test] +fn unfiltered_inline_image_lengths_use_components_and_parameter_aliases() { + for (parameters, pixels) in [ + (b"/W 4 /H 1 /CS /G /BPC 8".as_slice(), b" EI ".as_slice()), + (b"/W 4 /H 1 /CS /G /BPC 8 /Filter null", b" EI "), + (b"/W 4 /H 1 /CS /G /BPC 8 /F []", b" EI "), + (b"/W 4 /Width 4 /H 1 /CS /G /BPC 8", b" EI "), + (b"/Width 4 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b" EI "), + (b"/W 2 /H 1 /CS /RGB /BPC 8", b"\0 EI \xff"), + (b"/Width 2 /Height 1 /ColorSpace /DeviceRGB /BitsPerComponent 8", b"\0 EI \xff"), + (b"/W 1 /H 1 /CS /CMYK /BPC 8", b" EI "), + (b"/Width 1 /Height 1 /ColorSpace /DeviceCMYK /BitsPerComponent 8", b" EI "), + ] { + let parsed = inline_pixels(parameters, pixels); + assert_eq!(parsed.skipped, 0); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), [b"q".as_slice(), b"BI", b"Q"]); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, pixels); + assert_eq!(parse(&serialize_ops(&parsed.ops)).ops[1].inline, parsed.ops[1].inline); + } +} + +#[test] +fn unfiltered_inline_image_rows_are_byte_aligned_for_every_sample_depth() { + // Two rows of two bytes each. Padding belongs to each row, not only the final row. + for (bits, width) in [(1, 9), (2, 5), (4, 3), (8, 2), (16, 1)] { + let parameters = format!("/W {width} /H 2 /BPC {bits} /CS /G"); + let parsed = inline_pixels(parameters.as_bytes(), b" EI "); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI ", "bits={bits}"); + assert_eq!(parsed.ops.len(), 3); + } +} + +#[test] +fn unfiltered_inline_masks_default_to_one_bit_and_indexed_images_have_one_component() { + for parameters in [b"/W 9 /H 2 /IM true".as_slice(), b"/Width 9 /Height 2 /ImageMask true /BitsPerComponent 1"] { + let parsed = inline_pixels(parameters, b" EI "); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI "); + assert_eq!(parsed.ops.len(), 3); + } + for name in ["Indexed", "I"] { + // A valid 256-entry RGB lookup table; samples are indices regardless of the base space. + let parameters = format!("/W 4 /H 1 /BPC 8 /CS [/{name} /DeviceRGB 255 <{}>]", "00".repeat(768)); + let parsed = inline_pixels(parameters.as_bytes(), b" EI "); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI "); + assert_eq!(parsed.ops.len(), 3); + // Indexed samples permit at most 8 bits, even though device spaces permit 16, so the + // length is unknown, and the renderer takes the second `EI`. + let invalid = parameters.replace("/W 4", "/W 2").replace("/BPC 8", "/BPC 16"); + let parsed = inline_pixels(invalid.as_bytes(), b" EI "); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b" EI "); + } +} + +#[test] +fn unfiltered_inline_image_terminator_must_follow_its_samples() { + for separator in [b" ".as_slice(), b"\r\n", b"\0\t\x0c\n"] { + let mut src = b"BI /W 4 /H 1 /BPC 8 /CS /G ID EI ".to_vec(); + src.extend_from_slice(separator); + src.extend_from_slice(b"EI/Q q"); + let parsed = parse(&src); + assert_eq!(parsed.ops[0].inline.as_ref().unwrap().1, b" EI "); + assert_eq!(&src[parsed.ops[0].span.clone()], &src[..src.len() - b"/Q q".len()]); + } + let parsed = parse(b"BI /W 4 /H 1 /BPC 8 /CS /G ID EI \nEI"); + assert_eq!(parsed.ops[0].inline.as_ref().unwrap().1, b" EI "); + // EI is a keyword: a prefix of a larger token is not a terminator. + let parsed = parse(b"BI /W 4 /H 1 /BPC 8 /CS /G ID abcd EIx\nEI Q"); + assert_eq!(parsed.ops[0].inline.as_ref().unwrap().1, b"abcd EIx"); +} + +#[test] +fn filtered_unknown_and_damaged_inline_images_end_where_the_renderer_ends_them() { + for parameters in [ + b"/W 4 /H 1 /BPC 8 /CS /G /F /A85".as_slice(), + b"/W 4 /H 1 /BPC 8 /CS /ResourceColor", + b"/W 0 /H 1 /BPC 8 /CS /G", + b"/W -1 /H 1 /BPC 8 /CS /G", + b"/W 4 /H -1 /BPC 8 /CS /G", + b"/W 4 /H 1 /BPC 3 /CS /G", + b"/W 4 /H 1 /IM true /BPC 8", + b"/W 4 /H 1 /IM true /BPC /Bad", + b"/Width 4 /W 1 /H 1 /BPC 8 /CS /G", + b"/W 4 /H 1 /BPC 8 /CS /G /ImageMask false /IM true", + b"/W 9223372036854775807 /H 9223372036854775807 /BPC 16 /CS /CMYK", + b"/W 9223372036854775807 /H 2 /BPC 8 /CS /G", + ] { + // What follows the first `EI` (`!!~>`) does not read as content, so the renderer takes + // the later one. + let parsed = inline_pixels(parameters, b"! EI !!~>"); + let label = String::from_utf8_lossy(parameters); + assert_eq!(parsed.ops[1].inline.as_ref().unwrap().1, b"! EI !!~>", "{label}"); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), [b"q".as_slice(), b"BI", b"Q"], "{label}"); + assert_eq!(parsed.ambiguous_images, 0, "{label}"); + } + // A short payload is read whole; without an `EI`, the renderer reads no image and this + // parser's end is a guess. + for (src, ambiguous) in [(b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc\nEI Q".as_slice(), 0), (b"BI /W 4 /H 1 /BPC 8 /CS /G ID abc", 1)] { + let parsed = parse(src); + assert_eq!(parsed.ops[0].inline.as_ref().unwrap().1, b"abc"); + assert_eq!(parsed.ambiguous_images, ambiguous); + let mut reader = ops(src); + reader.by_ref().for_each(drop); + assert_eq!(reader.ambiguous_images(), ambiguous, "the streaming reader counts it too"); + } +} + +/// The renderer reads an image whose parameters this parser can't (a stray operand before `ID`): +/// the page draws an image `ops` doesn't hold, so it is counted as ambiguous rather than dropped +/// silently. +#[test] +fn an_inline_image_read_by_the_renderer_but_not_here_is_ambiguous() { + let parsed = parse(b"q BI /W 1 /H 1 /BPC 8 /CS /G 7 ID \x80 EI Q"); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), [b"q".as_slice(), b"Q"]); + assert_eq!(parsed.ambiguous_images, 1); +} + +/// A zlib stream (one stored block) for 64 DeviceGray samples whose bytes hold " EI\n" at offset 41, +/// the way a stored deflate block copies incompressible samples verbatim (#3579). +const FLATE_WITH_EI: &str = "789c014000bfffed746159d50788f7ea604984cd08e00b4c8983207f56fe82c77ef8dfb8b9a3a586382045490a3120\ + 31206c2025630aaa4353e6813e233b96539c18bc3a2c5009e9d51b81"; + +#[test] +fn flate_inline_image_is_not_cut_at_an_ei_inside_its_compressed_bytes() { + let flate: Vec = (0..FLATE_WITH_EI.len()).step_by(2).map(|i| u8::from_str_radix(&FLATE_WITH_EI[i..i + 2], 16).unwrap()).collect(); + assert_eq!(flate.len(), 75); + assert_eq!(&flate[41..45], b" EI\n"); + for parameters in [b"/W 64 /H 1 /BPC 8 /CS /G /F /Fl".as_slice(), b"/W 64 /H 1 /BPC 8 /CS /G /F /Fl /L 75"] { + let mut src = b"q 200 0 0 200 100 300 cm BI ".to_vec(); + src.extend_from_slice(parameters); + src.extend_from_slice(b" ID "); + src.extend_from_slice(&flate); + src.extend_from_slice(b"\nEI Q BT /F1 24 Tf 72 200 Td (AFTER) Tj ET"); + let parsed = parse(&src); + let label = String::from_utf8_lossy(parameters); + assert_eq!(parsed.ops[2].inline.as_ref().unwrap().1.len(), 75, "{label}"); + let names: Vec<&[u8]> = parsed.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..], b"cm", b"BI", b"Q", b"BT", b"Tf", b"Td", b"Tj", b"ET"], "{label}"); + assert_eq!(parsed.skipped, 0, "{label}"); + assert_eq!(parse(&serialize_ops(&parsed.ops)).ops[2].inline, parsed.ops[2].inline, "{label}"); + } +} + +fn inline_pdf(content: &[u8], object_streams: bool) -> Vec { + use crate::cos::{Document, SaveOptions, Stream, write_full}; + + let mut doc = Document::new_empty(); + let pages = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Pages").unwrap(); + let contents = doc.add(Object::Stream(Stream::flate(Dict::new(), content))); + let page = doc.add(Object::Dict( + [ + (b"Type".to_vec(), Object::name("Page")), + (b"Parent".to_vec(), Object::Ref(pages)), + (b"MediaBox".to_vec(), Object::Array([0, 0, 4, 1].map(Object::Int).to_vec())), + (b"Resources".to_vec(), Object::Dict(Dict::new())), + (b"Contents".to_vec(), Object::Ref(contents)), + ] + .into_iter() + .collect(), + )); + doc.update_dict(pages, |dict| { + dict.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(page)])); + dict.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap() +} + +#[test] +fn rewritten_inline_image_pixels_survive_saving_and_reopening_a_pdf() { + use crate::cos::{Document, Object}; + use std::sync::Arc; + + let original = b"q 4 0 0 1 0 0 cm BI /W 4 /H 1 /BPC 8 /CS /G ID EI \nEI Q"; + let rewritten = serialize_ops(&parse(original).ops); + for object_streams in [false, true] { + let bytes = inline_pdf(&rewritten, object_streams); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert!(doc.repair_log().is_empty()); + let pages = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Pages").unwrap(); + let pages = doc.get(pages); + let page = doc.resolve(&pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap()[0]); + let content = doc.resolve(page.as_dict().unwrap().get(b"Contents").unwrap()); + let Object::Stream(stream) = &*content else { panic!("page content stream") }; + assert_eq!(parse(&stream.decoded_strict().unwrap()).ops[2].inline.as_ref().unwrap().1, b" EI "); + // Opt-in artifacts for independent readers/renderers; synthetic PDFs are never committed. + if let Some(dir) = std::env::var_os("PDFCRAFT_INLINE_FIXTURE_DIR") { + let dir = std::path::PathBuf::from(dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("original.pdf"), inline_pdf(original, false)).unwrap(); + std::fs::write(dir.join(if object_streams { "rewritten-packed.pdf" } else { "rewritten-classic.pdf" }), bytes).unwrap(); + } + } +} + +#[test] +fn damage_is_skipped_not_fatal() { + let p = parse(b"1 0 0 RG ) ] 10 20 m 30 40 l S (unterminated"); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"RG"[..], b"m", b"l", b"S"]); + assert!(p.skipped >= 2); + assert!(parse(b"").ops.is_empty()); + assert!(parse(b"BI /W 1").ops.len() == 1); +} + +/// Issue #1318: an inline image whose dictionary can't be read is skipped up to its `EI`, counted, +/// and what follows it is parsed. +#[test] +fn an_unreadable_inline_image_is_skipped_to_its_end() { + let src = b"q BI /W 4 H 4 ID aa EI Q BT (After) Tj ET"; + let p = parse(src); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..], b"Q", b"BT", b"Tj", b"ET"]); + assert_eq!(p.skipped, b"BI /W 4 H 4 ID aa EI".len()); + let control = parse(b"q BI /W 4 /H 4 ID aa EI Q BT (After) Tj ET"); + assert_eq!(control.skipped, 0); + assert_eq!(control.ops.len(), 6); +} + +/// Issue #1318, follow-up: without an `EI`, an unreadable inline image runs to the end of the +/// stream, and those bytes are counted. A readable one whose data is cut short is kept, data and +/// all, so nothing of it is skipped. +#[test] +fn an_unreadable_inline_image_without_ei_is_counted() { + let p = parse(b"q BI /W 4 H 4 ID aa"); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..]]); + assert_eq!(p.skipped, b"BI /W 4 H 4 ID aa".len()); + let control = parse(b"q BI /W 4 /H 4 ID aa"); + assert_eq!(control.skipped, 0); + let names: Vec<&[u8]> = control.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"q"[..], b"BI"]); + assert_eq!(control.ops[1].inline.as_ref().map(|(_, data)| data.as_slice()), Some(&b"aa"[..])); +} + +#[test] +fn matrices_compose_and_invert() { + let a = Matrix([2.0, 0.0, 0.0, 3.0, 10.0, 20.0]); + let b = Matrix([0.0, 1.0, -1.0, 0.0, 5.0, 0.0]); + let ab = a.then(&b); + let (x, y) = ab.apply(1.0, 1.0); + let (x1, y1) = a.apply(1.0, 1.0); + assert_eq!((x, y), b.apply(x1, y1)); + let inv = ab.invert().unwrap(); + let (u, v) = inv.apply(x, y); + assert!((u - 1.0).abs() < 1e-9 && (v - 1.0).abs() < 1e-9); + assert_eq!(Matrix([2.0, 0.0, 0.0, 2.0, 1.0, 1.0]).bbox([0.0, 0.0, 1.0, 1.0]), [1.0, 1.0, 3.0, 3.0]); + assert!(overlaps([0.0, 0.0, 10.0, 10.0], [5.0, 5.0, 20.0, 20.0], 0.0)); + assert!(!overlaps([0.0, 0.0, 10.0, 10.0], [10.0, 0.0, 20.0, 10.0], 0.0)); +} + +#[test] +fn pieces_parse_as_one_stream_and_splice_back() { + // An operator's operands end one piece and its keyword starts the next (§7.8.2). + let pieces = Pieces::join(&[&b"BT 72 700 Td [(After) -20 (wards)]"[..], b"TJ ET", b"q Q"]); + assert_eq!(pieces.len(), 3); + let ops = pieces.parse().ops; + let names: Vec<&[u8]> = ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"BT"[..], b"Td", b"TJ", b"ET", b"q", b"Q"]); + assert_eq!(pieces.pieces_of(&ops[2]), (0, 1)); + assert_eq!(pieces.pieces_of(&ops[3]), (1, 1)); + assert_eq!(pieces.piece_of(usize::MAX), 2); + // Nothing changed: the pieces come back byte for byte. + assert_eq!(pieces.splice([]), [&b"BT 72 700 Td [(After) -20 (wards)]"[..], b"TJ ET", b"q Q"]); + // Replacing the split TJ rewrites both pieces it spans; the third is untouched. + let out = pieces.splice([(ops[2].span.clone(), b"(New) Tj".to_vec())]); + assert_eq!(out, [&b"BT 72 700 Td (New) Tj"[..], b" ET", b"q Q"]); + // Removing it leaves no stray operands or keyword. + let out = pieces.splice([(ops[2].span.clone(), Vec::new())]); + assert_eq!(out, [&b"BT 72 700 Td "[..], b" ET", b"q Q"]); + // Out-of-range or reversed edits are clamped, never a panic. + let out = pieces.splice([(std::ops::Range { start: 10, end: 5 }, b"x".to_vec()), (1000..2000, Vec::new())]); + assert_eq!(out.len(), 3); + assert!(Pieces::join::<&[u8]>(&[]).splice([(0..1, b"x".to_vec())]).is_empty()); +} + +#[test] +fn stream_boundary_comment_matches_space_concatenation() { + for eol in [b"\n".as_slice(), b"\r", b"\r\n"] { + let mut tail = b"(HIDDEN) Tj".to_vec(); + tail.extend(eol); + tail.extend(b"(LIVE) Tj"); + let pieces = Pieces::join(&[b"BT % comment".as_slice(), b"(ALSO HIDDEN) Tj", &tail, b"ET"]); + let parsed = pieces.parse(); + let shown: Vec<_> = parsed.ops.iter().filter(|op| op.is("Tj")).collect(); + assert_eq!(shown.len(), 1); + assert_eq!(shown[0].operands[0].as_string().unwrap().bytes, b"LIVE"); + } +} + +#[test] +fn comment_edges_terminate_at_next_piece_byte_zero() { + for eol in [b"\n".as_slice(), b"\r", b"\r\n"] { + let mut next = eol.to_vec(); + next.extend_from_slice(b"q Q"); + let pieces = Pieces::join(&[b"% outgoing".as_slice(), next.as_slice()]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, true), (true, false)], "EOL {eol:?}"); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), vec![b"q".as_slice(), b"Q"]); + } +} + +#[test] +fn comment_edges_span_empty_pieces_and_end_of_content() { + let pieces = Pieces::join(&[b"q % outgoing".as_slice(), b"", b"", b"ignored\nQ"]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, true), (true, true), (true, true), (true, false)]); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), vec![b"q".as_slice(), b"Q"]); + + let pieces = Pieces::join(&[b"% outgoing".as_slice(), b"", b"still ignored"]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, true), (true, true), (true, true)]); + assert!(parsed.ops.is_empty()); + + let pieces = Pieces::join::<&[u8]>(&[]); + assert!(pieces.comment_edges(&pieces.parse()).is_empty()); +} + +#[test] +fn comment_edges_ignore_percent_inside_operands_and_image_data() { + for first in [ + b"(literal % not a comment) Tj".as_slice(), + b"/Percent#25Name MP", + b"[(literal % not a comment)] TJ", + b"/P << /ActualText (literal % not a comment) >> BDC EMC", + b"BI /W 1 /H 1 /BPC 8 /CS /G ID % EI", + ] { + let pieces = Pieces::join(&[first, b"q Q"]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, false), (false, false)], "{first:?}"); + assert_eq!(parsed.ops.last().unwrap().op, b"Q"); + assert_eq!(parsed.skipped, 0); + } +} + +#[test] +fn compound_operand_comment_split_is_owned_by_spanning_operator() { + for parts in [[b"[(A) % ignored".as_slice(), b"still ignored\n(B)] TJ"], [b"/P << /MCID 0 % ignored".as_slice(), b"still ignored\n>> BDC"]] { + let pieces = Pieces::join(&parts); + let parsed = pieces.parse(); + // The object lexer consumes this nested comment. Its containing operator spans + // both pieces, which forces both to be rewritten together by the redactor. + assert_eq!(pieces.comment_edges(&parsed), vec![(false, false), (false, false)]); + assert_eq!(parsed.ops.len(), 1); + assert_eq!(pieces.pieces_of(&parsed.ops[0]), (0, 1)); + assert_eq!(parsed.skipped, 0); + } +} + +#[test] +fn comment_edges_do_not_confuse_local_comments_with_boundary_comments() { + let pieces = Pieces::join(&[b"% local\nq % crossing".as_slice(), b"ignored\n% local again\nQ\n% final local\n", b"q Q"]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, true), (true, false), (false, false)]); + assert_eq!(parsed.ops.iter().map(|op| op.op.as_slice()).collect::>(), vec![b"q".as_slice(), b"Q", b"q", b"Q"]); +} + +#[test] +fn comment_edge_storage_depends_on_piece_count_not_comment_count() { + let mut first = b"%\n".repeat(16_384); + first.extend_from_slice(b"q % crossing"); + let pieces = Pieces::join(&[first.as_slice(), b"ignored\nQ"]); + let parsed = pieces.parse(); + assert_eq!(pieces.comment_edges(&parsed), vec![(false, true), (true, false)]); + assert_eq!(parsed.ops.len(), 2); + // Also exercise public parsing without piece-boundary metadata. + assert_eq!(parse(&first).ops.len(), 1); +} + +#[test] +fn piece_lookup_preserves_exact_boundary_and_empty_piece_mapping() { + // Joined layout: qQ; separator belongs to prior piece. + let pieces = Pieces::join(&[b"q".as_slice(), b"", b"Q"]); + for (at, expected) in [(0, 0), (1, 0), (2, 1), (3, 2), (4, 2), (5, 2), (usize::MAX, 2)] { + assert_eq!(pieces.piece_of(at), expected, "position {at}"); + } + let empty = Pieces::join::<&[u8]>(&[]); + assert_eq!(empty.piece_of(0), 0); + assert_eq!(empty.piece_of(usize::MAX), 0); +} + +#[test] +fn inline_image_terminator_without_preceding_whitespace() { + let cases: &[(&[u8], &[u8])] = &[ + (b"BI /W 4 /H 1 /BPC 8 /CS /G /F /AHx ID 2060A0E0>EI Q BT /F1 18 Tf (After) Tj ET", b"2060A0E0>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /Filter /ASCIIHexDecode ID 2060A0E0>EI Q BT /F1 18 Tf (After) Tj ET", b"2060A0E0>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /F [/AHx] ID 2060A0E0>EI Q BT /F1 18 Tf (After) Tj ET", b"2060A0E0>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /F [/AHx /RL] ID 2060A0E0>EI Q BT /F1 18 Tf (After) Tj ET", b"2060A0E0>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /F /A85 ID 5s,^g~>EI Q BT /F1 18 Tf (After) Tj ET", b"5s,^g~>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /Filter /ASCII85Decode ID 5s,^g~>EI Q BT /F1 18 Tf (After) Tj ET", b"5s,^g~>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G /F [/A85 /RL] ID 5s,^g~>EI Q BT /F1 18 Tf (After) Tj ET", b"5s,^g~>"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G ID ab()EI Q BT /F1 18 Tf (After) Tj ET", b"ab()"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G ID ab[]EI Q BT /F1 18 Tf (After) Tj ET", b"ab[]"), + (b"BI /W 4 /H 1 /BPC 8 /CS /G ID \x20\x60\xa0\xe0EI Q BT /F1 18 Tf (After) Tj ET", b"\x20\x60\xa0\xe0"), + ]; + + for (src, expected_data) in cases { + let p = parse(src); + assert_eq!(p.skipped, 0, "stream should have 0 skipped bytes: {}", String::from_utf8_lossy(src)); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!( + names, + [&b"BI"[..], b"Q", b"BT", b"Tf", b"Tj", b"ET"], + "operators after inline image must be preserved: {}", + String::from_utf8_lossy(src) + ); + let (_, img) = p.ops[0].inline.as_ref().unwrap(); + assert_eq!(img, expected_data, "inline image data must match: {}", String::from_utf8_lossy(src)); + } +} + +#[test] +fn ascii85_inline_image_does_not_terminate_at_gt_ei_inside_payload() { + for src in [ + b"BI /W 4 /H 1 /BPC 8 /CS /G /F /A85 ID !!!!>EI !!~>\nEI Q BT /F1 18 Tf (After) Tj ET".as_slice(), + b"BI /W 4 /H 1 /BPC 8 /CS /G /F [/A85 /AHx] ID !!!!>EI !!~>\nEI Q BT /F1 18 Tf (After) Tj ET".as_slice(), + ] { + let p = parse(src); + assert_eq!(p.skipped, 0); + let names: Vec<&[u8]> = p.ops.iter().map(|o| o.op.as_slice()).collect(); + assert_eq!(names, [&b"BI"[..], b"Q", b"BT", b"Tf", b"Tj", b"ET"]); + let (_, img) = p.ops[0].inline.as_ref().unwrap(); + assert_eq!(img, b"!!!!>EI !!~>"); + } +} + +/// The operators the renderer's tokenizer reads from `data`. +fn renderer_operators(data: &[u8]) -> Vec> { + let mut iter = hayro_syntax::content::UntypedIter::new(data); + let mut out = Vec::new(); + while let Some(instruction) = iter.next() { + out.push(instruction.operator.as_ref().to_vec()); + } + out +} + +/// Inline images end where the renderer ends them, so both read the same operators after one: +/// an `EI` without whitespace before it in the data, upper-case text ending in "EI" after the +/// image, and an `EI` inside compressed bytes (#3579). +#[test] +fn inline_images_end_where_the_renderer_ends_them() { + let flate: Vec = (0..FLATE_WITH_EI.len()).step_by(2).map(|i| u8::from_str_radix(&FLATE_WITH_EI[i..i + 2], 16).unwrap()).collect(); + let mut stored = b"q BI /W 64 /H 1 /BPC 8 /CS /G /F /Fl ID ".to_vec(); + stored.extend_from_slice(&flate); + stored.extend_from_slice(b"\nEI Q BT /F1 24 Tf 72 200 Td (AFTER) Tj ET"); + let fixtures: [&[u8]; 6] = [ + b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7FEI 1 0 0 1 200 0 cm EI BT /F1 10 Tf 10 100 Td (SECRET) Tj ET", + b"BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F> EI BT /F1 10 Tf 10 100 Td (ZWEI) Tj (EIN FREI BEI) Tj ET", + b"q BI /W 2 /H 1 /CS /G /BPC 8 /F /AHx ID 7F7F> EI Q BT (DREI) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 00> EI", + b"BI /W 4 /H 1 /BPC 8 /CS /G /F /A85 ID ! EI !!~>\nEI Q", + b"q BI /W 4 /H 1 /BPC 8 /CS /G ID EI \nEI Q", + &stored, + ]; + for data in fixtures { + let parsed = parse(data); + let ours: Vec> = parsed.ops.iter().map(|op| op.op.clone()).collect(); + assert_eq!(ours, renderer_operators(data), "{}", String::from_utf8_lossy(data)); + assert_eq!((parsed.skipped, parsed.ambiguous_images), (0, 0), "{}", String::from_utf8_lossy(data)); + } +} + +#[test] +fn invert_accepts_small_scale_and_rejects_singular() { + let m = Matrix([1e-7, 0., 0., 1e-7, 0., 0.]); + let inv = m.invert().expect("a uniform 1e-7 scale is invertible"); + let id = m.then(&inv).0; + for (got, want) in id.iter().zip(Matrix::IDENTITY.0) { + assert!((got - want).abs() < 1e-9, "{id:?}"); + } + assert!(Matrix([1., 2., 2., 4., 0., 0.]).invert().is_none()); + assert!(Matrix([f64::NAN, 0., 0., 1., 0., 0.]).invert().is_none()); + // A subnormal scale passes the relative check but its inverse overflows (CodeRabbit on #5703). + assert!(Matrix([1e-310, 0., 0., 1., 0., 0.]).invert().is_none()); +}